From 9d5f177aa2a6bdff29fda254fc45c1d2bb670b39 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 4 Sep 2024 07:13:28 -0500 Subject: [PATCH 1/6] fix: Missed changes from keycloak 21 upgrade --- docker-compose.yaml | 1 - services/keycloak/startup-scripts/10-newrelic.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 1ca889a151..1562b9aa0b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -180,7 +180,6 @@ services: # - NEW_RELIC_LICENSE_KEY= # - NEW_RELIC_APP_NAME=keycloak-local volumes: - - "./services/keycloak/profile.properties:/opt/keycloak/standalone/configuration/profile.properties" - "./services/keycloak/startup-scripts:/opt/keycloak/startup-scripts" - "./services/keycloak/themes/lagoon:/opt/keycloak/themes/lagoon" - "./local-dev/keycloak:/lagoon/keycloak" diff --git a/services/keycloak/startup-scripts/10-newrelic.sh b/services/keycloak/startup-scripts/10-newrelic.sh index 9b930dd0a0..efe7065b72 100755 --- a/services/keycloak/startup-scripts/10-newrelic.sh +++ b/services/keycloak/startup-scripts/10-newrelic.sh @@ -4,7 +4,7 @@ if [ -v NEW_RELIC_LICENSE_KEY ] then echo "Enabling newrelic monitor" - cat << 'EOF' >> /opt/jboss/keycloak/bin/standalone.conf + cat << 'EOF' >> /opt/keycloak/bin/standalone.conf JAVA_OPTS="$JAVA_OPTS -javaagent:/opt/newrelic/newrelic.jar" EOF From 3b5f0dca9b728bb1e82f7824a3ed255512d41887 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 4 Sep 2024 07:13:29 -0500 Subject: [PATCH 2/6] Upgrade to keycloak 22.0.11 --- services/keycloak/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/keycloak/Dockerfile b/services/keycloak/Dockerfile index 0104cbb9dc..61a519f896 100644 --- a/services/keycloak/Dockerfile +++ b/services/keycloak/Dockerfile @@ -17,7 +17,7 @@ COPY javascript /tmp/lagoon-scripts RUN cd /tmp/lagoon-scripts && zip -r ../lagoon-scripts.jar * -FROM quay.io/keycloak/keycloak:21.1.2 +FROM quay.io/keycloak/keycloak:22.0.11 COPY --from=ubi-micro-build /mnt/rootfs / ARG LAGOON_VERSION From 4282a7497704ca662dba2ccc4e4d7da50c6ea416 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 4 Sep 2024 07:13:29 -0500 Subject: [PATCH 3/6] Upgrade to keycloak 23.0.7 --- services/keycloak/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/keycloak/Dockerfile b/services/keycloak/Dockerfile index 61a519f896..f881c9544f 100644 --- a/services/keycloak/Dockerfile +++ b/services/keycloak/Dockerfile @@ -17,7 +17,7 @@ COPY javascript /tmp/lagoon-scripts RUN cd /tmp/lagoon-scripts && zip -r ../lagoon-scripts.jar * -FROM quay.io/keycloak/keycloak:22.0.11 +FROM quay.io/keycloak/keycloak:23.0.7 COPY --from=ubi-micro-build /mnt/rootfs / ARG LAGOON_VERSION From 5e17b939c4fe075894a2377aa12517e840e20698 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 4 Sep 2024 07:13:30 -0500 Subject: [PATCH 4/6] Use latest keycloak-admin-client --- services/api/package.json | 3 +- services/api/src/clients/keycloak-admin.ts | 99 +++++++---------- services/api/src/models/group.ts | 2 +- services/api/src/models/user.ts | 2 +- .../api/src/resources/project/keycloak.ts | 32 ------ .../api/src/resources/project/resolvers.ts | 1 - yarn.lock | 102 ++++++------------ 7 files changed, 69 insertions(+), 172 deletions(-) delete mode 100644 services/api/src/resources/project/keycloak.ts diff --git a/services/api/package.json b/services/api/package.json index d26eb61363..8fb41801c1 100644 --- a/services/api/package.json +++ b/services/api/package.json @@ -22,6 +22,7 @@ "license": "MIT", "dependencies": { "@lagoon/commons": "4.0.0", + "@s3pweb/keycloak-admin-client-cjs": "^25.0.2", "@supercharge/request-ip": "^1.1.2", "apollo-server-express": "^2.14.2", "aws-sdk": "^2.378.0", @@ -46,7 +47,6 @@ "graphql-type-json": "^0.3.0", "graphql-upload": "^12.0.0", "jsonwebtoken": "^8.0.1", - "keycloak-admin": "https://github.com/amazeeio/keycloak-admin.git#bd015d2e34634f262c0827f00620657427e3c252", "keycloak-connect": "^5.0.0", "knex": "^3.0.1", "mariadb": "^2.5.2", @@ -69,7 +69,6 @@ "@types/jest": "^29.5.12", "@types/ramda": "types/npm-ramda#dist", "@types/redis": "^2.8.32", - "axios": "^0.21.1", "jest": "^29.7.0", "prettier": "^3.2.5", "prettier-eslint-cli": "^8.0.1", diff --git a/services/api/src/clients/keycloak-admin.ts b/services/api/src/clients/keycloak-admin.ts index 2466728ed4..db30576ae9 100644 --- a/services/api/src/clients/keycloak-admin.ts +++ b/services/api/src/clients/keycloak-admin.ts @@ -1,78 +1,51 @@ -import axios from 'axios'; +import { decode } from 'jsonwebtoken'; +import { KeycloakAdminClient } from '@s3pweb/keycloak-admin-client-cjs'; import { logger } from '../loggers/logger'; -const KeycloakAdmin = require('keycloak-admin').default; -const { Agent: KeycloakAgent } = require('keycloak-admin/lib/resources/agent'); import { config } from './keycloakClient'; -/** - * Everytime an API request is made, check if the access_token is (or will soon - * be) expired. If so, get a new token before making the request. - */ -class RetryAgent extends KeycloakAgent { - constructor(args) { - super(args); - } +export const getKeycloakAdminClient = async (): Promise => { + const keycloakAdminClient = new KeycloakAdminClient({ + baseUrl: `${config.origin}/auth`, + realmName: config.realm, + }); - request(args) { - const parentRequest = super.request(args); - return async payload => { - const accessToken = this.client.getAccessToken(); - const tokenRaw = Buffer.from(accessToken.split('.')[1], 'base64'); - const token = JSON.parse(tokenRaw.toString()); - const date = new Date(); - const now = Math.floor(date.getTime() / 1000); + /** + * Use a custom token provider that can automatically refresh expired tokens. + */ + keycloakAdminClient.registerTokenProvider({ + async getAccessToken() { - if (token.exp - 10 <= now) { - logger.debug('keycloakAdminClient: refreshing expired token'); - await this.client.auth(); - } + if (keycloakAdminClient.accessToken) { + const token = decode(keycloakAdminClient.accessToken); + const now = Math.floor(Date.now() / 1000); - return parentRequest(payload); - }; - } -} + if (token.exp - 30 > now) { + return keycloakAdminClient.accessToken; + } -/** - * Use our custom RetryAgent and save credentials internally for re-auth tasks. - */ -export class RetryKeycloakAdmin extends KeycloakAdmin { - constructor(connectionConfig, credentials) { - const agent = new RetryAgent({ - getUrlParams: client => ({ - realm: client.realmName - }), - getBaseUrl: client => client.baseUrl, - axios - }); + logger.debug('keycloakAdminClient: refreshing expired token'); + } - super(connectionConfig, agent); + // Always auth against master realm. + const curRealm = keycloakAdminClient.realmName; + keycloakAdminClient.setConfig({ + realmName: 'master', + }); - this.credentials = credentials; - } + await keycloakAdminClient.auth({ + username: config.user, + password: config.pass, + grantType: 'password', + clientId: 'admin-cli', + }); - async auth() { - const curRealm = this.realmName; - this.realmName = this.credentials.realmName; - await super.auth(this.credentials); - this.realmName = curRealm; - } -} + keycloakAdminClient.setConfig({ + realmName: curRealm, + }); -export const getKeycloakAdminClient = async () => { - const keycloakAdminClient = new RetryKeycloakAdmin( - { - baseUrl: `${config.origin}/auth`, - realmName: config.realm + return keycloakAdminClient.accessToken; }, - { - realmName: 'master', - username: config.user, - password: config.pass, - grantType: 'password', - clientId: 'admin-cli' - } - ); - await keycloakAdminClient.auth(); + }); return keycloakAdminClient; }; diff --git a/services/api/src/models/group.ts b/services/api/src/models/group.ts index bbbed3bcd1..531df251eb 100644 --- a/services/api/src/models/group.ts +++ b/services/api/src/models/group.ts @@ -3,7 +3,7 @@ import { Pool } from 'mariadb'; import { asyncPipe } from '@lagoon/commons/dist/util/func'; import pickNonNil from '../util/pickNonNil'; import { logger } from '../loggers/logger'; -import GroupRepresentation from 'keycloak-admin/lib/defs/groupRepresentation'; +import type { GroupRepresentation } from '@s3pweb/keycloak-admin-client-cjs'; import { User } from './user'; import { groupCacheExpiry, get, del, redisClient } from '../clients/redisClient'; import { Helpers as projectHelpers } from '../resources/project/helpers'; diff --git a/services/api/src/models/user.ts b/services/api/src/models/user.ts index 8b855a68aa..89b1b7fcf8 100644 --- a/services/api/src/models/user.ts +++ b/services/api/src/models/user.ts @@ -1,7 +1,7 @@ import * as R from 'ramda'; import pickNonNil from '../util/pickNonNil'; import { logger } from '../loggers/logger'; -import UserRepresentation from 'keycloak-admin/lib/defs/userRepresentation'; +import type { UserRepresentation } from '@s3pweb/keycloak-admin-client-cjs'; import { Group } from './group'; import { sqlClientPool } from '../clients/sqlClient'; import { query } from '../util/db'; diff --git a/services/api/src/resources/project/keycloak.ts b/services/api/src/resources/project/keycloak.ts deleted file mode 100644 index 507fe58ca2..0000000000 --- a/services/api/src/resources/project/keycloak.ts +++ /dev/null @@ -1,32 +0,0 @@ -import * as R from 'ramda'; -import { getKeycloakAdminClient } from '../../clients/keycloak-admin'; -import { logger } from '../../loggers/logger'; - -export const KeycloakOperations = { - findGroupIdByName: async (name: string) => { - const keycloakAdminClient = await getKeycloakAdminClient(); - - return R.path( - [0, 'id'], - await keycloakAdminClient.groups.find({ - search: name, - }), - ); - }, - deleteGroup: async (name: string) => { - const keycloakAdminClient = await getKeycloakAdminClient(); - - try { - // Find the Keycloak group id with the name - const keycloakGroupId = await KeycloakOperations.findGroupIdByName(name); - - // Delete the group - await keycloakAdminClient.groups.del({ id: keycloakGroupId }); - - logger.debug(`Deleted Keycloak group "${name}"`); - } catch (err) { - logger.error(`Error deleting Keycloak group: ${err}`); - throw new Error(`Error deleting Keycloak group: ${err}`); - } - }, -}; diff --git a/services/api/src/resources/project/resolvers.ts b/services/api/src/resources/project/resolvers.ts index 78eae6e5e3..787cefd81f 100644 --- a/services/api/src/resources/project/resolvers.ts +++ b/services/api/src/resources/project/resolvers.ts @@ -5,7 +5,6 @@ import { logger } from '../../loggers/logger'; import { knex, query, isPatchEmpty } from '../../util/db'; import { validateKey, generatePrivateKey as genpk } from '../../util/func'; import { Helpers } from './helpers'; -import { KeycloakOperations } from './keycloak'; import { OpendistroSecurityOperations } from '../group/opendistroSecurity'; import { Sql } from './sql'; import { Sql as SshKeySql} from '../sshKey/sql'; diff --git a/yarn.lock b/yarn.lock index 29308ab39d..18c35205d7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -712,6 +712,15 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@keycloak/keycloak-admin-client@25.0.2": + version "25.0.2" + resolved "https://registry.yarnpkg.com/@keycloak/keycloak-admin-client/-/keycloak-admin-client-25.0.2.tgz#4cd0447a4a190ed123a2efecd4078ec0f96add39" + integrity sha512-rHlJ1bQYQKLA3fnIktYZqofQP9imSPTia0yX7329fTpJ49A2MoPduAEFNsTa9QJIPHcuz2TBgZ6eI5NnfaP6/w== + dependencies: + camelize-ts "^3.0.0" + url-join "^5.0.0" + url-template "^3.1.1" + "@lagoon/lokka-transport-http@^1.6.1": version "1.6.1" resolved "https://registry.yarnpkg.com/@lagoon/lokka-transport-http/-/lokka-transport-http-1.6.1.tgz#2b2dc9a98a3702649696ae23524ad038722d2c55" @@ -907,6 +916,13 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== +"@s3pweb/keycloak-admin-client-cjs@^25.0.2": + version "25.0.2" + resolved "https://registry.yarnpkg.com/@s3pweb/keycloak-admin-client-cjs/-/keycloak-admin-client-cjs-25.0.2.tgz#c504f03bad786721db75e67921dfdd292c70c659" + integrity sha512-vPxrfR9znezbohaEWougqY5KHyOVd4z6whi82usFL0ahRvpaHenEZPPr6uopHx7CiQi7H1E5YeKQk8g2fGUw8g== + dependencies: + "@keycloak/keycloak-admin-client" "25.0.2" + "@sinclair/typebox@^0.27.8": version "0.27.8" resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e" @@ -1067,11 +1083,6 @@ resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.10.tgz#61cc8469849e5bcdd0c7044122265c39cec10cf4" integrity sha512-C7srjHiVG3Ey1nR6d511dtDkCEjxuN9W1HWAEjGq8kpcwmNM6JJkpC0xvabM7BXTG2wDq8Eu33iH9aQKa7IvLQ== -"@types/debug@^0.0.30": - version "0.0.30" - resolved "https://registry.yarnpkg.com/@types/debug/-/debug-0.0.30.tgz#dc1e40f7af3b9c815013a7860e6252f6352a84df" - integrity sha512-orGL5LXERPYsLov6CWs3Fh6203+dXzJkR7OnddIr2514Hsecwc8xRpzCapshBbKFImCsvS/mk6+FWiN5LyZJAQ== - "@types/express-serve-static-core@^4.17.21", "@types/express-serve-static-core@^4.17.33": version "4.17.43" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.43.tgz#10d8444be560cb789c4735aea5eac6e5af45df54" @@ -1914,14 +1925,6 @@ aws4@1.12.0, aws4@^1.12.0, aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.12.0.tgz#ce1c9d143389679e253b314241ea9aa5cec980d3" integrity sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg== -axios@^0.18.0: - version "0.18.1" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.18.1.tgz#ff3f0de2e7b5d180e757ad98000f1081b87bcea3" - integrity sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g== - dependencies: - follow-redirects "1.5.10" - is-buffer "^2.0.2" - axios@^0.21.1: version "0.21.4" resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" @@ -2103,7 +2106,7 @@ bluebird@^2.6.2: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-2.11.0.tgz#534b9033c022c9579c56ba3b3e5a5caafbb650e1" integrity sha512-UfFSr22dmHPQqPP9XWHRhq+gWnHCYguQGkXQlbyPtW5qTnhFWA8/iXg765tH0cAjy7l/zPJ1aBTO0g5XgA7kvQ== -bluebird@^3.5.1, bluebird@^3.5.2, bluebird@^3.7.2: +bluebird@^3.5.2, bluebird@^3.7.2: version "3.7.2" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== @@ -2313,10 +2316,10 @@ camelcase@^8.0.0: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-8.0.0.tgz#c0d36d418753fb6ad9c5e0437579745c1c14a534" integrity sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA== -camelize@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/camelize/-/camelize-1.0.1.tgz#89b7e16884056331a35d6b5ad064332c91daa6c3" - integrity sha512-dU+Tx2fsypxTgtLoE36npi3UqcjSSMNYfkqgmoEhtZrraP5VWq0K7FkWVTYa8eMPtnU/G2txVsfdCJTn9uzpuQ== +camelize-ts@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/camelize-ts/-/camelize-ts-3.0.0.tgz#b9a7b4ff802464dc3d6475637a64a9742ad3db09" + integrity sha512-cgRwKKavoDKLTjO4FQTs3dRBePZp/2Y9Xpud0FhuCOTE86M2cniKN4CCXgRnsyXNMmQMifVHcv6SPaMtTx6ofQ== caniuse-lite@^1.0.30001587: version "1.0.30001597" @@ -2747,14 +2750,7 @@ debug@4, debug@4.3.4, debug@^4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@ dependencies: ms "2.1.2" -debug@=3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" - integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== - dependencies: - ms "2.0.0" - -debug@^3.0.0, debug@^3.1.0, debug@^3.2.7: +debug@^3.0.0, debug@^3.2.7: version "3.2.7" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== @@ -3573,13 +3569,6 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@1.5.10: - version "1.5.10" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a" - integrity sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ== - dependencies: - debug "=3.1.0" - follow-redirects@^1.14.0: version "1.15.5" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.5.tgz#54d4d6d062c0fa7d9d17feb008461550e3ba8020" @@ -4349,11 +4338,6 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@^2.0.2: - version "2.0.5" - resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.5.tgz#ebc252e400d22ff8d77fa09888821a24a658c191" - integrity sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ== - is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.7: version "1.2.7" resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.7.tgz#3bc2a85ea742d9e36205dcacdd72ca1fdc51b055" @@ -5146,19 +5130,6 @@ jws@^4.0.0: jwa "^2.0.0" safe-buffer "^5.0.1" -"keycloak-admin@https://github.com/amazeeio/keycloak-admin.git#bd015d2e34634f262c0827f00620657427e3c252": - version "1.12.0" - resolved "https://github.com/amazeeio/keycloak-admin.git#bd015d2e34634f262c0827f00620657427e3c252" - dependencies: - "@types/debug" "^0.0.30" - axios "^0.18.0" - bluebird "^3.5.1" - camelize "^1.0.0" - debug "^3.1.0" - lodash "^4.17.10" - url-join "^4.0.0" - url-template "^2.0.8" - keycloak-connect@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/keycloak-connect/-/keycloak-connect-5.0.0.tgz#fa0ebfe1e1236381d0e1b94bfb1594dab5a21497" @@ -6761,11 +6732,6 @@ serialised-error@1.1.3: stack-trace "0.0.9" uuid "^3.0.0" -serialize-javascript@^2.1.1: - version "2.1.2" - resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-2.1.2.tgz#ecec53b0e0317bdc95ef76ab7074b7384785fa61" - integrity sha512-rs9OggEUF0V4jUSecXazOYsLfu7OGK2qIn3c7IPBiffz32XniEp/TX9Xmc9LQfK2nQ2QKHvZ2oygKUGU0lG4jQ== - serve-static@1.15.0: version "1.15.0" resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.15.0.tgz#faaef08cffe0a1a62f60cad0c4e513cff0ac9540" @@ -6935,14 +6901,6 @@ sqlstring@2.3.1: resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.1.tgz#475393ff9e91479aea62dcaf0ca3d14983a7fb40" integrity sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ== -srcset@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/srcset/-/srcset-1.0.0.tgz#a5669de12b42f3b1d5e83ed03c71046fc48f41ef" - integrity sha512-UH8e80l36aWnhACzjdtLspd4TAWldXJMa45NuOkTTU+stwekswObdqM63TtQixN4PPd/vO/kxLa6RD+tUPeFMg== - dependencies: - array-uniq "^1.0.2" - number-is-nan "^1.0.0" - sshpk@^1.14.1, sshpk@^1.7.0: version "1.18.0" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.18.0.tgz#1663e55cddf4d688b86a46b77f0d5fe363aba028" @@ -7526,10 +7484,10 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" -url-join@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/url-join/-/url-join-4.0.1.tgz#b642e21a2646808ffa178c4c5fda39844e12cde7" - integrity sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA== +url-join@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/url-join/-/url-join-5.0.0.tgz#c2f1e5cbd95fa91082a93b58a1f42fecb4bdbcf1" + integrity sha512-n2huDr9h9yzd6exQVnH/jU5mr+Pfx08LRXXZhkLLetAMESRj+anQsTAh940iMrIetKAmry9coFuZQ2jY8/p3WA== url-parse-lax@^3.0.0: version "3.0.0" @@ -7554,10 +7512,10 @@ url-parse@~1.4.3: querystringify "^2.1.1" requires-port "^1.0.0" -url-template@^2.0.8: - version "2.0.8" - resolved "https://registry.yarnpkg.com/url-template/-/url-template-2.0.8.tgz#fc565a3cccbff7730c775f5641f9555791439f21" - integrity sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw== +url-template@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/url-template/-/url-template-3.1.1.tgz#c220d5f3f793d28b0de341002112879cc8a43905" + integrity sha512-4oszoaEKE/mQOtAmdMWqIRHmkxWkUZMnXFnjQ5i01CuRSK3uluxcH1MRVVVWmhlnzT1SCDfKxxficm2G37qzCA== url@0.10.3: version "0.10.3" From c76e5ce4bcb3a9ac73e004f07de2f8825050388b Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 4 Sep 2024 07:13:31 -0500 Subject: [PATCH 5/6] Refactor Group model to account for keycloak API changes --- node-packages/commons/src/util/func.ts | 118 +-- services/api/src/apolloServer.js | 10 +- services/api/src/clients/keycloak-admin.ts | 16 + services/api/src/clients/keycloakClient.ts | 5 +- services/api/src/clients/redisClient.ts | 78 +- .../api/src/helpers/reset-project-keys.ts | 152 +-- .../sync-groups-opendistro-security.ts | 2 - .../3-remove-billing-data/billing-groups.ts | 50 - .../20240312000000_group_projects.ts | 2 - services/api/src/models/environment.ts | 210 ++-- services/api/src/models/group.ts | 994 +++++++++++------- services/api/src/models/project.ts | 54 - services/api/src/models/user.ts | 79 +- services/api/src/resources/index.ts | 28 +- .../src/resources/organization/resolvers.ts | 9 +- .../api/src/resources/project/resolvers.ts | 4 +- services/api/src/resources/sshKey/sql.ts | 4 +- services/api/src/resources/user/resolvers.ts | 4 +- 18 files changed, 967 insertions(+), 852 deletions(-) delete mode 100644 services/api/src/migrations/3-remove-billing-data/billing-groups.ts delete mode 100644 services/api/src/models/project.ts diff --git a/node-packages/commons/src/util/func.ts b/node-packages/commons/src/util/func.ts index ec4950d7cb..1856c5bb5e 100644 --- a/node-packages/commons/src/util/func.ts +++ b/node-packages/commons/src/util/func.ts @@ -1,6 +1,4 @@ import { unless, is, isNil, isEmpty, partialRight, complement } from 'ramda'; -import http from 'http'; -import querystring from 'querystring'; import { getConfigFromEnv } from './config'; export const isNumber = is(Number); @@ -13,85 +11,85 @@ export const notArray = complement(isArray); export const isNotNil = complement(isNil); export const isNotEmpty = complement(isEmpty); -export const asyncPipe = (...functions) => input => - functions.reduce((chain, func) => chain.then(func), Promise.resolve(input)); +export const asyncPipe = + (...functions) => + (input) => + functions.reduce((chain, func) => chain.then(func), Promise.resolve(input)); -export const jsonMerge = function(a, b, prop) { - var reduced = a.filter(function(aitem) { - return !b.find(function(bitem) { +export const jsonMerge = function (a, b, prop) { + var reduced = a.filter(function (aitem) { + return !b.find(function (bitem) { return aitem[prop] === bitem[prop]; }); }); return reduced.concat(b); -} +}; // will return only what is in a1 that isn't in a2 // eg: // a1 = [1,2,3,4] // a2 = [1,2,3,5] // arrayDiff(a1,a2) = [4] -export const arrayDiff = (a:Array, b:Array) => a.filter(e => !b.includes(e)); +export const arrayDiff = (a: Array, b: Array) => + a.filter((e) => !b.includes(e)); + +interface PublicKeyResponse { + error?: string; + publickey?: string; + type?: string; + value?: string; + sha256fingerprint?: string; + md5fingerprint?: string; + comment?: string; +} + +interface PrivateKeyResponse { + error: string; + publickey: string; + publickeypem: string; + sha256fingerprint: string; + md5fingerprint: string; + type: string; + value: string; + privatekeypem: string; +} // helper that will use the crypto handler service to check if a public or private key is valid or not -export async function validateKey(key, type) { - const data = querystring.stringify({'key': key}); - const options = { - hostname: getConfigFromEnv("SIDECAR_HANDLER_HOST", "localhost"), - port: 3333, - path: `/validate/${type}`, +export async function validateKey( + key: string, + type: 'public' | 'private', +): Promise { + let response = await fetch( + `http://${getConfigFromEnv('SIDECAR_HANDLER_HOST', 'localhost')}:3333/validate/${type}`, + { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', - 'Content-Length': Buffer.byteLength(data) }, - }; - let p = new Promise((resolve, reject) => { - const req = http.request(options, (res) => { - res.setEncoding('utf8'); - let responseBody = ''; + body: new URLSearchParams({ key: key }).toString(), + }, + ); - res.on('data', (chunk) => { - responseBody += chunk; - }); + if (!response.ok) { + throw new Error(`Error validating key: ${response.status}`); + } - res.on('end', () => { - resolve(JSON.parse(responseBody)); - }); - }); - req.on('error', (err) => { - reject(err); - }); - req.write(data) - req.end(); - }); - return await p; + if (type === 'public') { + return (await response.json()) as PublicKeyResponse; + } else { + return (await response.json()) as PrivateKeyResponse; + } } // helper that will use the crypto handler service to generate a private key with associated public key -export async function generatePrivateKey() { - const options = { - hostname: getConfigFromEnv("SIDECAR_HANDLER_HOST", "localhost"), - port: 3333, - path: '/generate/ed25519', - method: 'GET', - }; - let p = new Promise((resolve, reject) => { - const req = http.request(options, (res) => { - res.setEncoding('utf8'); - let responseBody = ''; +export async function generatePrivateKey(): Promise { + let response = await fetch( + `http://${getConfigFromEnv('SIDECAR_HANDLER_HOST', 'localhost')}:3333/generate/ed25519`, + ); - res.on('data', (chunk) => { - responseBody += chunk; - }); + if (!response.ok) { + throw new Error(`Error generating key: ${response.status}`); + } - res.on('end', () => { - resolve(JSON.parse(responseBody)); - }); - }); - req.on('error', (err) => { - reject(err); - }); - req.end(); - }); - return await p; -} \ No newline at end of file + return (await response.json()) as PrivateKeyResponse; +} diff --git a/services/api/src/apolloServer.js b/services/api/src/apolloServer.js index 2f8e569f02..ea2f270242 100644 --- a/services/api/src/apolloServer.js +++ b/services/api/src/apolloServer.js @@ -25,12 +25,10 @@ const { userActivityLogger } = require('./loggers/userActivityLogger'); const typeDefs = require('./typeDefs'); const resolvers = require('./resolvers'); const { keycloakGrantManager } = require('./clients/keycloakClient'); -const { getRedisKeycloakCache, saveRedisKeycloakCache } = require('./clients/redisClient'); const User = require('./models/user'); const Group = require('./models/group'); -const ProjectModel = require('./models/project'); -const EnvironmentModel = require('./models/environment'); +const Environment = require('./models/environment'); const schema = makeExecutableSchema({ typeDefs, resolvers }); @@ -166,8 +164,7 @@ const apolloServer = new ApolloServer({ models: { UserModel: User.User(modelClients), GroupModel: Group.Group(modelClients), - ProjectModel: ProjectModel.ProjectModel(modelClients), - EnvironmentModel: EnvironmentModel.EnvironmentModel(modelClients) + EnvironmentModel: Environment.Environment(modelClients) }, keycloakUsersGroups, adminScopes: { @@ -273,8 +270,7 @@ const apolloServer = new ApolloServer({ models: { UserModel: User.User(modelClients), GroupModel: Group.Group(modelClients), - ProjectModel: ProjectModel.ProjectModel(modelClients), - EnvironmentModel: EnvironmentModel.EnvironmentModel(modelClients) + EnvironmentModel: Environment.Environment(modelClients) }, keycloakUsersGroups, adminScopes: { diff --git a/services/api/src/clients/keycloak-admin.ts b/services/api/src/clients/keycloak-admin.ts index db30576ae9..9605c06838 100644 --- a/services/api/src/clients/keycloak-admin.ts +++ b/services/api/src/clients/keycloak-admin.ts @@ -3,6 +3,22 @@ import { KeycloakAdminClient } from '@s3pweb/keycloak-admin-client-cjs'; import { logger } from '../loggers/logger'; import { config } from './keycloakClient'; +/// Helper to type check try/catch. Remove when we can stop using the @s3pweb +// commonJS version. +export const isNetworkError = ( + error: unknown, +): error is { + response: { + status: number; + }; +} => { + return ( + typeof error === 'object' && + 'response' in error && + typeof error.response === 'object' + ); +}; + export const getKeycloakAdminClient = async (): Promise => { const keycloakAdminClient = new KeycloakAdminClient({ baseUrl: `${config.origin}/auth`, diff --git a/services/api/src/clients/keycloakClient.ts b/services/api/src/clients/keycloakClient.ts index 71944fc59a..66208c319c 100644 --- a/services/api/src/clients/keycloakClient.ts +++ b/services/api/src/clients/keycloakClient.ts @@ -30,9 +30,8 @@ const keycloakConfig = new KeycloakConfig({ export const keycloakGrantManager = new KeycloakGrantManager(keycloakConfig); // Override the library "validateToken" function because it is very strict about -// verifying the ISS, which is the URI of the keycloak server. This will almost -// always fail since the URI will be different for end users authenticated via -// the web console and services communicating via backchannel. +// verifying the ISS, which is the URI of the keycloak server. This fails when +// the URL used in the UI doesn't match what's used in the API. keycloakGrantManager.validateToken = function validateToken( token, expectedType diff --git a/services/api/src/clients/redisClient.ts b/services/api/src/clients/redisClient.ts index d2556db4b7..40a741d92c 100644 --- a/services/api/src/clients/redisClient.ts +++ b/services/api/src/clients/redisClient.ts @@ -1,9 +1,22 @@ -import * as R from 'ramda'; -import redis, { ClientOpts } from 'redis'; +import redis from 'redis'; import { promisify } from 'util'; import { toNumber } from '../util/func'; import { getConfigFromEnv, envHasConfig } from '../util/config'; +export class RedisCacheLoadError extends Error { + constructor(message: string) { + super(message); + this.name = 'RedisCacheLoadError'; + } +} + +export class RedisCacheSaveError extends Error { + constructor(message: string) { + super(message); + this.name = 'RedisCacheSaveError'; + } +} + export const config: { hostname: string; port: number; @@ -13,72 +26,23 @@ export const config: { port: toNumber(getConfigFromEnv('REDIS_PORT', '6379')), pass: envHasConfig('REDIS_PASSWORD') ? getConfigFromEnv('REDIS_PASSWORD') - : undefined + : undefined, }; export const redisClient = redis.createClient({ host: config.hostname, port: config.port, password: config.pass, - enable_offline_queue: false + enable_offline_queue: false, }); -redisClient.on('error', function(error) { +redisClient.on('error', function (error) { console.error(error); }); -export const groupCacheExpiry = toNumber(getConfigFromEnv('REDIS_GROUP_CACHE_EXPIRY', '172800')) +export const groupCacheExpiry = toNumber( + getConfigFromEnv('REDIS_GROUP_CACHE_EXPIRY', '172800'), +); export const get = promisify(redisClient.get).bind(redisClient); -const hgetall = promisify(redisClient.hgetall).bind(redisClient); -const smembers = promisify(redisClient.smembers).bind(redisClient); -const sadd = promisify(redisClient.sadd).bind(redisClient); export const del = promisify(redisClient.del).bind(redisClient); - -interface IUserResourceScope { - resource: string; - scope: string; - currentUserId: string; - project?: number; - group?: string; - users?: number[]; -} - -const hashKey = ({ resource, project, group, scope }: IUserResourceScope) => - `${resource}:${project ? `${project}:` : ''}${ - group ? `${group}:` : '' - }${scope}`; - -export const getRedisKeycloakCache = async (key: string) => { - const redisHash = await hgetall(`cache:keycloak`); - - return R.prop(key, redisHash); -}; - -export const saveRedisKeycloakCache = async ( - key: string, - value: number | string -) => { - await redisClient.hmset( - `cache:keycloak`, - key, - value - ); -}; - -export const deleteRedisUserCache = userId => del(`cache:authz:${userId}`); - -export const getProjectGroupsCache = async projectId => - smembers(`project-groups:${projectId}`); -export const saveProjectGroupsCache = async (projectId, groupIds) => - sadd(`project-groups:${projectId}`, groupIds); -export const deleteProjectGroupsCache = async projectId => - del(`project-groups:${projectId}`); - -export default { - getRedisKeycloakCache, - saveRedisKeycloakCache, - deleteRedisUserCache, - getProjectGroupsCache, - saveProjectGroupsCache -}; diff --git a/services/api/src/helpers/reset-project-keys.ts b/services/api/src/helpers/reset-project-keys.ts index bbcfe5e6f4..0d0b4842b2 100644 --- a/services/api/src/helpers/reset-project-keys.ts +++ b/services/api/src/helpers/reset-project-keys.ts @@ -5,9 +5,8 @@ import * as gitlabApi from '@lagoon/commons/dist/gitlab/api'; import { getKeycloakAdminClient } from '../clients/keycloak-admin'; import { sqlClientPool } from '../clients/sqlClient'; import { esClient } from '../clients/esClient'; -import redisClient from '../clients/redisClient'; import { query } from '../util/db'; -import { Group } from '../models/group'; +import { Group, SparseGroup } from '../models/group'; import { User } from '../models/user'; import { validateKey, generatePrivateKey as genpk } from '../util/func'; import { Sql as sshKeySql } from '../resources/sshKey/sql'; @@ -30,13 +29,11 @@ interface GitlabProject { sqlClientPool, keycloakAdminClient, esClient, - redisClient }); const UserModel = User({ sqlClientPool, keycloakAdminClient, esClient, - redisClient }); const projectArgs = process.argv.slice(2); @@ -45,13 +42,14 @@ interface GitlabProject { return; } - const allGitlabProjects = (await gitlabApi.getAllProjects()) as GitlabProject[]; + const allGitlabProjects = + (await gitlabApi.getAllProjects()) as GitlabProject[]; const projectRecords = await query( sqlClientPool, 'SELECT * FROM `project` WHERE name IN (:projects)', { - projects: projectArgs - } + projects: projectArgs, + }, ); for (const project of projectRecords) { @@ -59,16 +57,22 @@ interface GitlabProject { const gitlabProject = R.find( (findProject: GitlabProject) => - sanitizeGroupName(findProject.path) === project.name - )(allGitlabProjects); + sanitizeGroupName(findProject.path) === project.name, + )(allGitlabProjects) as GitlabProject; // Load default group const projectGroupName = `project-${project.name}`; - let keycloakGroup; + let keycloakGroup: SparseGroup | undefined; try { - keycloakGroup = await GroupModel.loadGroupByName(projectGroupName); - } catch (err) { - logger.error(`Could not load group ${projectGroupName}: ${err.message}`); + keycloakGroup = await GroupModel.loadSparseGroupByName(projectGroupName); + } catch (err: unknown) { + if (err instanceof Error) { + logger.error( + `Could not load group ${projectGroupName}: ${err.message}`, + ); + } else { + throw err; + } } // ////////////////////// @@ -77,25 +81,35 @@ interface GitlabProject { if (R.prop('privateKey', project)) { let keyPair = {} as any; try { - const privkey = new Buffer((R.prop('privateKey', project))).toString('base64') - const publickey = await validateKey(privkey, "private") + const privkey = new Buffer(R.prop('privateKey', project)).toString( + 'base64', + ); + const publickey = await validateKey(privkey, 'private'); keyPair = { ...keyPair, - private: R.replace(/\n/g, '\n', (R.prop('privateKey', project)).toString('openssh')), + private: R.replace( + /\n/g, + '\n', + R.prop('privateKey', project).toString('openssh'), + ), public: publickey['publickey'], - fingerprint: publickey['sha256fingerprint'] + fingerprint: publickey['sha256fingerprint'], }; - } catch (err) { - throw new Error( - `There was an error with the privateKey: ${err.message}` - ); + } catch (err: unknown) { + if (err instanceof Error) { + throw new Error( + `There was an error with the privateKey: ${err.message}`, + ); + } else { + throw err; + } } const keyParts = keyPair.public.split(' '); // Delete users with current key const userRows = await query( sqlClientPool, - sshKeySql.selectUserIdsBySshKeyFingerprint(keyPair.fingerprint) + sshKeySql.selectUserIdsBySshKeyFingerprint(keyPair.fingerprint), ); for (const userRow of userRows) { @@ -103,9 +117,11 @@ interface GitlabProject { try { await UserModel.deleteUser(userId); - } catch (err) { - if (!err.message.includes('User not found')) { - throw new Error(err); + } catch (err: unknown) { + if (err instanceof Error) { + if (!err.message.includes('User not found')) { + throw err; + } } } @@ -114,8 +130,8 @@ interface GitlabProject { sqlClientPool, 'DELETE FROM ssh_key WHERE key_value = :key', { - key: keyParts[1] - } + key: keyParts[1], + }, ); // Delete user_ssh_key link @@ -123,8 +139,8 @@ interface GitlabProject { sqlClientPool, 'DELETE FROM user_ssh_key WHERE usid = :usid', { - usid: userId - } + usid: userId, + }, ); } @@ -133,8 +149,8 @@ interface GitlabProject { sqlClientPool, 'UPDATE project p SET private_key = NULL WHERE id = :pid', { - pid: project.id - } + pid: project.id, + }, ); } @@ -143,12 +159,12 @@ interface GitlabProject { // //////////////// // Generate new keypair - const genkey = await genpk() + const genkey = await genpk(); const keyPair = { private: genkey['privatekeypem'], public: genkey['publickey'], fingerprint: genkey['sha256fingerprint'], - type: genkey['type'] + type: genkey['type'], }; // Save the newly generated key @@ -157,25 +173,25 @@ interface GitlabProject { 'UPDATE project p SET private_key = :pkey WHERE id = :pid', { pkey: keyPair.private, - pid: project.id - } + pid: project.id, + }, ); // Find or create a user that has the public key linked to them const userRows = await query( sqlClientPool, - sshKeySql.selectUserIdsBySshKeyFingerprint(keyPair.fingerprint) + sshKeySql.selectUserIdsBySshKeyFingerprint(keyPair.fingerprint), ); const userId = R.path([0, 'usid'], userRows) as string; - let user; + let user: (User & { id: string }) | undefined; if (!userId) { try { - user = await UserModel.addUser({ + user = (await UserModel.addUser({ email: `default-user@${project.name}`, username: `default-user@${project.name}`, - comment: `autogenerated user for project ${project.name}` - }); + comment: `autogenerated user for project ${project.name}`, + })) as User & { id: string }; const keyParts = keyPair.public.split(' '); @@ -186,38 +202,54 @@ interface GitlabProject { name: 'auto-add via reset', keyValue: keyParts[1], keyType: keyParts[0], - keyFingerprint: keyPair.fingerprint - }) + keyFingerprint: keyPair.fingerprint, + }), ); await query( sqlClientPool, - sshKeySql.addSshKeyToUser({ sshKeyId: insertId, userId: user.id }) - ); - } catch (err) { - logger.error( - `Could not create default project user for ${project.name}: ${err.message}` + sshKeySql.addSshKeyToUser({ sshKeyId: insertId, userId: user.id }), ); + } catch (err: unknown) { + if (err instanceof Error) { + logger.error( + `Could not create default project user for ${project.name}: ${err.message}`, + ); + } else { + throw err; + } } } else { - user = await UserModel.loadUserById(userId); + user = (await UserModel.loadUserById(userId)) as User & { id: string }; } - // Add the user (with linked public key) to the default group as maintainer - try { - await GroupModel.addUserToGroup(user, keycloakGroup, 'maintainer'); - } catch (err) { - logger.error( - `Could not link user to default projet group for ${project.name}: ${err.message}` - ); + if (user && keycloakGroup) { + // Add the user (with linked public key) to the default group as maintainer + try { + await GroupModel.addUserToGroup( + user, + { id: keycloakGroup.id }, + 'maintainer', + ); + } catch (err: unknown) { + if (err instanceof Error) { + logger.error( + `Could not link user to default projet group for ${project.name}: ${err.message}`, + ); + } else { + throw err; + } + } } try { await gitlabApi.addDeployKeyToProject(gitlabProject.id, keyPair.public); - } catch (err) { - if (!err.message.includes('has already been taken')) { - throw new Error( - `Could not add deploy_key to gitlab project ${gitlabProject.id}, reason: ${err}` - ); + } catch (err: unknown) { + if (err instanceof Error) { + if (!err.message.includes('has already been taken')) { + throw new Error( + `Could not add deploy_key to gitlab project ${gitlabProject.id}, reason: ${err}`, + ); + } } } } diff --git a/services/api/src/helpers/sync-groups-opendistro-security.ts b/services/api/src/helpers/sync-groups-opendistro-security.ts index d990e296e6..245ae56a6d 100755 --- a/services/api/src/helpers/sync-groups-opendistro-security.ts +++ b/services/api/src/helpers/sync-groups-opendistro-security.ts @@ -2,7 +2,6 @@ import * as R from 'ramda'; import { logger } from '@lagoon/commons/dist/logs/local-logger'; import { sqlClientPool } from '../clients/sqlClient'; import { esClient } from '../clients/esClient'; -import redisClient from '../clients/redisClient'; import { Group } from '../models/group'; import { OpendistroSecurityOperations } from '../resources/group/opendistroSecurity'; import { getKeycloakAdminClient } from '../clients/keycloak-admin'; @@ -13,7 +12,6 @@ import { getKeycloakAdminClient } from '../clients/keycloak-admin'; sqlClientPool, keycloakAdminClient, esClient, - redisClient }); const groupRegex = process.env.GROUP_REGEX diff --git a/services/api/src/migrations/3-remove-billing-data/billing-groups.ts b/services/api/src/migrations/3-remove-billing-data/billing-groups.ts deleted file mode 100644 index fcfbde978c..0000000000 --- a/services/api/src/migrations/3-remove-billing-data/billing-groups.ts +++ /dev/null @@ -1,50 +0,0 @@ -import * as R from 'ramda'; -import { logger } from '@lagoon/commons/dist/logs/local-logger'; -import { getKeycloakAdminClient } from '../../clients/keycloak-admin'; -import { sqlClientPool } from '../../clients/sqlClient'; -import { esClient } from '../../clients/esClient'; -import redisClient from '../../clients/redisClient'; -import { Group } from '../../models/group'; - -(async () => { - const keycloakAdminClient = await getKeycloakAdminClient(); - - const GroupModel = Group({ - sqlClientPool, - keycloakAdminClient, - esClient, - redisClient - }); - - logger.debug('Removing billing groups'); - - let billingGroups = [] as Group[]; - try { - billingGroups = await GroupModel.loadGroupsByAttribute( - ({ name, value }) => { - return name === 'type' && value[0] === 'billing'; - } - ); - } catch (err) { - logger.error('Could not load billing groups'); - throw new Error(err); - } - - if (billingGroups.length == 0) { - logger.info('No billing groups found'); - } - - for (const group of billingGroups) { - if (group.groups && group.groups.length > 0) { - logger.error(`Found subgroups for "${group.name}". This is unexpected config, skipping deleting, please fix manually`); - continue; - } - - logger.debug(`Deleting billing group "${group.name}"`); - await GroupModel.deleteGroup(group.id); - } - - logger.info('Migration completed'); - - return; -})(); diff --git a/services/api/src/migrations/lagoon/migrations/20240312000000_group_projects.ts b/services/api/src/migrations/lagoon/migrations/20240312000000_group_projects.ts index 4a32137977..669ac0d3ea 100644 --- a/services/api/src/migrations/lagoon/migrations/20240312000000_group_projects.ts +++ b/services/api/src/migrations/lagoon/migrations/20240312000000_group_projects.ts @@ -3,7 +3,6 @@ import { logger } from '@lagoon/commons/dist/logs/local-logger'; import { getKeycloakAdminClient } from '../../../clients/keycloak-admin'; import { sqlClientPool } from '../../../clients/sqlClient'; import { esClient } from '../../../clients/esClient'; -import redisClient from '../../../clients/redisClient'; import { Group } from '../../../models/group'; import { Helpers } from '../../../resources/group/helpers'; @@ -14,7 +13,6 @@ export const up = async (migrate) => { sqlClientPool, keycloakAdminClient, esClient, - redisClient }); // load all groups from keycloak diff --git a/services/api/src/models/environment.ts b/services/api/src/models/environment.ts index c43b12592b..ba2588e655 100644 --- a/services/api/src/models/environment.ts +++ b/services/api/src/models/environment.ts @@ -22,22 +22,22 @@ export interface Environment { deployTitle?: string; // varchar(300) COLLATE utf8_bin DEFAULT NULL, } -interface EnvironmentData { - id: string; - name: string; - type: string; - hits: any; - storage: any; - hours: any; +export interface EnvironmentModel { + projectEnvironments: (pid, type, includeDeleted) => any; + environmentsByProjectId: (pid, type, includeDeleted) => any; + environmentStorageMonthByEnvironmentId: (eid, month) => any; + environmentHoursMonthByEnvironmentId: (eid: number, yearMonth: string) => any; + environmentHitsMonthByEnvironmentId: ( + project, + openshiftProjectName, + yearMonth, + ) => any; + calculateHitsFromESData: (legacyResult: any, newResult: any) => any; } -type projectEnvWithDataType = ( - pid: string, - projectName: string, - month: string -) => Promise; - -export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { +export const Environment = (clients: { + sqlClientPool: Pool; +}): EnvironmentModel => { const { sqlClientPool } = clients; /** @@ -50,13 +50,19 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { * @return {Promise<[Environments]>} An array of all project environments */ const projectEnvironments = async (pid, type, includeDeleted = false) => { - let query = knex('environment') - .where(knex.raw('project = ?', pid)) + let query = knex('environment').where(knex.raw('project = ?', pid)); - if (!includeDeleted) { query = query.andWhere('deleted', '0000-00-00 00:00:00') } - if (type) { query = query.andWhere(knex.raw('environment_type = ?', type)) } + if (!includeDeleted) { + query = query.andWhere('deleted', '0000-00-00 00:00:00'); + } + if (type) { + query = query.andWhere(knex.raw('environment_type = ?', type)); + } - const environments: [Environment] = await query(sqlClientPool, query.toString()); + const environments: [Environment] = await query( + sqlClientPool, + query.toString(), + ); return environments; }; @@ -64,7 +70,7 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { const environmentsByProjectId = projectEnvironments; // Needed for local Dev - Required if not connected to openshift - const errorCatcherFn = (msg, responseObj) => err => { + const errorCatcherFn = (msg, responseObj) => (err) => { const errMsg = err && err.status && err.message ? `${err.status} : ${err.message} : ${err.headers} : ${err.url}` @@ -75,28 +81,32 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { const environmentStorageMonthByEnvironmentId = async (eid, month) => { let q = knex('environment_storage') - .select(knex.raw('SUM(kib_used) as kib_used')) - .select(knex.raw('SUM(kib_used) as bytes_used')) // @DEPRECATE when `bytesUsed` is completely removed, this can be removed - .select(knex.raw(`max(DATE_FORMAT(updated, '%Y-%m')) as month`)) - .where('environment', eid) - .andWhere(knex.raw(`YEAR(updated) = YEAR(STR_TO_DATE(?, '%Y-%m'))`, month)) - .andWhere(knex.raw(`MONTH(updated) = MONTH(STR_TO_DATE(?, '%Y-%m'))`, month)) + .select(knex.raw('SUM(kib_used) as kib_used')) + .select(knex.raw('SUM(kib_used) as bytes_used')) // @DEPRECATE when `bytesUsed` is completely removed, this can be removed + .select(knex.raw(`max(DATE_FORMAT(updated, '%Y-%m')) as month`)) + .where('environment', eid) + .andWhere( + knex.raw(`YEAR(updated) = YEAR(STR_TO_DATE(?, '%Y-%m'))`, month), + ) + .andWhere( + knex.raw(`MONTH(updated) = MONTH(STR_TO_DATE(?, '%Y-%m'))`, month), + ); const rows = await query(sqlClientPool, q.toString()); - rows.map(row => ({ ...row, bytesUsed: row.kibUsed})); // @DEPRECATE when `bytesUsed` is completely removed, this can be removed + rows.map((row) => ({ ...row, bytesUsed: row.kibUsed })); // @DEPRECATE when `bytesUsed` is completely removed, this can be removed return rows[0]; }; const environmentHoursMonthByEnvironmentId = async ( eid: number, - yearMonth: string + yearMonth: string, ) => { const rows = await query( sqlClientPool, 'SELECT e.created, e.deleted FROM environment e WHERE e.id = :eid', - { eid } + { eid }, ); const { created, deleted } = rows[0]; @@ -179,7 +189,7 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { openshiftProjectName, interestedYearMonth, interestedDateBeginString, - interestedDateEndString + interestedDateEndString, ) => { try { // LEGACY LOGGING SYSTEM - REMOVE ONCE WE MIGRATE EVERYTHING TO THE NEW SYSTEM @@ -195,38 +205,38 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { '@timestamp': { gte: `${interestedYearMonth}||/M`, lte: `${interestedYearMonth}||/M`, - format: 'strict_year_month' - } - } - } + format: 'strict_year_month', + }, + }, + }, ], must_not: [ { match_phrase: { request_header_useragent: { // Legacy Logging - OpenShift Router: Exclude requests from StatusCake - query: 'StatusCake' - } - } + query: 'StatusCake', + }, + }, }, { match_phrase: { request_header_useragent: { // Legacy Logging - OpenShift Router: Exclude requests from UptimeRobot - query: 'UptimeRobot' - } - } + query: 'UptimeRobot', + }, + }, }, { match_phrase: { http_request: { // Legacy Logging - OpenShift Router: Exclude requests to acme challenges - query: 'acme-challenge' - } - } - } - ] - } + query: 'acme-challenge', + }, + }, + }, + ], + }, }, aggs: { hourly: { @@ -236,25 +246,25 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { min_doc_count: 0, extended_bounds: { min: interestedDateBeginString, - max: interestedDateEndString - } + max: interestedDateEndString, + }, }, aggs: { count: { value_count: { - field: '@timestamp' - } - } - } + field: '@timestamp', + }, + }, + }, }, average: { avg_bucket: { buckets_path: 'hourly>count', - gap_policy: 'skip' // makes sure that we don't use empty buckets as average calculation - } - } - } - } + gap_policy: 'skip', // makes sure that we don't use empty buckets as average calculation + }, + }, + }, + }, }; const legacyResult = await esClient.search(legacyQuery); @@ -271,67 +281,67 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { '@timestamp': { gte: `${interestedYearMonth}||/M`, lte: `${interestedYearMonth}||/M`, - format: 'strict_year_month' - } - } + format: 'strict_year_month', + }, + }, }, { match_phrase: { - 'kubernetes.namespace_name': `${openshiftProjectName}` - } - } + 'kubernetes.namespace_name': `${openshiftProjectName}`, + }, + }, ], must_not: [ { match_phrase: { http_user_agent: { // New Logging - Kubernetes Ingress: Exclude requests from Statuscake - query: 'StatusCake' - } - } + query: 'StatusCake', + }, + }, }, { match_phrase: { http_user_agent: { // New Logging - Kubernetes Ingress: Exclude requests from UptimeRobot - query: 'UptimeRobot' - } - } + query: 'UptimeRobot', + }, + }, }, { match_phrase: { request_user_agent: { // New Logging - OpenShift Router: Exclude requests from Statuscake - query: 'StatusCake' - } - } + query: 'StatusCake', + }, + }, }, { match_phrase: { request_user_agent: { // New Logging - OpenShift Router: Exclude requests from UptimeRobot - query: 'UptimeRobot' - } - } + query: 'UptimeRobot', + }, + }, }, { match_phrase: { request_uri: { // New Logging - Kubernetes Ingress: Exclude requests to acme challenges - query: 'acme-challenge' - } - } + query: 'acme-challenge', + }, + }, }, { match_phrase: { http_request: { // New Logging - OpenShift Router: Exclude requests to acme challenges - query: 'acme-challenge' - } - } - } - ] - } + query: 'acme-challenge', + }, + }, + }, + ], + }, }, aggs: { hourly: { @@ -341,25 +351,25 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { min_doc_count: 0, extended_bounds: { min: interestedDateBeginString, - max: interestedDateEndString - } + max: interestedDateEndString, + }, }, aggs: { count: { value_count: { - field: '@timestamp' - } - } - } + field: '@timestamp', + }, + }, + }, }, average: { avg_bucket: { buckets_path: 'hourly>count', - gap_policy: 'skip' // makes sure that we don't use empty buckets as average calculation - } - } - } - } + gap_policy: 'skip', // makes sure that we don't use empty buckets as average calculation + }, + }, + }, + }, }; const newResult = await esClient.search(newQuery); @@ -368,7 +378,7 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { logger.error( `Elastic Search Query Error: ${ JSON.stringify(e) != '{}' ? JSON.stringify(e) : e - }` + }`, ); // const noHits = { total: 0 }; @@ -490,7 +500,7 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { const environmentHitsMonthByEnvironmentId = async ( project, openshiftProjectName, - yearMonth + yearMonth, ) => { const interestedDate = yearMonth ? new Date(yearMonth) : new Date(); const year = interestedDate.getUTCFullYear(); @@ -517,7 +527,7 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { openshiftProjectName, interestedYearMonth, interestedDateBeginString, - interestedDateEndString + interestedDateEndString, ); if (newResult === null || legacyResult === null) { @@ -535,8 +545,6 @@ export const EnvironmentModel = (clients: { sqlClientPool: Pool }) => { environmentStorageMonthByEnvironmentId, environmentHoursMonthByEnvironmentId, environmentHitsMonthByEnvironmentId, - calculateHitsFromESData + calculateHitsFromESData, }; }; - -export default EnvironmentModel; diff --git a/services/api/src/models/group.ts b/services/api/src/models/group.ts index 531df251eb..34b5033a8c 100644 --- a/services/api/src/models/group.ts +++ b/services/api/src/models/group.ts @@ -2,19 +2,48 @@ import * as R from 'ramda'; import { Pool } from 'mariadb'; import { asyncPipe } from '@lagoon/commons/dist/util/func'; import pickNonNil from '../util/pickNonNil'; +import { toNumber } from '../util/func'; import { logger } from '../loggers/logger'; import type { GroupRepresentation } from '@s3pweb/keycloak-admin-client-cjs'; +import { isNetworkError } from '../clients/keycloak-admin'; import { User } from './user'; -import { groupCacheExpiry, get, del, redisClient } from '../clients/redisClient'; +import { groupCacheExpiry, get, del, redisClient, RedisCacheLoadError, RedisCacheSaveError } from '../clients/redisClient'; import { Helpers as projectHelpers } from '../resources/project/helpers'; import { Helpers as groupHelpers } from '../resources/group/helpers'; import { sqlClientPool } from '../clients/sqlClient'; -interface IGroupAttributes { - 'lagoon-projects'?: [string]; - 'lagoon-organization'?: [string]; - comment?: [string]; - [propName: string]: any; +interface KeycloakLagoonGroupAttributes { + type?: string[]; + 'lagoon-projects'?: string[]; + 'lagoon-organization'?: string[]; + 'group-lagoon-project-ids'?: string[]; +} + +export interface KeycloakLagoonGroup extends GroupRepresentation { + attributes?: KeycloakLagoonGroupAttributes; + parentId?: string; +} + +export enum GroupType { + ROLE_SUBGROUP = 'role-subgroup', + PROJECT_DEFAULT_GROUP = 'project-default-group', +} + +export interface SparseGroup { + id: string; + name: string; + organization: number | null; + parentGroupId: string | null; + projects: Set; + subGroupCount: number; + type: GroupType | null; +} + +interface HieararchicalGroup extends SparseGroup { + // Groups as represented by the API spec. + groups: HieararchicalGroup[]; + // All groups, including internal ones. + allGroups: HieararchicalGroup[]; } export interface Group { @@ -30,7 +59,7 @@ export interface Group { members?: GroupMembership[]; // All subgroups according to keycloak. subGroups?: GroupRepresentation[]; - attributes?: IGroupAttributes; + attributes?: KeycloakLagoonGroupAttributes; realmRoles?: any[]; } @@ -70,9 +99,11 @@ export class GroupNotFoundError extends Error { } } +// Group types that are for internal use only. +const internalGroupTypes = [GroupType.ROLE_SUBGROUP]; + const attrLens = R.lensPath(['attributes']); const lagoonProjectsLens = R.lensPath(['lagoon-projects']); -const lagoonOrganizationLens = R.lensPath(['lagoon-organization']); const attrLagoonProjectsLens = R.compose( // @ts-ignore @@ -81,13 +112,6 @@ const attrLagoonProjectsLens = R.compose( R.lensPath([0]) ); -const attrLagoonOrganizationLens = R.compose( - // @ts-ignore - attrLens, - lagoonOrganizationLens, - R.lensPath([0]) -); - const getProjectIdsFromGroup = R.pipe( // @ts-ignore R.view(attrLagoonProjectsLens), @@ -97,29 +121,65 @@ const getProjectIdsFromGroup = R.pipe( R.map(id => parseInt(id, 10)) ); -const getOrganizationIdFromGroup = R.pipe( - // @ts-ignore - R.view(attrLagoonOrganizationLens), - R.defaultTo(''), - R.split(','), - R.reject(R.isEmpty), - R.map(id => parseInt(id, 10)) -); - export const isRoleSubgroup = R.pathEq( ['attributes', 'type', 0], 'role-subgroup' ); +const isInternalGroup = (group: SparseGroup) => { + if (!group.type) { + return false; + } + + if (internalGroupTypes.includes(group.type)) { + return true; + } + + return false; +} + const attributeKVOrNull = (key: string, group: GroupRepresentation) => String(R.pathOr(null, ['attributes', key], group)); +const parseGroupType = (type: string): GroupType | null => { + if (Object.values(GroupType).includes(type as GroupType)) { + return type as GroupType; + } + + return null; +}; + +export interface GroupModel { + loadAllGroups: () => Promise + loadGroupById: (id: string) => Promise + loadSparseGroupByName: (name: string) => Promise + loadGroupByName: (name: string) => Promise + loadGroupByIdOrName: (groupInput: GroupInput) => Promise + loadParentGroup: (groupInput: Group) => Promise + createGroupFromKeycloak: (group: KeycloakLagoonGroup) => SparseGroup + getProjectsFromGroupAndParents: (group: Group) => Promise + getProjectsFromGroupAndSubgroups: (group: Group) => Promise + getProjectsFromGroup: (group: Group) => Promise + addGroup: (groupInput: Group, projectId?: number, organizationId?: number) => Promise + updateGroup: (groupInput: GroupEdit) => Promise + deleteGroup: (id: string) => Promise + addUserToGroup: (user: User, groupInput: GroupInput, roleName: string) => Promise + removeUserFromGroup: (user: User, group: Group ) => Promise + removeUserFromGroups: (user: User,groups: Group[]) => Promise + addProjectToGroup: (projectId: number, groupInput: any) => Promise + removeProjectFromGroup: (projectId: number, group: Group) => Promise + removeProjectFromGroups: (projectId: number, groups: Group[]) => Promise + transformKeycloakGroups: (keycloakGroups: GroupRepresentation[]) => Promise + getGroupMembership: (group: Group) => Promise + getGroupMemberCount: (group: Group) => Promise + removeNonProjectDefaultUsersFromGroup: (group: Group, project: String) => Promise +} + export const Group = (clients: { keycloakAdminClient: any; - redisClient: any; sqlClientPool: Pool; esClient: any; -}) => { +}): GroupModel => { const { keycloakAdminClient } = clients; const transformKeycloakGroups = async ( @@ -156,250 +216,297 @@ export const Group = (clients: { return groupsWithGroupsAndMembers; }; - const loadGroupById = async (id: string): Promise => { - let group; + const createGroupFromKeycloak = (group: KeycloakLagoonGroup): SparseGroup => { + const groupAttr = group.attributes?.['type']?.[0]; + const projectsAttr = group.attributes?.['lagoon-projects']?.[0]; + const projectsArray = projectsAttr + ? projectsAttr + .split(',') + .map((id) => id.trim()) + .filter((id) => !!id) + .map(toNumber) + : []; + const organizationAttr = group.attributes?.['lagoon-organization']?.[0]; - // check if the group is in the cache - try { - group = await getGroupCacheById(id) - } catch(err) { - logger.warn(`Error reading redis, falling back to direct lookup: ${err.message}`); + if (!group.id) { + throw new Error('Missing group id'); } - // if not in the cache, check keycloak - if (!group) { - let keycloakGroup: Group - keycloakGroup = await keycloakAdminClient.groups.findOne({ - id, - briefRepresentation: false, - }); - if (R.isNil(keycloakGroup)) { - throw new GroupNotFoundError(`Group not found: ${id}`); - } - const groups = await transformKeycloakGroups([keycloakGroup]); - keycloakGroup = groups[0] - // then save the entry in the cache - try { - await saveGroupCache(keycloakGroup) - } catch (err) { - logger.info (`Error saving redis: ${err.message}`) - } - return keycloakGroup; + if (!group.name) { + throw new Error('Missing group name'); } - return group; + + return { + id: group.id, + name: group.name, + organization: organizationAttr ? toNumber(organizationAttr) : null, + parentGroupId: group.parentId ?? null, + projects: new Set(projectsArray), + subGroupCount: group.subGroupCount ?? 0, + type: groupAttr ? parseGroupType(groupAttr) : null, + }; }; - const loadGroupByName = async (name: string): Promise => { - let group; + // Load a single, lightweight group (no subgroups/members). + const loadSparseGroupById = async (id: string): Promise => { + const keycloakGroup = (await keycloakAdminClient.groups.findOne({ + id, + })) as KeycloakLagoonGroup; - try { - group = await getGroupCacheByName(name) - } catch(err) { - logger.warn(`Error reading redis, falling back to direct lookup: ${err.message}`); + if (!keycloakGroup) { + throw new GroupNotFoundError(`Group not found: ${id}`); } - if (!group) { - const keycloakGroups = await keycloakAdminClient.groups.find({ - search: name - }); + return createGroupFromKeycloak(keycloakGroup); + }; - if (R.isEmpty(keycloakGroups)) { - throw new GroupNotFoundError(`Group not found: ${name}`); - } + /** + * Loads a "full" group from keycloak. + * @deprecated Use `loadSparseGroupById` instead. + */ + const loadGroupById = async (id: string): Promise => { + let fullGroup: Group | null = null; - // Use mutable operations to avoid running out of heap memory - const flattenGroups = (groups, group) => { - groups.push(R.omit(['subGroups'], group)); - const flatSubGroups = group.subGroups.reduce(flattenGroups, []); - return groups.concat(flatSubGroups); - }; + try { + fullGroup = await getGroupCacheById(id); + } catch (err: unknown) { + if (err instanceof RedisCacheLoadError) { + logger.info(err.message); + } else { + throw err; + } + } - group = R.pipe( - R.reduce(flattenGroups, []), - R.filter(R.propEq('name', name)) - )(keycloakGroups); + if (!fullGroup) { + const keycloakGroup = (await keycloakAdminClient.groups.findOne({ + id, + })) as KeycloakLagoonGroup; - if (R.isEmpty(group)) { - throw new GroupNotFoundError(`Group not found: ${name}`); + if (!keycloakGroup) { + throw new GroupNotFoundError(`Group not found: ${id}`); } - const keycloakGroup = await keycloakAdminClient.groups.findOne({ - id: group[0].id, - briefRepresentation: false, - }); - const groups = await transformKeycloakGroups([keycloakGroup]); - group = groups[0]; + const groupWithSubgroups = await loadKeycloakChildren(keycloakGroup); + const groups = await transformKeycloakGroups([groupWithSubgroups]); + fullGroup = groups[0]; + try { - await saveGroupCache(group) - } catch (err) { - logger.info (`Error saving redis: ${err.message}`) + await saveGroupCache(fullGroup); + } catch (err: unknown) { + if (err instanceof RedisCacheSaveError) { + logger.info(err.message); + } else { + throw err; + } } } - // @ts-ignore - return group; + return fullGroup; }; - const loadGroupByIdOrName = async ( - groupInput: GroupInput - ): Promise => { - if (R.prop('id', groupInput)) { - return loadGroupById(R.prop('id', groupInput)); + // Load a single, lightweight group (no subgroups/members). + const loadSparseGroupByName = async (name: string): Promise => { + const keycloakGroups = (await keycloakAdminClient.groups.find({ + search: name, + exact: true, // Exact name match + briefRepresentation: false, // Returns attributes + // @ts-ignore https://github.com/keycloak/keycloak/issues/32136 + populateHierarchy: false, // Return a single group instead of the full tree + })) as KeycloakLagoonGroup[]; + + if (keycloakGroups.length === 0) { + throw new GroupNotFoundError(`Group not found: ${name}`); } - if (R.prop('name', groupInput)) { - return loadGroupByName(R.prop('name', groupInput)); + if (keycloakGroups.length > 1) { + throw new Error( + `Too many results (${keycloakGroups.length}) for ${name}.`, + ); } - throw new Error('You must provide a group id or name'); + return createGroupFromKeycloak(keycloakGroups[0]); }; - const loadAllGroups = async (): Promise => { - // briefRepresentation pulls all the group information from keycloak including the attributes - // this means we don't need to iterate over all the groups one by one anymore to get the full group information - const fullGroups = await keycloakAdminClient.groups.find({briefRepresentation: false}); - // no need to transform, just return the full response, only the `allGroups` resolvers use this - // and the `sync-groups-opendistro-security` consumption of this helper sync script is going to - // go away in the future when we move to the `lagoon-opensearch-sync` supporting service - return fullGroups; - }; - - const loadParentGroup = async (groupInput: Group): Promise => - asyncPipe( - R.prop('path'), - R.split('/'), - R.nth(-2), - R.cond([[R.isEmpty, R.always(null)], [R.T, loadGroupByName]]) - )(groupInput); - - const filterGroupsByAttribute = ( - groups: Group[], - filterFn: AttributeFilterFn - ): Group[] => - R.filter((group: Group) => - R.pipe( - R.toPairs, - R.reduce((isMatch: boolean, attribute: [string, string[]]): boolean => { - if (!isMatch) { - return filterFn({ - name: attribute[0], - value: attribute[1] - }); - } - return isMatch; - }, false) - )(group.attributes) - )(groups); - - const loadGroupsByAttribute = async ( - filterFn: AttributeFilterFn - ): Promise => { - const keycloakGroups = await keycloakAdminClient.groups.find({briefRepresentation: false}); + /** + * Loads a "full" group from keycloak. + * @deprecated Use `loadSparseGroupByName` instead. + */ + const loadGroupByName = async (name: string): Promise => { + let fullGroup: Group | null = null; - const filteredGroups = filterGroupsByAttribute(keycloakGroups, filterFn); + try { + fullGroup = await getGroupCacheByName(name); + } catch (err: unknown) { + if (err instanceof RedisCacheLoadError) { + logger.info(err.message); + } else { + throw err; + } + } - return await transformKeycloakGroups(filteredGroups); - }; + if (!fullGroup) { + const keycloakGroups = (await keycloakAdminClient.groups.find({ + search: name, + exact: true, // Exact name match + briefRepresentation: false, // Returns attributes + // @ts-ignore https://github.com/keycloak/keycloak/issues/32136 + populateHierarchy: false, // Return a single group instead of the full tree + })) as KeycloakLagoonGroup[]; - const loadGroupsByProjectId = async (projectId: number): Promise => { - const filterFn = attribute => { - if (attribute.name === 'lagoon-projects') { - const value = R.is(Array, attribute.value) - ? R.path(['value', 0], attribute) - : attribute.value; - return R.test(new RegExp(`\\b${projectId}\\b`), value); + if (keycloakGroups.length === 0) { + throw new GroupNotFoundError(`Group not found: ${name}`); } - return false; - }; - - // this request will be huge, but it is significantly faster than the alternative iteration that followed previously - // briefRepresentation pulls all the group information from keycloak including the attributes - // this means we don't need to iterate over all the groups one by one anymore to get the full group information - const groups = await keycloakAdminClient.groups.find({briefRepresentation: false}); + if (keycloakGroups.length > 1) { + throw new Error( + `Too many results (${keycloakGroups.length}) for ${name}.`, + ); + } - const filteredGroups = filterGroupsByAttribute(groups, filterFn); + const groupWithSubgroups = await loadKeycloakChildren(keycloakGroups[0]); + const groups = await transformKeycloakGroups([groupWithSubgroups]); + fullGroup = groups[0]; - const fullGroups = await transformKeycloakGroups(filteredGroups); + try { + await saveGroupCache(fullGroup); + } catch (err: unknown) { + if (err instanceof RedisCacheSaveError) { + logger.info(err.message); + } else { + throw err; + } + } + } - return fullGroups; + return fullGroup; }; + const loadSparseGroupByIdOrName = async (groupInput: { + id?: string; + name?: string; + }): Promise => { + if (groupInput.id) { + return loadSparseGroupById(groupInput.id); + } else if (groupInput.name) { + return loadSparseGroupByName(groupInput.name); + } else { + throw new Error('You must provide a group id or name'); + } + }; - // loadGroupsByProjectIdFromGroups does the same thing as loadGroupsByProjectId, except takes a groups input in the arguments - // from another source that has already calculated the required groups - const loadGroupsByProjectIdFromGroups = async (projectId: number, groups: Group[]): Promise => { - const filterFn = attribute => { - if (attribute.name === 'lagoon-projects') { - const value = R.is(Array, attribute.value) - ? R.path(['value', 0], attribute) - : attribute.value; - return R.test(new RegExp(`\\b${projectId}\\b`), value); - } - - return false; - }; - const filteredGroups = filterGroupsByAttribute(groups, filterFn); + /** + * Loads a "full" group from keycloak. + * @deprecated Use `loadSparseGroupByIdOrName` instead. + */ + const loadGroupByIdOrName = async ( + groupInput: GroupInput, + ): Promise => { + if (groupInput.id) { + return loadGroupById(groupInput.id); + } else if (groupInput.name) { + return loadGroupByName(groupInput.name); + } else { + throw new Error('You must provide a group id or name'); + } + }; - const fullGroups = await transformKeycloakGroups(filteredGroups); + const loadAllGroups = async (): Promise => { + // briefRepresentation pulls all the group information from keycloak + // including the attributes this means we don't need to iterate over all the + // groups one by one anymore to get the full group information + const keycloakGroups = await keycloakAdminClient.groups.find({ + briefRepresentation: false, + }); + + let fullGroups: KeycloakLagoonGroup[] = []; + for (const group of keycloakGroups) { + fullGroups.push(await loadKeycloakChildren(group)); + } + // no need to transform, just return the full response, only the `allGroups` + // resolvers use this and the `sync-groups-opendistro-security` consumption + // of this helper sync script is going to go away in the future when we move + // to the `lagoon-opensearch-sync` supporting service return fullGroups; }; - // used by organization resolver to list all groups attached to the organization - const loadGroupsByOrganizationId = async (organizationId: number): Promise => { - const filterFn = attribute => { - if (attribute.name === 'lagoon-organization') { - const value = R.is(Array, attribute.value) - ? R.path(['value', 0], attribute) - : attribute.value; - return R.test(new RegExp(`\\b${organizationId}\\b`), value); - } - - return false; - }; + const loadSubGroups = async ( + group: SparseGroup | HieararchicalGroup, + depth: number | null = null, + ): Promise => { + // Subgroups already loaded. + if ("groups" in group && "allGroups" in group) { + return group; + } - let groupIds = [] - const keycloakGroups = await keycloakAdminClient.groups.find(); - // @ts-ignore - groupIds = R.pluck('id', keycloakGroups); + // There are no subgroups to load. + if (group.subGroupCount < 1 || depth === 0) { + return { + ...group, + allGroups: [], + groups: [], + }; + } - let fullGroups = []; - for (const id of groupIds) { - const fullGroup = await keycloakAdminClient.groups.findOne({ - id + let keycloakGroups = await keycloakAdminClient.groups.listSubGroups({ + parentId: group.id, + max: group.subGroupCount, + briefRepresentation: false, // Returns attributes + }); + + let subGroups: HieararchicalGroup[] = []; + for (const keycloakGroup of keycloakGroups) { + const subGroup = createGroupFromKeycloak(keycloakGroup); + + const groupWithSubgroups = await loadSubGroups(subGroup, depth ? depth -1 : null); + subGroups.push({ + ...subGroup, + allGroups: groupWithSubgroups.allGroups, + groups: groupWithSubgroups.allGroups.filter((group: HieararchicalGroup): boolean => + !isInternalGroup(group) + ), }); - fullGroups = [...fullGroups, fullGroup]; } - try { - const filteredGroups = filterGroupsByAttribute(fullGroups, filterFn); - const groups = await transformKeycloakGroups(filteredGroups); - return groups; - } catch (err) { - return null - } + const groupWithSubgroups = { + ...group, + allGroups: subGroups, + groups: subGroups.filter((group: HieararchicalGroup): boolean => + !isInternalGroup(group) + ), + }; + return groupWithSubgroups }; - // used by organization resolver to list all groups attached to the organization - const loadGroupsByOrganizationIdFromGroups = async (organizationId: number, groups: Group[]): Promise => { - const filterFn = attribute => { - if (attribute.name === 'lagoon-organization') { - const value = R.is(Array, attribute.value) - ? R.path(['value', 0], attribute) - : attribute.value; - return R.test(new RegExp(`\\b${organizationId}\\b`), value); - } + const loadKeycloakChildren = async(group: KeycloakLagoonGroup): Promise => { + let keycloakGroups = await keycloakAdminClient.groups.listSubGroups({ + parentId: group.id, + briefRepresentation: false, // Returns attributes + }); + + let subGroups: KeycloakLagoonGroup[] = []; + for (const keycloakGroup of keycloakGroups) { + const groupWithSubgroups = await loadKeycloakChildren(keycloakGroup); + subGroups.push({ + ...keycloakGroup, + subGroups: groupWithSubgroups.subGroups, + }); + } - return false; + return { + ...group, + subGroups, }; - const filteredGroups = filterGroupsByAttribute(groups, filterFn); - - const fullGroups = await transformKeycloakGroups(filteredGroups); + } - return fullGroups; - }; + const loadParentGroup = async (groupInput: Group): Promise => + asyncPipe( + R.prop('path'), + R.split('/'), + R.nth(-2), + R.cond([[R.isEmpty, R.always(null)], [R.T, loadGroupByName]]) + )(groupInput); // Recursive function to load membership "up" the group chain const getMembersFromGroupAndParents = async ( @@ -543,74 +650,87 @@ export const Group = (clients: { return membership.length; }; - const addGroup = async (groupInput: Group, projectId?: number, organizationId?: number): Promise => { + const addGroup = async ( + groupInput: Group, + projectId?: number, + organizationId?: number, + ): Promise => { // Don't allow duplicate subgroup names try { - const existingGroup = await loadGroupByName(groupInput.name); - throw new Error('group-with-name-exists'); - } catch (err) { + await loadSparseGroupByName(groupInput.name); + throw new GroupExistsError(`Group ${R.prop('name', groupInput)} exists`); + } catch (err: unknown) { if (err instanceof GroupNotFoundError) { // No group exists with this name already, continue - } else if (err.message == 'group-with-name-exists') { - throw new GroupExistsError( - `Group ${R.prop('name', groupInput)} exists` - ); } else { throw err; } } + // Check if parent group exists + let parentGroup: SparseGroup | undefined; + if (groupInput?.parentGroupId) { + try { + parentGroup = await loadSparseGroupById(groupInput.parentGroupId); + } catch (err: unknown) { + if (err instanceof GroupNotFoundError) { + throw new GroupNotFoundError( + `Parent group not found ${R.prop('parentGroupId', groupInput)}`, + ); + } + } + } + let response: { id: string }; try { // @ts-ignore response = await keycloakAdminClient.groups.create({ - ...pickNonNil(['id', 'name', 'attributes'], groupInput) + ...pickNonNil(['id', 'name', 'attributes'], groupInput), }); - } catch (err) { - if (err.response.status && err.response.status === 409) { + } catch (err: unknown) { + if (isNetworkError(err) && err.response.status === 409) { throw new GroupExistsError( - `Group ${R.prop('name', groupInput)} exists` + `Group ${R.prop('name', groupInput)} exists`, ); - } else { + } else if (err instanceof Error) { throw new Error(`Error creating Keycloak group: ${err.message}`); + } else { + throw err; } } - const group = await loadGroupById(response.id); + const group = (await loadGroupById(response.id)) as Group & { id: string }; if (projectId) { - await groupHelpers(sqlClientPool).addProjectToGroup(projectId, group.id) + await groupHelpers(sqlClientPool).addProjectToGroup(projectId, group.id); } if (organizationId) { - await groupHelpers(sqlClientPool).addOrganizationToGroup(organizationId, group.id) + await groupHelpers(sqlClientPool).addOrganizationToGroup( + organizationId, + group.id, + ); } // Set the parent group - if (R.prop('parentGroupId', groupInput)) { + if (parentGroup) { try { - const parentGroup = await loadGroupById( - R.prop('parentGroupId', groupInput) - ); - - await keycloakAdminClient.groups.setOrCreateChild( + await keycloakAdminClient.groups.updateChildGroup( { - id: parentGroup.id + id: parentGroup.id, }, { id: group.id, - name: group.name - } + name: group.name, + }, ); - } catch (err) { - if (err instanceof GroupNotFoundError) { - throw new GroupNotFoundError( - `Parent group not found ${R.prop('parentGroupId', groupInput)}` - ); - } else if ( - err.message.includes('location header is not found in request') - ) { - // This is a bug in the keycloak client, ignore + } catch (err: unknown) { + if (err instanceof Error) { + if (err.message.includes('location header is not found in request')) { + // This is a bug in the keycloak client, ignore + } else { + throw Error(`Could not set parent group: ${err.message}`); + } } else { - logger.error(`Could not set parent group: ${err.message}`); + throw err; } } } @@ -619,90 +739,118 @@ export const Group = (clients: { }; const updateGroup = async (groupInput: GroupEdit): Promise => { - const oldGroup = await loadGroupById(groupInput.id); + const oldGroup = await loadSparseGroupById(groupInput.id); try { - await purgeGroupCache(oldGroup) await keycloakAdminClient.groups.update( { - id: groupInput.id + id: groupInput.id, }, //@ts-ignore { - ...pickNonNil(['name', 'attributes'], groupInput) - } + ...pickNonNil(['name', 'attributes'], groupInput), + }, ); - } catch (err) { - if (err.response.status && err.response.status === 409) { - throw new GroupExistsError( - `Group ${R.prop('name', groupInput)} exists` - ); - } else if (err.response.status && err.response.status === 404) { - throw new GroupNotFoundError(`Group not found: ${groupInput.id}`); - } else { + } catch (err: unknown) { + if (isNetworkError(err)) { + if (err.response.status === 409) { + throw new GroupExistsError( + `Group ${R.prop('name', groupInput)} exists`, + ); + } else if (err.response.status === 404) { + throw new GroupNotFoundError(`Group not found: ${groupInput.id}`); + } + } else if (err instanceof Error) { throw new Error(`Error updating Keycloak group: ${err.message}`); + } else { + throw err; } } - let newGroup = await loadGroupById(groupInput.id); - - if (oldGroup.name != newGroup.name) { - const roleSubgroups = newGroup.subGroups.filter(isRoleSubgroup); - - for (const roleSubgroup of roleSubgroups) { - await updateGroup({ - id: roleSubgroup.id, - name: R.replace(oldGroup.name, newGroup.name, roleSubgroup.name) - }); + if (groupInput.name && oldGroup.name != groupInput.name) { + try { + const newGroup = await loadSparseGroupById(groupInput.id); + const fullGroup = await loadSubGroups(newGroup, 1); + for (const subGroup of fullGroup.allGroups) { + if (subGroup.type == GroupType.ROLE_SUBGROUP) { + await keycloakAdminClient.groups.update( + { + id: subGroup.id, + }, + { + name: R.replace(oldGroup.name, groupInput.name, subGroup.name), + }, + ); + } + } + } catch (err: unknown) { + await purgeGroupCache(oldGroup); + if (err instanceof Error) { + throw new Error( + `Error renaming role subgroups from ${oldGroup.name} to ${groupInput.name}`, + ); + } else { + throw err; + } } } - return newGroup; + + await purgeGroupCache(oldGroup); + return await loadGroupById(groupInput.id); }; const deleteGroup = async (id: string): Promise => { try { - const keycloakGroup = await keycloakAdminClient.groups.findOne({ - id, - briefRepresentation: false, - }); - await groupHelpers(sqlClientPool).deleteGroup(id) - await purgeGroupCache(keycloakGroup) + const group = await loadSparseGroupById(id); + await groupHelpers(sqlClientPool).deleteGroup(id); + await purgeGroupCache(group); await keycloakAdminClient.groups.del({ id }); - } catch (err) { - if (err.response.status && err.response.status === 404) { - throw new GroupNotFoundError(`Group not found: ${id}`); + } catch (err: unknown) { + if (err instanceof GroupNotFoundError) { + throw err; + } else if (err instanceof Error) { + throw new Error(`Error deleting group ${id}: ${err.message}`); } else { - throw new Error(`Error deleting group ${id}: ${err}`); + throw err; } } }; const addUserToGroup = async ( user: User, - groupInput: Group, - roleName: string + groupInput: GroupInput, + roleName: string, ): Promise => { - const group = await loadGroupById(groupInput.id); + const group = await loadSparseGroupByIdOrName(groupInput); + // Load or create the role subgroup. - let roleSubgroup: Group; - // @ts-ignore - roleSubgroup = R.find(R.propEq('name', `${group.name}-${roleName}`))( - group.subGroups - ); - if (!roleSubgroup) { - roleSubgroup = await addGroup({ + let roleSubgroupID: string | undefined; + try { + const roleSubgroup = await loadSparseGroupByName( + `${group.name}-${roleName}`, + ); + roleSubgroupID = roleSubgroup.id; + } catch (err: unknown) { + if (!(err instanceof GroupNotFoundError)) { + throw err; + } + } + + if (!roleSubgroupID) { + const roleSubgroup = await addGroup({ name: `${group.name}-${roleName}`, parentGroupId: group.id, attributes: { - type: ['role-subgroup'] - } + type: ['role-subgroup'], + }, }); + roleSubgroupID = roleSubgroup.id; const role = await keycloakAdminClient.roles.findOneByName({ - name: roleName + name: roleName, }); await keycloakAdminClient.groups.addRealmRoleMappings({ - id: roleSubgroup.id, - roles: [{ id: role.id, name: role.name }] + id: roleSubgroupID, + roles: [{ id: role.id, name: role.name }], }); } @@ -710,21 +858,25 @@ export const Group = (clients: { try { await keycloakAdminClient.users.addToGroup({ id: user.id, - groupId: roleSubgroup.id + groupId: roleSubgroupID, }); - } catch (err) { - throw new Error(`Could not add user to group: ${err.message}`); + } catch (err: unknown) { + if (err instanceof Error) { + throw new Error(`Could not add user to group: ${err.message}`); + } else { + throw err; + } } - await purgeGroupCache(group) + await purgeGroupCache(group); return await loadGroupById(group.id); }; const removeUserFromGroup = async ( user: User, - group: Group + group: Group, ): Promise => { // purge the caches to ensure current data - await purgeGroupCache(group, true) + await purgeGroupCache(group, true); const members = await getGroupMembership(group); const userMembership = R.find(R.pathEq(['user', 'id'], user.id))(members); @@ -735,90 +887,109 @@ export const Group = (clients: { // @ts-ignore id: userMembership.user.id, // @ts-ignore - groupId: userMembership.roleSubgroupId + groupId: userMembership.roleSubgroupId, }); - } catch (err) { - throw new Error(`Could not remove user from group: ${err.message}`); + } catch (err: unknown) { + if (err instanceof Error) { + throw new Error(`Could not remove user from group: ${err.message}`); + } else { + throw err; + } } // purge after removing the user to ensure current data - await purgeGroupCache(group) + await purgeGroupCache(group); } - return await loadGroupById(group.id); + return await loadGroupByIdOrName(group); }; const addProjectToGroup = async ( projectId: number, - groupInput: any + groupInput: Group, ): Promise => { - const group = await loadGroupById(groupInput.id); - let newGroupProjects = "" - if (projectId) { - const groupProjectIds = getProjectIdsFromGroup(group) - newGroupProjects = R.pipe( - R.append(projectId), - R.uniq, - R.join(',') - // @ts-ignore - )(groupProjectIds); + const group = await loadSparseGroupByIdOrName(groupInput); + group.projects.add(projectId); + + const attributes: KeycloakLagoonGroupAttributes = {}; + if (group.type) { + attributes.type = [group.type]; + } + if (group.organization) { + attributes['lagoon-organization'] = [group.organization.toString()]; } + attributes['lagoon-projects'] = [Array.from(group.projects).join(',')]; + attributes['group-lagoon-project-ids'] = [ + `{${JSON.stringify(group.name)}:[${Array.from(group.projects).join(',')}]}`, + ]; + try { - await groupHelpers(sqlClientPool).addProjectToGroup(projectId, group.id) await keycloakAdminClient.groups.update( { - id: group.id + id: group.id, }, { name: group.name, - attributes: { - ...group.attributes, - 'lagoon-projects': [newGroupProjects], - 'group-lagoon-project-ids': [`{${JSON.stringify(group.name)}:[${newGroupProjects}]}`] - } - } + attributes, + }, ); + await groupHelpers(sqlClientPool).addProjectToGroup(projectId, group.id); // purge the caches to ensure current data - await purgeGroupCache(group) - } catch (err) { - throw new Error( - `Error setting projects for group ${group.name}: ${err.message}` - ); + await purgeGroupCache(group); + } catch (err: unknown) { + if (err instanceof Error) { + throw new Error( + `Error setting projects for group ${group.name}: ${err.message}`, + ); + } else { + throw err; + } } }; const removeProjectFromGroup = async ( projectId: number, - group: Group + groupInput: Group, ): Promise => { - const groupProjectIds = getProjectIdsFromGroup(group) - const newGroupProjects = R.pipe( - R.without([projectId]), - R.uniq, - R.join(',') - // @ts-ignore - )(groupProjectIds); + const group = await loadSparseGroupByIdOrName(groupInput); + group.projects.delete(projectId); + + const attributes: KeycloakLagoonGroupAttributes = {}; + if (group.type) { + attributes.type = [group.type]; + } + if (group.organization) { + attributes['lagoon-organization'] = [group.organization.toString()]; + } + + attributes['lagoon-projects'] = [Array.from(group.projects).join(',')]; + attributes['group-lagoon-project-ids'] = [ + `{${JSON.stringify(group.name)}:[${Array.from(group.projects).join(',')}]}`, + ]; try { - await groupHelpers(sqlClientPool).removeProjectFromGroup(projectId, group.id) await keycloakAdminClient.groups.update( { - id: group.id + id: group.id, }, { name: group.name, - attributes: { - ...group.attributes, - 'lagoon-projects': [newGroupProjects], - 'group-lagoon-project-ids': [`{${JSON.stringify(group.name)}:[${newGroupProjects}]}`] - } - } + attributes, + }, ); - // purge the caches to ensure current data - await purgeGroupCache(group) - } catch (err) { - throw new Error( - `Error setting projects for group ${group.name}: ${err.message}` + await groupHelpers(sqlClientPool).removeProjectFromGroup( + projectId, + group.id, ); + // purge the caches to ensure current data + await purgeGroupCache(group); + } catch (err: unknown) { + if (err instanceof Error) { + throw new Error( + `Error setting projects for group ${group.name}: ${err.message}`, + ); + } else { + throw err; + } } }; @@ -895,61 +1066,100 @@ export const Group = (clients: { return await loadGroupById(group.id); }; - const purgeGroupCache = async (group: Group, membersOnly: Boolean=false): Promise => { - if (!membersOnly) { - await del(`cache:keycloak:group-id:${group.name}`); - await del(`cache:keycloak:group:${group.id}`); + const purgeGroupCache = async ( + group: Pick, + membersOnly: Boolean = false, + ): Promise => { + try { + if (!membersOnly) { + // @ts-ignore + await del(`cache:keycloak:group-id:${group.name}`); + // @ts-ignore + await del(`cache:keycloak:group:${group.id}`); + } + // @ts-ignore + await del(`cache:keycloak:group-members:${group.id}`); + } catch (err: unknown) { + if (err instanceof Error) { + throw new RedisCacheSaveError(`Error deleting group cache: ${err.message}`); + } else { + throw err; + } } - await del(`cache:keycloak:group-members:${group.id}`); }; const saveGroupCache = async (group: Group): Promise => { + if (!group.id) { + throw new RedisCacheSaveError('Error saving group cache: Missing group ID'); + } + const idCacheKey = `cache:keycloak:group:${group.id}`; const nameCacheKey = `cache:keycloak:group-id:${group.name}`; - const data = Buffer.from(JSON.stringify(group)).toString('base64') - await redisClient.multi() - .set(nameCacheKey, group.id) - .expire(nameCacheKey, groupCacheExpiry) // 48 hours - .exec(); - await redisClient.multi() - .set(idCacheKey, data) - .expire(idCacheKey, groupCacheExpiry) // 48 hours - .exec(); + + try { + const data = Buffer.from(JSON.stringify(group)).toString('base64'); + await redisClient + .multi() + .set(nameCacheKey, group.id) + .expire(nameCacheKey, groupCacheExpiry) // 48 hours + .set(idCacheKey, data) + .expire(idCacheKey, groupCacheExpiry) // 48 hours + .exec(); + } catch (err: unknown) { + if (err instanceof Error) { + throw new RedisCacheSaveError(`Error saving group cache: ${err.message}`); + } else { + throw err; + } + } }; - const getGroupCacheByName =async (name: String) => { - const nameCacheKey = `cache:keycloak:group-id:${name}`; - let group; - let groupId = await get(nameCacheKey); - group = await getGroupCacheById(groupId) - } + const getGroupCacheByName = async (name: string): Promise => { + try { + const groupId = await get(`cache:keycloak:group-id:${name}`); + return groupId ? await getGroupCacheById(groupId) : null; + } catch (err: unknown) { + if (err instanceof RedisCacheLoadError) { + throw err; + } else if (err instanceof Error) { + throw new RedisCacheLoadError(`Error loading group cache by name: ${err.message}`); + } else { + throw err; + } + } - const getGroupCacheById = async (groupId: String): Promise => { - let group; - const idCacheKey = `cache:keycloak:group:${groupId}`; + return null; + }; + + const getGroupCacheById = async (groupId: string): Promise => { try { - let data = await get(idCacheKey); - if (data) { - let buff = Buffer.from(data, 'base64'); - group = JSON.parse(buff.toString('utf-8')); - return group + let data = await get(`cache:keycloak:group:${groupId}`); + + if (!data) { + return null; + } + + let buff = Buffer.from(data, 'base64'); + return JSON.parse(buff.toString('utf-8')); + } catch (err: unknown) { + if (err instanceof Error) { + throw new RedisCacheLoadError(`Error loading group cache by id: ${err.message}`); + } else { + throw err; } - } catch(err) { - logger.warn(`Error reading redis ${idCacheKey}, falling back to direct lookup: ${err.message}`); } - } + + return null; + }; return { loadAllGroups, loadGroupById, + loadSparseGroupByName, loadGroupByName, loadGroupByIdOrName, loadParentGroup, - loadGroupsByAttribute, - loadGroupsByProjectId, - loadGroupsByOrganizationId, - loadGroupsByOrganizationIdFromGroups, - loadGroupsByProjectIdFromGroups, + createGroupFromKeycloak, getProjectsFromGroupAndParents, getProjectsFromGroupAndSubgroups, getProjectsFromGroup, diff --git a/services/api/src/models/project.ts b/services/api/src/models/project.ts deleted file mode 100644 index 52ecc2ff7b..0000000000 --- a/services/api/src/models/project.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { Pool } from 'mariadb'; -import { Group } from './group'; -import { Helpers } from '../resources/project/helpers'; - -export interface Project { - id: Number; // int(11) NOT NULL AUTO_INCREMENT, - name: String; // varchar(100) COLLATE utf8_bin DEFAULT NULL, - git_url: String; // varchar(300) COLLATE utf8_bin DEFAULT NULL, - active_systems_deploy: String; // varchar(300) COLLATE utf8_bin DEFAULT NULL, - active_systems_remove: String; // varchar(300) COLLATE utf8_bin DEFAULT NULL, - branches: String; // varchar(300) COLLATE utf8_bin DEFAULT NULL, - pullrequests: String; // varchar(300) COLLATE utf8_bin DEFAULT NULL, - production_environment: String; // varchar(100) COLLATE utf8_bin DEFAULT NULL, - production_routes?: string; // text COLLATE utf8_bin DEFAULT NULL, - production_alias?: string; // varchar(100) COLLATE utf8_bin DEFAULT 'lagoon-production', - standby_production_environment: String; // varchar(100) COLLATE utf8_bin DEFAULT NULL, - standby_routes?: string; // text COLLATE utf8_bin DEFAULT NULL, - standby_alias?: string; // varchar(100) COLLATE utf8_bin DEFAULT 'lagoon-standby', - openshift: Number; // int(11) DEFAULT NULL, - created: String; // timestamp NOT NULL DEFAULT current_timestamp(), - active_systems_promote: String; // varchar(300) COLLATE utf8_bin DEFAULT NULL, - auto_idle: Boolean; // int(1) NOT NULL DEFAULT 1, - storage_calc: Boolean; // int(1) NOT NULL DEFAULT 1, - facts_ui: Boolean; // int(1) NOT NULL DEFAULT 1, - subfolder: String; // varchar(300) COLLATE utf8_bin DEFAULT NULL, - openshift_project_pattern: String; // varchar(300) COLLATE utf8_bin DEFAULT NULL, - development_environments_limit: Number; // int(11) DEFAULT NULL, - active_systems_task: String; // varchar(300) COLLATE utf8_bin DEFAULT NULL, - private_key: String; // varchar(5000) COLLATE utf8_bin DEFAULT NULL, - availability: String; // varchar(50) COLLATE utf8_bin DEFAULT NULL, - metadata: JSON; // JSON DEFAULT NULL, -} - -export const ProjectModel = (clients: { - sqlClientPool: Pool; - keycloakAdminClient: any; - redisClient: any; - esClient: any; -}) => { - const { sqlClientPool } = clients; - - const projectsByGroup = async (group: Group) => { - const GroupModel = Group(clients); - const projectIds = await GroupModel.getProjectsFromGroupAndSubgroups(group); - const projects = await Helpers(sqlClientPool).getProjectsByIds(projectIds); - return projects; - }; - - return { - projectsByGroup - }; -}; - -export default ProjectModel; diff --git a/services/api/src/models/user.ts b/services/api/src/models/user.ts index 89b1b7fcf8..c6f126d019 100644 --- a/services/api/src/models/user.ts +++ b/services/api/src/models/user.ts @@ -1,14 +1,12 @@ import * as R from 'ramda'; -import pickNonNil from '../util/pickNonNil'; -import { logger } from '../loggers/logger'; import type { UserRepresentation } from '@s3pweb/keycloak-admin-client-cjs'; -import { Group } from './group'; import { sqlClientPool } from '../clients/sqlClient'; +import pickNonNil from '../util/pickNonNil'; import { query } from '../util/db'; +import { toNumber } from '../util/func'; +import { Group, GroupType, KeycloakLagoonGroup } from './group'; import { Sql } from '../resources/user/sql'; import { getConfigFromEnv } from '../util/config'; -import { Helpers as groupHelpers } from '../resources/group/helpers'; -import { getRedisKeycloakCache } from '../clients/redisClient'; interface IUserAttributes { comment?: [string]; @@ -36,16 +34,18 @@ interface UserEdit { lastName?: string; comment?: string; gitlabId?: string; + organization?: number; + remove?: boolean; } -interface UserModel { +export interface UserModel { loadAllUsers: () => Promise; loadUserById: (id: string) => Promise; loadUserByUsername: (email: string) => Promise; loadUserByIdOrUsername: (userInput: UserEdit) => Promise; loadUsersByOrganizationId: (organizationId: number) => Promise; - getAllOrganizationIdsForUser: (userInput: User) => Promise; - getAllGroupsForUser: (userId: string) => Promise; + getAllOrganizationIdsForUser: (userInput: UserEdit) => Promise; + getAllGroupsForUser: (userId: string, organization?: number) => Promise; getAllProjectsIdsForUser: (userId: string, groups?: Group[]) => Promise<{}>; getUserRolesForProject: ( userInput: User, @@ -85,15 +85,6 @@ const lagoonOrganizationsLens = R.lensPath(['lagoon-organizations']); const lagoonOrganizationsAdminLens = R.lensPath(['lagoon-organizations-admin']); const lagoonOrganizationsViewerLens = R.lensPath(['lagoon-organizations-viewer']); -const attrLagoonProjectsLens = R.compose( - // @ts-ignore - attrLens, - lagoonOrganizationsLens, - lagoonOrganizationsAdminLens, - lagoonOrganizationsViewerLens, - R.lensPath([0]) -); - const attrLagoonOrgOwnerLens = R.compose( // @ts-ignore attrLens, @@ -124,7 +115,6 @@ const attrCommentLens = R.compose( export const User = (clients: { keycloakAdminClient: any; - redisClient: any; sqlClientPool: any; esClient: any; }): UserModel => { @@ -351,35 +341,44 @@ export const User = (clients: { return users; }; - const getAllGroupsForUser = async (userId: string, organization?: number): Promise => { + const getAllGroupsForUser = async ( + userId: string, + organization?: number, + ): Promise => { const GroupModel = Group(clients); - const roleSubgroups = await keycloakAdminClient.users.listGroups({ + const keycloakGroups = (await keycloakAdminClient.users.listGroups({ id: userId, - briefRepresentation: false - }); + briefRepresentation: false, + })) as KeycloakLagoonGroup[]; + const roleSubgroups = keycloakGroups.map( + GroupModel.createGroupFromKeycloak, + ); - const regexp = /-(owner|maintainer|developer|reporter|guest)$/g; - let userGroups = []; + let userGroups: { + [key: string]: Group; + } = {}; for (const ug of roleSubgroups) { - // push the group ids into an array of group ids only for sql lookups - let index = userGroups.findIndex((item) => item.name === ug.name.replace(regexp, "")); - if (index === -1) { - const parentGroup = await GroupModel.loadGroupByName(ug.name.replace(regexp, "")) - if (organization) { - const parentOrg = R.defaultTo('', R.prop('lagoon-organization', parentGroup.attributes)).toString() - const orgid = parentOrg.split(',')[0] - if (parseInt(orgid, 10) != organization) { - continue - } + if (ug.type !== GroupType.ROLE_SUBGROUP || !ug.parentGroupId) { + continue; + } + + if (!userGroups[ug.parentGroupId]) { + const parentGroup = await GroupModel.loadGroupById(ug.parentGroupId); + const parentOrg = parentGroup.attributes?.['lagoon-organization']?.[0]; + if (organization && parentOrg && toNumber(parentOrg) != organization) { + continue; } - // only set the users role-group as the subgroup, this is because `loadGroupByName` retrieves all the subgroups not just the one the user is in - parentGroup.subGroups = [ug] - userGroups.push(parentGroup); + // only set the users role-group as the subgroup, this is because + // `loadGroupByName` retrieves all the subgroups not just the one the + // user is in + parentGroup.subGroups = parentGroup.subGroups?.filter( + (g) => g.id === ug.id, + ); + userGroups[ug.parentGroupId] = parentGroup; } } - const retGroups = await GroupModel.transformKeycloakGroups(userGroups); - return retGroups; + return Object.values(userGroups); }; const getAllProjectsIdsForUser = async ( @@ -679,7 +678,7 @@ export const User = (clients: { }; const getAllOrganizationIdsForUser = async ( - userInput: User + userInput: UserEdit ): Promise => { let organizations = []; diff --git a/services/api/src/resources/index.ts b/services/api/src/resources/index.ts index 8d259322e3..f5f458d014 100644 --- a/services/api/src/resources/index.ts +++ b/services/api/src/resources/index.ts @@ -1,4 +1,7 @@ import { Pool } from 'mariadb'; +import type { UserModel } from '../models/user'; +import type { GroupModel } from '../models/group'; +import type { EnvironmentModel } from '../models/environment'; interface hasPermission { (resource: string, scope: any, attributes?: any): Promise; @@ -9,20 +12,19 @@ export interface ResolverFn { parent, args, context: { - sqlClientPool: Pool, - hasPermission: hasPermission, - keycloakGrant: any | null, - legacyGrant: any | null, - userActivityLogger: any | null, + sqlClientPool: Pool; + hasPermission: hasPermission; + keycloakGrant: any | null; + legacyGrant: any | null; + userActivityLogger: any | null; models: { - UserModel, - GroupModel, - ProjectModel, - EnvironmentModel, - }, - keycloakUsersGroups?: any | null, - adminScopes?: any | null, + UserModel: UserModel; + GroupModel: GroupModel; + EnvironmentModel: EnvironmentModel; + }; + keycloakUsersGroups?: any | null; + adminScopes?: any | null; }, - info? + info?, ): any; } diff --git a/services/api/src/resources/organization/resolvers.ts b/services/api/src/resources/organization/resolvers.ts index be7778c56d..7eeeced9a4 100644 --- a/services/api/src/resources/organization/resolvers.ts +++ b/services/api/src/resources/organization/resolvers.ts @@ -5,7 +5,7 @@ import { query, isPatchEmpty, knex } from '../../util/db'; import { Helpers as projectHelpers } from '../project/helpers'; import { Helpers} from './helpers'; import { Sql } from './sql'; -import { arrayDiff } from '../../util/func'; +import { arrayDiff, toNumber } from '../../util/func'; import { Helpers as openshiftHelpers } from '../openshift/helpers'; import { Helpers as notificationHelpers } from '../notification/helpers'; import { Helpers as groupHelpers } from '../group/helpers'; @@ -500,10 +500,9 @@ export const getGroupsByNameAndOrganizationId: ResolverFn = async ( }); const group = await models.GroupModel.loadGroupByName(name); - if (R.prop('lagoon-organization', group.attributes)) { - if (R.prop('lagoon-organization', group.attributes).toString() == organization) { - return group - } + const groupOrg = group.attributes?.['lagoon-organization']?.[0]; + if (groupOrg && toNumber(groupOrg) == organization) { + return group } } catch (err) { return []; diff --git a/services/api/src/resources/project/resolvers.ts b/services/api/src/resources/project/resolvers.ts index 787cefd81f..b4217a1498 100644 --- a/services/api/src/resources/project/resolvers.ts +++ b/services/api/src/resources/project/resolvers.ts @@ -565,7 +565,7 @@ export const deleteProject: ResolverFn = async ( // Remove the default project group try { - const group = await models.GroupModel.loadGroupByName( + const group = await models.GroupModel.loadSparseGroupByName( `project-${project.name}` ); await models.GroupModel.deleteGroup(group.id); @@ -841,7 +841,7 @@ export const updateProject: ResolverFn = async ( // Rename the default group and user if (patch.name && oldProject.name !== patch.name) { try { - const group = await models.GroupModel.loadGroupByName( + const group = await models.GroupModel.loadSparseGroupByName( `project-${oldProject.name}` ); await models.GroupModel.updateGroup({ diff --git a/services/api/src/resources/sshKey/sql.ts b/services/api/src/resources/sshKey/sql.ts index 3bc92f7543..aa9f18f8e6 100644 --- a/services/api/src/resources/sshKey/sql.ts +++ b/services/api/src/resources/sshKey/sql.ts @@ -48,7 +48,7 @@ export const Sql = { keyType, keyFingerprint, }: { - id: number, + id: number | null, name: string, keyValue: string, keyType: string, @@ -64,7 +64,7 @@ export const Sql = { }) .toString(), addSshKeyToUser: ( - { sshKeyId, userId }: { sshKeyId: number, userId: number }) => + { sshKeyId, userId }: { sshKeyId: number, userId: string }) => knex('user_ssh_key') .insert({ usid: userId, diff --git a/services/api/src/resources/user/resolvers.ts b/services/api/src/resources/user/resolvers.ts index 16bebc8a00..3719e12edd 100644 --- a/services/api/src/resources/user/resolvers.ts +++ b/services/api/src/resources/user/resolvers.ts @@ -47,7 +47,7 @@ export const getUserBySshKey: ResolverFn = async ( ); const userId = R.map(R.prop('usid'), rows); - const user = await models.UserModel.loadUserById(userId); + const user = await models.UserModel.loadUserById(userId[0]); return user; }; @@ -68,7 +68,7 @@ export const getUserBySshFingerprint: ResolverFn = async ( Sql.selectUserIdBySshFingerprint({keyFingerprint: fingerprint}), ); const userId = R.map(R.prop('usid'), rows); - const user = await models.UserModel.loadUserById(userId); + const user = await models.UserModel.loadUserById(userId[0]); return user; } catch (err) { throw new UserNotFoundError("No user found matching provided fingerprint"); From d7a076f9bf4f0532ddb6dbd1f499051f6a721f8d Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Wed, 4 Sep 2024 07:13:31 -0500 Subject: [PATCH 6/6] Upgrade to keycloak 24.0.5 --- services/api/src/models/user.ts | 51 +++++++------------ services/keycloak/Dockerfile | 2 +- .../themes/lagoon/account/theme.properties | 2 +- 3 files changed, 21 insertions(+), 34 deletions(-) diff --git a/services/api/src/models/user.ts b/services/api/src/models/user.ts index c6f126d019..4f0e98524f 100644 --- a/services/api/src/models/user.ts +++ b/services/api/src/models/user.ts @@ -549,41 +549,28 @@ export const User = (clients: { // set the last accessed as a unix timestamp on the user attributes // @TODO: no op last accessed for the time being due to raciness // @TODO: refactor later - /* - try { - const lastAccessed = {last_accessed: Math.floor(Date.now() / 1000)} - await keycloakAdminClient.users.update( - { - id: userInput.id - }, - { - attributes: { - ...userInput.attributes, - ...lastAccessed - } - } - ); - } catch (err) { - if (err.response.status && err.response.status === 404) { - throw new UserNotFoundError(`User not found: ${userInput.id}`); - } else { - logger.warn(`Error updating Keycloak user: ${err.message}`); - } - } - */ return true }; const updateUser = async (userInput: UserEdit): Promise => { - // comments used to be removed when updating a user, now they aren't - let organizations = null; - let organizationsAdmin = null; - let organizationsView = null; - let comment = null; // update a users organization if required, hooks into the existing update user function, but is used by the addusertoorganization resolver try { - // collect users existing attributes + let organizations = null; + let organizationsAdmin = null; + let organizationsView = null; + let comment = null; + + // Since keycloak 24, partial user updates are not supported. + // Get the current users data to use as default. const user = await loadUserById(userInput.id); + + const { + email: curEmail, + username: curUsername, + firstName: curFirstName, + lastName: curLastName + } = user; + // set the comment if provided if (R.prop('comment', userInput)) { comment = {comment: R.prop('comment', userInput)} @@ -619,10 +606,10 @@ export const User = (clients: { id: userInput.id }, { - ...pickNonNil( - ['email', 'username', 'firstName', 'lastName'], - userInput - ), + email: userInput.email ?? curEmail, + username: userInput.username ?? curUsername, + firstName: userInput.firstName ?? curFirstName, + lastName: userInput.lastName ?? curLastName, attributes: { ...user.attributes, ...organizations, diff --git a/services/keycloak/Dockerfile b/services/keycloak/Dockerfile index f881c9544f..7f6f11c2e9 100644 --- a/services/keycloak/Dockerfile +++ b/services/keycloak/Dockerfile @@ -17,7 +17,7 @@ COPY javascript /tmp/lagoon-scripts RUN cd /tmp/lagoon-scripts && zip -r ../lagoon-scripts.jar * -FROM quay.io/keycloak/keycloak:23.0.7 +FROM quay.io/keycloak/keycloak:24.0.5 COPY --from=ubi-micro-build /mnt/rootfs / ARG LAGOON_VERSION diff --git a/services/keycloak/themes/lagoon/account/theme.properties b/services/keycloak/themes/lagoon/account/theme.properties index 484ac246df..938f8f4f61 100644 --- a/services/keycloak/themes/lagoon/account/theme.properties +++ b/services/keycloak/themes/lagoon/account/theme.properties @@ -1,4 +1,4 @@ -parent=keycloak.v2 +parent=keycloak.v3 deprecatedMode=false developmentMode=false