feat: use sshkey-handler to allow for more sshkey types

Makefile

@@ -160,6 +160,7 @@ services :=	api \
 			actions-handler \
 			backup-handler \
 			broker \
+			sshkey-handler \
 			keycloak \
 			keycloak-db \
 			logs2notifications \
@@ -185,6 +186,7 @@ build/api-redis: services/api-redis/Dockerfile
 build/actions-handler: services/actions-handler/Dockerfile
 build/backup-handler: services/backup-handler/Dockerfile
 build/broker: services/broker/Dockerfile
+build/sshkey-handler: services/sshkey-handler/Dockerfile
 build/keycloak-db: services/keycloak-db/Dockerfile
 build/keycloak: services/keycloak/Dockerfile
 build/logs2notifications: services/logs2notifications/Dockerfile
@@ -253,7 +255,7 @@ wait-for-keycloak:
 	grep -m 1 "Config of Keycloak done." <(docker-compose -p $(CI_BUILD_TAG) --compatibility logs -f keycloak 2>&1)
 
 # Define a list of which Lagoon Services are needed for running any deployment testing
-main-test-services = actions-handler broker logs2notifications api api-db api-redis keycloak keycloak-db ssh auth-server local-git local-api-data-watcher-pusher local-minio
+main-test-services = actions-handler broker sshkey-handler logs2notifications api api-db api-redis sshkey-handler keycloak keycloak-db ssh auth-server local-git local-api-data-watcher-pusher local-minio
 
 # List of Lagoon Services needed for webhook endpoint testing
 webhooks-test-services = webhook-handler webhooks2tasks backup-handler
@@ -351,15 +353,15 @@ local-dev-yarn-stop:
 
 .PHONY: ui-development
 ui-development: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis
+	IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db sshkey-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis
 
 .PHONY: api-development
 api-development: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db local-api-data-watcher-pusher keycloak keycloak-db broker api-redis
+	IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db sshkey-handler local-api-data-watcher-pusher keycloak keycloak-db broker api-redis
 
 .PHONY: ui-logs-development
 ui-logs-development: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db actions-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog
+	IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db actions-handler sshkey-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog
 
 ## CI targets
 
@@ -371,7 +373,7 @@ STERN_VERSION = v2.6.1
 CHART_TESTING_VERSION = v3.10.1
 K3D_IMAGE = docker.io/rancher/k3s:v1.28.6-k3s2
 TESTS = [nginx,api,features-kubernetes,bulk-deployment,features-kubernetes-2,features-variables,active-standby-kubernetes,tasks,drush,python,gitlab,github,bitbucket,services,workflows]
-CHARTS_TREEISH = main
+CHARTS_TREEISH = sshkey-handler
 TASK_IMAGES = task-activestandby
 
 # Symlink the installed kubectl client if the correct version is already
@@ -481,7 +483,7 @@ ifeq ($(ARCH), darwin)
       tcp-listen:32080,fork,reuseaddr tcp-connect:target:32080
 endif
 
-K3D_SERVICES = api api-db api-redis auth-server actions-handler broker keycloak keycloak-db logs2notifications webhook-handler webhooks2tasks local-api-data-watcher-pusher local-git ssh tests workflows $(TASK_IMAGES)
+K3D_SERVICES = api api-db api-redis auth-server actions-handler broker sshkey-handler keycloak keycloak-db logs2notifications webhook-handler webhooks2tasks local-api-data-watcher-pusher local-git ssh tests workflows $(TASK_IMAGES)
 K3D_TESTS = local-api-data-watcher-pusher local-git tests
 K3D_TOOLS = k3d helm kubectl jq stern
 
@@ -517,7 +519,7 @@ k3d/test: k3d/cluster helm/repos $(addprefix local-dev/,$(K3D_TOOLS)) build
 			"quay.io/helmpack/chart-testing:$(CHART_TESTING_VERSION)" \
 			ct install --helm-extra-args "--timeout 60m"
 
-LOCAL_DEV_SERVICES = api auth-server actions-handler logs2notifications webhook-handler webhooks2tasks
+LOCAL_DEV_SERVICES = api auth-server actions-handler sshkey-handler logs2notifications webhook-handler webhooks2tasks
 
 # install lagoon charts in a Kind cluster
 .PHONY: k3d/setup

docker-bake.hcl

@@ -51,6 +51,7 @@ group "default" {
     "auth-server",
     "backup-handler",
     "broker",
+    "sshkey-handler",
     "keycloak-db",
     "keycloak",
     "local-api-data-watcher-pusher",
@@ -72,6 +73,7 @@ group "ui-logs-development" {
     "api-redis",
     "api",
     "broker",
+    "sshkey-handler",
     "keycloak-db",
     "keycloak",
     "local-api-data-watcher-pusher",
@@ -95,6 +97,7 @@ group "prod-images" {
     "auth-server",
     "backup-handler",
     "broker",
+    "sshkey-handler",
     "keycloak-db",
     "keycloak",
     "logs2notifications",
@@ -182,6 +185,15 @@ target "broker" {
   tags = ["${IMAGE_REPO}/broker:${TAG}"]
 }
 
+target "sshkey-handler" {
+  inherits = ["default"]
+  context = "services/sshkey-handler"
+  labels = {
+    "org.opencontainers.image.title": "lagoon-core/sshkey-handler - the sshkey-handler service for Lagoon"
+  }
+  tags = ["${IMAGE_REPO}/sshkey-handler:${TAG}"]
+}
+
 target "keycloak" {
   inherits = ["default"]
   context = "services/keycloak"

docker-compose.yaml

@@ -26,6 +26,12 @@ services:
     ports:
       - '15672:15672'
       - '5672:5672'
+  sshkey-handler:
+    # this is neded for the internal dns references
+    container_name: sshkeyhandler
+    image: ${IMAGE_REPO:-lagoon}/sshkey-handler
+    ports:
+      - '3333:3333'
   logs2notifications:
     image: ${IMAGE_REPO:-lagoon}/logs2notifications
     environment:
@@ -43,6 +49,7 @@ services:
       - ./node-packages:/app/node-packages:delegated
     environment:
       - CONSOLE_LOGGING_LEVEL=trace
+      - SSHKEY_HANDLER_HOST=sshkeyhandler
   api-db-init:
     image: ${IMAGE_REPO:-lagoon}/api
     command: >
@@ -71,6 +78,8 @@ services:
         condition: service_completed_successfully # don't start the lagoon migrations until the db migrations is completed
       keycloak:
         condition: service_started
+      sshkey-handler:
+        condition: service_started
   api:
     image: ${IMAGE_REPO:-lagoon}/api
     command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index"
@@ -89,11 +98,14 @@ services:
       - S3_BAAS_ACCESS_KEY_ID=minio
       - S3_BAAS_SECRET_ACCESS_KEY=minio123
       - CONSOLE_LOGGING_LEVEL=debug
+      - SSHKEY_HANDLER_HOST=sshkeyhandler
     depends_on:
       api-lagoon-migrations:
         condition: service_started
       keycloak:
         condition: service_started
+      keycloak:
+        condition: service_started
     ports:
       - '3000:3000'
     # Uncomment for local new relic tracking

local-dev/api-data-watcher-pusher/api-data/03-populate-api-data-ci-local-control-k8s.gql

@@ -38,45 +38,54 @@ mutation PopulateApi {
   }
 
   ### SSH Keys
-  CiCustomerSshKeyRsa: addSshKey(
+  CiCustomerSshKeyRsa: addUserSSHPublicKey(
     input: {
       id: 4
       name: "ci-customer-sshkey-rsa"
-      keyValue: "AAAAB3NzaC1yc2EAAAADAQABAAACAQDEZlms5XsiyWjmnnUyhpt93VgHypse9Bl8kNkmZJTiM3Ex/wZAfwogzqd2LrTEiIOWSH1HnQazR+Cc9oHCmMyNxRrLkS/MEl0yZ38Q+GDfn37h/llCIZNVoHlSgYkqD0MQrhfGL5AulDUKIle93dA6qdCUlnZZjDPiR0vEXR36xGuX7QYAhK30aD2SrrBruTtFGvj87IP/0OEOvUZe8dcU9G/pCoqrTzgKqJRpqs/s5xtkqLkTIyR/SzzplO21A+pCKNax6csDDq3snS8zfx6iM8MwVfh8nvBW9seax1zBvZjHAPSTsjzmZXm4z32/ujAn/RhIkZw3ZgRKrxzryttGnWJJ8OFyF31JTJgwWWuPdH53G15PC83ZbmEgSV3win51RZRVppN4uQUuaqZWG9wwk2a6P5aen1RLCSLpTkd2mAEk9PlgmJrf8vITkiU9pF9n68ENCoo556qSdxW2pxnjrzKVPSqmqO1Xg5K4LOX4/9N4n4qkLEOiqnzzJClhFif3O28RW86RPxERGdPT81UI0oDAcU5euQr8Emz+Hd+PY1115UIld3CIHib5PYL9Ee0bFUKiWpR/acSe1fHB64mCoHP7hjFepGsq7inkvg2651wUDKBshGltpNkMj6+aZedNc0/rKYyjl80nT8g8QECgOSRzpmYp0zli2HpFoLOiWw=="
-      keyType: SSH_RSA
+      publicKey: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDEZlms5XsiyWjmnnUyhpt93VgHypse9Bl8kNkmZJTiM3Ex/wZAfwogzqd2LrTEiIOWSH1HnQazR+Cc9oHCmMyNxRrLkS/MEl0yZ38Q+GDfn37h/llCIZNVoHlSgYkqD0MQrhfGL5AulDUKIle93dA6qdCUlnZZjDPiR0vEXR36xGuX7QYAhK30aD2SrrBruTtFGvj87IP/0OEOvUZe8dcU9G/pCoqrTzgKqJRpqs/s5xtkqLkTIyR/SzzplO21A+pCKNax6csDDq3snS8zfx6iM8MwVfh8nvBW9seax1zBvZjHAPSTsjzmZXm4z32/ujAn/RhIkZw3ZgRKrxzryttGnWJJ8OFyF31JTJgwWWuPdH53G15PC83ZbmEgSV3win51RZRVppN4uQUuaqZWG9wwk2a6P5aen1RLCSLpTkd2mAEk9PlgmJrf8vITkiU9pF9n68ENCoo556qSdxW2pxnjrzKVPSqmqO1Xg5K4LOX4/9N4n4qkLEOiqnzzJClhFif3O28RW86RPxERGdPT81UI0oDAcU5euQr8Emz+Hd+PY1115UIld3CIHib5PYL9Ee0bFUKiWpR/acSe1fHB64mCoHP7hjFepGsq7inkvg2651wUDKBshGltpNkMj6+aZedNc0/rKYyjl80nT8g8QECgOSRzpmYp0zli2HpFoLOiWw=="
       user: {
         email: "ci-customer-user-rsa@example.com"
       }
     }
   ) {
     id
   }
-  CiCustomerSshKeyEd25519: addSshKey(
+  CiCustomerSshKeyEd25519: addUserSSHPublicKey(
     input: {
       id: 5
       name: "ci-customer-sshkey-ed25519"
-      keyValue: "AAAAC3NzaC1lZDI1NTE5AAAAIMdEs1h19jv2UrbtKcqPDatUxT9lPYcbGlEAbInsY8Ka"
-      keyType: SSH_ED25519
+      publicKey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMdEs1h19jv2UrbtKcqPDatUxT9lPYcbGlEAbInsY8Ka"
       user: {
         email: "ci-customer-user-ed25519@example.com"
       }
     }
   ) {
     id
   }
-  CiCustomerSshKeyEcdsa: addSshKey(
+  CiCustomerSshKeyEcdsa: addUserSSHPublicKey(
     input: {
       id: 6
       name: "ci-customer-sshkey-ecdsa"
-      keyValue: "AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAD8E5wfvLg8vvfO9mmHVsZQK8dNgdKM5FrTxL4ORDq66Z50O8zUzBwF1VTO5Zx+qwB7najMdWsnW00BC6PMysSNJQD5HI4CokyKqmGdeSXcROYwvYOjlDQ+jD5qOSmkllRZZnkEYXE5FVBXaZWToyfGUGIoECvKGUQZxkBDHsbK13JdfA=="
-      keyType: ECDSA_SHA2_NISTP521
+      publicKey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAD8E5wfvLg8vvfO9mmHVsZQK8dNgdKM5FrTxL4ORDq66Z50O8zUzBwF1VTO5Zx+qwB7najMdWsnW00BC6PMysSNJQD5HI4CokyKqmGdeSXcROYwvYOjlDQ+jD5qOSmkllRZZnkEYXE5FVBXaZWToyfGUGIoECvKGUQZxkBDHsbK13JdfA=="
       user: {
         email: "ci-customer-user-ecdsa@example.com"
       }
     }
   ) {
     id
   }
+  CiCustomerSshKeyEd25519SK: addUserSSHPublicKey(
+    input: {
+      id: 7
+      name: "ci-customer-sshkey-ed25519-sk"
+      publicKey: "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIPjqGSQd+w7qQxioI6qj+KWX/pEg9mNvVGZ7aUoXfsC0AAAABHNzaDo="
+      user: {
+        email: "ci-customer-user-ed25519@example.com"
+      }
+    }
+  ) {
+    id
+  }
 
   ### Add Users to Groups
   CiCustomerUserAddRsa: addUserToGroup(

node-packages/commons/src/util/func.ts

@@ -1,4 +1,7 @@
 import { unless, is, isNil, isEmpty, partialRight, complement } from 'ramda';
+import http from 'http';
+import querystring from 'querystring';
+import { getConfigFromEnv } from './config';
 
 export const isNumber = is(Number);
 export const isArray = is(Array);
@@ -28,3 +31,67 @@ export const jsonMerge = function(a, b, prop) {
 // a2 = [1,2,3,5]
 // arrayDiff(a1,a2) = [4]
 export const arrayDiff = (a:Array<any>, b:Array<any>) =>  a.filter(e => !b.includes(e));
+
+// helper that will use the crypto handler service to check if a public or private key is valid or not
+export async function validateKey(key, type) {
+  const data = querystring.stringify({'key': key});
+  const options = {
+      hostname: getConfigFromEnv("SSHKEY_HANDLER_HOST", "localhost"),
+      port: 3333,
+      path: `/validate/${type}`,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'Content-Length': Buffer.byteLength(data)
+      },
+  };
+  let p = new Promise((resolve, reject) => {
+      const req = http.request(options, (res) => {
+          res.setEncoding('utf8');
+          let responseBody = '';
+
+          res.on('data', (chunk) => {
+              responseBody += chunk;
+          });
+
+          res.on('end', () => {
+              resolve(JSON.parse(responseBody));
+          });
+      });
+      req.on('error', (err) => {
+          reject(err);
+      });
+      req.write(data)
+      req.end();
+  });
+  return await p;
+}
+
+// helper that will use the crypto handler service to generate a private key with associated public key
+export async function generatePrivateKey() {
+  const options = {
+      hostname: getConfigFromEnv("SSHKEY_HANDLER_HOST", "localhost"),
+      port: 3333,
+      path: '/generate/ed25519',
+      method: 'GET',
+  };
+  let p = new Promise((resolve, reject) => {
+      const req = http.request(options, (res) => {
+          res.setEncoding('utf8');
+          let responseBody = '';
+
+          res.on('data', (chunk) => {
+              responseBody += chunk;
+          });
+
+          res.on('end', () => {
+              resolve(JSON.parse(responseBody));
+          });
+      });
+      req.on('error', (err) => {
+          reject(err);
+      });
+      req.end();
+  });
+  return await p;
+}

services/api/database/migrations/20240304000000_sshkey.js

@@ -0,0 +1,41 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = async function(knex) {
+    return knex.schema.table('ssh_key', (table) => {
+        table.string('key_type_new').notNullable().defaultTo('ssh-rsa');
+    }).then(() => {
+        return knex('ssh_key').update({
+            key_type_new: knex.ref('key_type')
+        });
+    }).then(function () {
+        return knex.schema.table('ssh_key', (table) => {
+            table.dropColumn('key_type');
+            table.renameColumn('key_type_new', 'key_type');
+        })
+    });
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = async function(knex) {
+    return knex.schema.table('ssh_key', (table) => {
+        table.enu('key_type_new', ['ssh-rsa', 'ssh-ed25519','ecdsa-sha2-nistp256','ecdsa-sha2-nistp384','ecdsa-sha2-nistp521']).notNullable().defaultTo('ssh-rsa');
+    }).then(() => {
+        return knex('ssh_key')
+        .whereNotIn('key_type', ['ssh-rsa', 'ssh-ed25519','ecdsa-sha2-nistp256','ecdsa-sha2-nistp384','ecdsa-sha2-nistp521'])
+        .del();
+    }).then(function () {
+        return knex('ssh_key').update({
+            key_type_new: knex.ref('key_type')
+        });
+    }).then(function () {
+        return knex.schema.table('ssh_key', (table) => {
+            table.dropColumn('key_type');
+            table.renameColumn('key_type_new', 'key_type');
+        })
+    });
+};

services/api/package.json

@@ -60,7 +60,6 @@
     "ramda": "0.25.0",
     "redis": "^3.0.2",
     "snakecase-keys": "^1.2.0",
-    "sshpk": "^1.14.2",
     "validator": "^10.8.0",
     "winston": "^3",
     "winston-transport": "^4.4.0"

services/api/src/gitlab-sync/projects.ts

@@ -1,5 +1,4 @@
 import * as R from 'ramda';
-import * as sshpk from 'sshpk';
 import * as gitlabApi from '@lagoon/commons/dist/gitlab/api';
 import {
   sanitizeGroupName,
@@ -10,6 +9,7 @@ import {
   addUserToGroup
 } from '@lagoon/commons/dist/api';
 import { logger } from '@lagoon/commons/dist/logs/local-logger';
+import { validateKey } from '../util/func';
 
 interface GitlabProject {
   id: number,
@@ -56,14 +56,10 @@ const syncProject = async (project) => {
   }
 
   try {
-    const privateKey = R.pipe(
-      R.prop('privateKey'),
-      sshpk.parsePrivateKey,
-    )(lagoonProject);
-    //@ts-ignore
-    const publicKey = privateKey.toPublic();
+    const privkey = new Buffer((R.prop('privateKey', lagoonProject))).toString('base64')
+    const publickey = await validateKey(privkey, "private")
 
-    await gitlabApi.addDeployKeyToProject(id, publicKey.toString());
+    await gitlabApi.addDeployKeyToProject(id, publickey['publickey']);
   } catch (err) {
     if (!err.message.includes('has already been taken')) {
       throw new Error(`Could not add deploy_key to gitlab project ${id}, reason: ${err}`);