Support datasource external storage

core/datasources-service/api/rest-api/routes/v1/datasource.js

@@ -42,10 +42,22 @@ const routes = () => {
             next();
         })
         .post(upload.array('files'), async (req, res, next) => {
-            const { name } = req.body;
+            const { name, storage, git } = req.body;
+            let gitConfig;
+            let storageConfig;
+            try {
+                gitConfig = git ? JSON.parse(git) : undefined;
+                storageConfig = storage ? JSON.parse(storage) : undefined;
+            } catch (error) {
+                throw new InvalidDataError(
+                    "invalid 'git' or 'storage' settings provided"
+                );
+            }
             try {
                 const response = await dataSource.create({
                     name,
+                    storage: storageConfig,
+                    git: gitConfig,
                     files: req.files,
                 });
                 res.status(HttpStatus.CREATED).json(response);

core/datasources-service/api/rest-api/swagger/definitions/create.request.yaml

@@ -4,6 +4,47 @@ properties:
         $ref: '#/components/schemas/DataSourceName'
     files:
         $ref: '#/components/schemas/Files'
+    storage:
+        type: object
+        properties:
+            accessKeyId:
+                type: string
+            secretAccessKey:
+                type: string
+            endpoint:
+                type: string
+            useSSL:
+                type: boolean
+            bucketName:
+                type: string
+        required:
+            - accessKeyId
+            - secretAccessKey
+            - endpoint
+            - bucketName
+    git:
+        type: object
+        properties:
+            organization:
+                type: string
+            endpoint:
+                type: string
+            token:
+                type: string
+            tokenName:
+                description: the token name is *required* when using a *gitlab* account
+                type: string
+            kind:
+                type: string
+                enum:
+                    - github
+                    - gitlab
+        required:
+            - endpoint
+            - token
+            - kind
 required:
     - name
     - files
+    - storage
+    - git

core/datasources-service/bootstrap.js

@@ -39,7 +39,7 @@ class Bootstrap {
                 true,
                 [storageManager.STORAGE_PREFIX.STORAGE_PREFIX.HKUBE_DATASOURCE]
             );
-            await storageManager.init(config, log );
+            await storageManager.init(config, log);
 
             for (const m of modules) {
                 await m.init(config);

core/datasources-service/config/main/config.base.js

@@ -1,5 +1,6 @@
 const packageJson = require(process.cwd() + '/package.json');
 const formatter = require(process.cwd() + '/lib/utils/formatters');
+const storageManager = require('@hkube/storage-manager');
 
 const config = {};
 config.serviceName = packageJson.name;
@@ -74,23 +75,43 @@ config.s3 = {
         'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
     endpoint: process.env.S3_ENDPOINT_URL || 'http://127.0.0.1:9000',
     useSSL: false,
+    bucketName: storageManager.STORAGE_PREFIX.STORAGE_PREFIX.HKUBE_DATASOURCE,
 };
 
-
 config.git = {
-    user: {
-        name: process.env.GIT_USER_NAME || 'hkube',
-        password: process.env.GIT_PASSWORD || '123456',
+    github: {
+        user: {
+            name: process.env.GIT_USER_NAME || 'hkube',
+            password: process.env.GIT_PASSWORD || '123456',
+        },
+        organization: process.env.GIT_ORGANIZATION || 'hkube-org',
+        endpoint: process.env.GITHUB_ENDPOINT_URL || 'http://localhost:3010',
+        /** @type {string} */
+        token: null,
+        kind: 'github',
+    },
+    gitlab: {
+        tokenName: process.env.GITLAB_TOKEN_NAME || `test-token`,
+        token: process.env.GITLAB_TOKEN || 'buBPxyFrzqHEfmthVSuy',
+        kind: 'gitlab',
+        endpoint: process.env.GITLAB_ENDPOINT_URL || 'http://localhost:3080',
+        test: {
+            gitlabTestEndpointToken: 'replace-me.com',
+        },
     },
-    endpoint: process.env.GIT_ENDPOINT_URL || 'localhost:3010',
 };
+
 config.fs = {
-    baseDirectory: process.env.BASE_FS_ADAPTER_DIRECTORY || '/var/tmp/fs/storage',
-    baseDatasourcesDirectory: process.env.BASE_DATASOURCES_DIRECTORY || '/var/tmp/fs/datasources-storage'
+    baseDirectory:
+        process.env.BASE_FS_ADAPTER_DIRECTORY || '/var/tmp/fs/storage',
+    baseDatasourcesDirectory:
+        process.env.BASE_DATASOURCES_DIRECTORY ||
+        '/var/tmp/fs/datasources-storage',
 };
 config.directories = {
     gitRepositories: 'temp/git-repositories',
-    dataSourcesInUse: process.env.DATASOURCES_IN_USE_FOLDER || 'dataSources-in-use',
+    dataSourcesInUse:
+        process.env.DATASOURCES_IN_USE_FOLDER || 'dataSources-in-use',
     prepareForDownload: 'temp/prepare-for-download',
     zipFiles: 'temp/zip-files',
 };

core/datasources-service/jsconfig.json

@@ -1,4 +1,7 @@
 {
-    "checkJs": true,
-    "target": "ES2020"
+    "compilerOptions": {
+        "checkJs": true,
+        "target": "ES2020",
+        "moduleResolution": "node"
+    }
 }

core/datasources-service/lib/service/dataSource.js

@@ -16,6 +16,8 @@ const { ResourceNotFoundError } = require('../errors');
  * @typedef {import('./../utils/types').SourceTargetArray} SourceTargetArray
  * @typedef {import('./../utils/types').config} config
  * @typedef {import('@hkube/db/lib/DataSource').DataSource} DataSourceItem;
+ * @typedef {import('@hkube/db/lib/DataSource').ExternalStorage} ExternalStorage;
+ * @typedef {import('@hkube/db/lib/DataSource').ExternalGit} ExternalGit;
  * @typedef {{ createdPath: string; fileName: string }} uploadFileResponse
  * @typedef {{ name?: string; id?: string }} NameOrId
  */
@@ -283,7 +285,9 @@ class DataSource {
         const repository = new Repository(
             name,
             this.config,
-            this.config.directories.gitRepositories
+            this.config.directories.gitRepositories,
+            createdVersion.repositoryUrl,
+            createdVersion._credentials
         );
 
         const { commitHash, files } = await this.commitChange({
@@ -304,19 +308,36 @@ class DataSource {
         });
     }
 
-    /** @param {{ name: string; files: MulterFile[] }} query */
-    async create({ name, files: _files }) {
-        validator.dataSources.create({ name, files: _files });
-        const createdDataSource = await this.db.dataSources.create({ name });
+    /**
+     * @param {{
+     *     name: string;
+     *     files: MulterFile[];
+     *     git: ExternalGit;
+     *     storage: ExternalStorage;
+     * }} query
+     */
+    async create({ name, git, storage, files: _files }) {
+        validator.dataSources.create({ name, git, storage, files: _files });
+        const createdDataSource = await this.db.dataSources.create({
+            name,
+            git,
+            storage,
+        });
         let updatedDataSource;
         /** @type {Repository} */
         const repository = new Repository(
             name,
             this.config,
-            this.config.directories.gitRepositories
+            this.config.directories.gitRepositories,
+            null,
+            { git, storage }
         );
         try {
-            await repository.setup();
+            const { repositoryUrl } = await repository.setup();
+            await this.db.dataSources.setRepositoryUrl(
+                { name },
+                { url: repositoryUrl }
+            );
             const { commitHash, files } = await this.commitChange({
                 repository,
                 commitMessage: 'initial upload',
@@ -368,10 +389,16 @@ class DataSource {
     // eslint-disable-next-line
     async sync({ name }) {
         validator.dataSources.sync({ name });
+        const {
+            _credentials,
+            repositoryUrl,
+        } = await this.db.dataSources.fetchWithCredentials({ name });
         const repository = new Repository(
             name,
             this.config,
-            this.config.directories.gitRepositories
+            this.config.directories.gitRepositories,
+            repositoryUrl,
+            _credentials
         );
         try {
             await repository.ensureClone();

core/datasources-service/lib/service/downloads.js

@@ -1,4 +1,5 @@
 const { uid } = require('@hkube/uid');
+const pathLib = require('path');
 const fse = require('fs-extra');
 const archiver = require('archiver');
 const {
@@ -31,9 +32,7 @@ class Downloads {
             const archive = archiver('zip', { zlib: { level: 9 } });
             archive.directory(rootDir, false);
             archive.finalize();
-            const output = fse.createWriteStream(
-                `${this.config.directories.zipFiles}/${downloadId}.zip`
-            );
+            const output = fse.createWriteStream(this.getZipPath(downloadId));
             archive.pipe(output);
             output.on('close', () => res(archive.pointer()));
             output.on('end', () => {
@@ -62,15 +61,20 @@ class Downloads {
             fileIds,
         });
         const downloadId = uid();
-        const dataSource = await this.db.dataSources.fetch({
+        const dataSource = await this.db.dataSources.fetchWithCredentials({
             id: dataSourceId,
         });
         const fileIdsSet = new Set(fileIds);
         // create a new directory with the downloadId as its name
         const repository = new Repository(
             dataSource.name,
             this.config,
-            `${this.config.directories.prepareForDownload}/${downloadId}`
+            pathLib.join(
+                this.config.directories.prepareForDownload,
+                downloadId
+            ),
+            dataSource.repositoryUrl,
+            dataSource._credentials
         );
 
         /** @type {{ filesToKeep: FileMeta[]; filesToDrop: FileMeta[] }} */
@@ -98,7 +102,10 @@ class Downloads {
 
     getZipPath(downloadId) {
         validator.downloads.validateDownloadId(downloadId);
-        return `${this.config.directories.zipFiles}/${downloadId}.zip`;
+        return pathLib.join(
+            this.config.directories.zipFiles,
+            `${downloadId}.zip`
+        );
     }
 }
 

core/datasources-service/lib/service/jobs-consumer.js

@@ -1,4 +1,5 @@
 const fse = require('fs-extra');
+const pathLib = require('path');
 const { Consumer } = require('@hkube/producer-consumer');
 const { taskStatuses } = require('@hkube/consts');
 const storageManager = require('@hkube/storage-manager');
@@ -94,10 +95,12 @@ class JobConsumer {
         let resolvedSnapshot;
         try {
             if (snapshot) {
-                resolvedSnapshot = await this.db.snapshots.fetchDataSource({
-                    snapshotName: snapshot.name,
-                    dataSourceName: dataSourceDescriptor.name,
-                });
+                resolvedSnapshot = await this.db.snapshots.fetchDataSourceWithCredentials(
+                    {
+                        snapshotName: snapshot.name,
+                        dataSourceName: dataSourceDescriptor.name,
+                    }
+                );
                 if (!resolvedSnapshot)
                     throw new ResourceNotFoundError(
                         'snapshot',
@@ -106,7 +109,7 @@ class JobConsumer {
                 dataSource = resolvedSnapshot.dataSource;
             } else {
                 const shouldGetLatest = !dataSourceDescriptor.version;
-                dataSource = await this.db.dataSources.fetch(
+                dataSource = await this.db.dataSources.fetchWithCredentials(
                     shouldGetLatest
                         ? { name: dataSourceDescriptor.name }
                         : { id: dataSourceDescriptor.version }
@@ -119,7 +122,9 @@ class JobConsumer {
         const repository = new Repository(
             dataSource.name,
             this.config,
-            `${this.rootDir}/${dataSource.name}/${dataSource.id}`
+            pathLib.join(this.rootDir, dataSource.name, dataSource.id),
+            dataSource.repositoryUrl,
+            dataSource._credentials
         );
 
         try {
@@ -192,7 +197,7 @@ class JobConsumer {
     }
 
     unmountDataSource(jobId) {
-        return fse.remove(`${this.rootDir}/${jobId}`);
+        return fse.remove(pathLib.join(this.rootDir, jobId));
     }
 }
 

core/datasources-service/lib/utils/GitRemoteClient/Base.js

@@ -0,0 +1,15 @@
+const Log = require('@hkube/logger');
+
+/** @typedef {import('./../types').gitConfig} gitConfig */
+
+/** @template T */
+class Base {
+    /** @param {T} config */
+    constructor(config, rawRepositoryUrl, serviceName) {
+        this.config = config;
+        this.rawRepositoryUrl = rawRepositoryUrl;
+        this.log = Log.GetLogFromContainer(serviceName);
+    }
+}
+
+module.exports = Base;