Graphql

core/api-server/.eslintrc.js

@@ -25,7 +25,7 @@ module.exports = {
     quotes: ["error", "single"],
     "comma-dangle": "off",
     "no-underscore-dangle": "off",
-    "linebreake-style": "off",
+    "linebreak-style": "off",
     "object-curly-newline": "off",
     "newline-per-chained-call": "off",
     "arrow-body-style": "off",

core/api-server/api/rest-api/app-server.js → core/api-server/api/app-server.js

@@ -4,10 +4,11 @@ const RestServer = require('@hkube/rest-server');
 const log = require('@hkube/logger').GetLogFromContanier();
 const { metrics } = require('@hkube/metrics');
 const HttpStatus = require('http-status-codes');
-const internal = require('./internal/index');
-const validator = require('../../lib/validation/api-validator');
-const component = require('../../lib/consts/componentNames').REST_API;
+const internal = require('./rest-api/internal/index');
+const validator = require('../lib/validation/api-validator');
+const component = require('../lib/consts/componentNames').REST_API;
 const rest = new RestServer();
+const graphqlServer = require('./graphql/graphql-server');
 const routeLogBlacklist = ['/metrics', '/swagger'];
 
 class AppServer {
@@ -28,16 +29,16 @@ class AppServer {
         const { prefix, port, rateLimit, poweredBy, bodySizeLimit } = options.rest;
         const routes = internal();
         routes.push(metrics.getRouter());
-        const versions = await fse.readdir(path.join(__dirname, 'routes'));
+        const versions = await fse.readdir(path.join(__dirname, './rest-api/routes'));
 
         await Promise.all(versions.map(async (v) => {
             swagger.servers.push({ url: path.join('/', options.swagger.path, prefix, v) });
-            const routers = await fse.readdir(path.join(__dirname, 'routes', v));
+            const routers = await fse.readdir(path.join(__dirname, './rest-api/routes', v));
             routers.forEach((f) => {
                 const file = path.basename(f, '.js');
                 routes.push({
                     route: path.join('/', prefix, v, file),
-                    router: require('./' + path.join('routes', v, file))({ ...options, version: v, file })  // eslint-disable-line
+                    router: require('./' + path.join('rest-api/routes', v, file))({ ...options, version: v, file })  // eslint-disable-line
                 });
             });
         }));
@@ -66,7 +67,9 @@ class AppServer {
                 }
             }
         };
+
         const data = await rest.start(opt);
+        graphqlServer(rest._app, rest._server, options.port, options.graphql);
         log.info(`🚀 ${data.message}`, { component });
     }
 }

core/api-server/api/graphql/graphql-schema.js

@@ -0,0 +1,51 @@
+const { gql } = require('apollo-server');
+const jobTypeDefs = require('./schemas/job-schema');
+const discoveryTypeDefs = require('./schemas/discovery-schema');
+const algorithmTypeDefs = require('./schemas/algorithm-schema');
+const pipelineTypeDefs = require('./schemas/pipeline-schema');
+const experimentTypeDefs = require('./schemas/experiment-schema');
+const nodeStatisticTypeDefs = require('./schemas/node-statistic-schema');
+const diskSpaceTypeDefs = require('./schemas/disk-space-schema');
+const pipelineStatsTypeDefs = require('./schemas/pipeline-stats-schema');
+const dataSourcesTypeDefs = require('./schemas/datasource-schema');
+const algorithmBuildsTypeDefs = require('./schemas/algorithm-builder-schema');
+const logsTypeDefs = require('./schemas/log-schema');
+const preferedTypeDefs = require('./schemas/prefered-schema');
+const errorLogsTypeDefs = require('./schemas/error-logs-schema');
+const SubscriptionIncNumbersTypeDefs = gql`
+type Subscription {
+    numberIncremented: Int
+  }
+  `;
+
+const Query = gql`  
+type Query {
+    currentNumber: Int
+  }  
+  `;
+const Subscription = gql`
+type Subscription {
+    numberIncremented: Int
+    numberIncrementedOdd(number: Int): Int
+  }
+  `;
+const types = [
+    dataSourcesTypeDefs,
+    algorithmBuildsTypeDefs,
+    jobTypeDefs,
+    algorithmTypeDefs,
+    pipelineTypeDefs,
+    experimentTypeDefs,
+    logsTypeDefs,
+    nodeStatisticTypeDefs,
+    diskSpaceTypeDefs,
+    discoveryTypeDefs,
+    pipelineStatsTypeDefs,
+    SubscriptionIncNumbersTypeDefs,
+    preferedTypeDefs,
+    errorLogsTypeDefs,
+    Query,
+    Subscription
+];
+
+module.exports = types;

core/api-server/api/graphql/graphql-server.js

@@ -0,0 +1,47 @@
+const { ApolloServer } = require('apollo-server-express');
+const { makeExecutableSchema } = require('@graphql-tools/schema');
+const log = require('@hkube/logger').GetLogFromContanier();
+const component = require('../../lib/consts/componentNames').GRAPHQL_SERVER;
+
+const _typeDefs = require('./graphql-schema');
+const _resolvers = require('./resolvers');
+
+async function startApolloServer(typeDefs, resolvers, app, httpServer, port, config) {
+    try {
+        const schema = makeExecutableSchema({
+            typeDefs,
+            resolvers,
+
+        });
+
+        const server = new ApolloServer({
+            schema,
+            plugins: [{
+                async serverWillStart() {
+                    return {
+                        async drainServer() {
+                            // subscriptionServer.close();
+                        }
+                    };
+                }
+            }],
+            introspection: config.introspection
+        });
+        await server.start();
+        server.applyMiddleware({ app });
+
+        log.info(`🚀 Query endpoint ready at http://localhost:${port}${server.graphqlPath}`, { component });
+        log.info(`🚀 Subscription endpoint ready at ws://localhost:${port}${server.graphqlPath}`, { component });
+    }
+    catch (error) {
+        log.error(`Error on running gr ${error}`,);
+    }
+}
+
+const graphqlServer = (app, httpServer, port, config) => {
+    startApolloServer(_typeDefs, _resolvers.getResolvers(), app, httpServer, port, config).catch(err => {
+        log.error(err, { component });
+    });
+};
+
+module.exports = graphqlServer;

core/api-server/api/graphql/queries/board-querier.js

@@ -0,0 +1,48 @@
+const storageManager = require('@hkube/storage-manager');
+
+const _mapBoards = (boards, key, innerKey) => {
+    return ((boards.length > 0) && boards.reduce((map, board) => {
+        const { id, status, boardReference } = board;
+        // eslint-disable-next-line no-param-reassign
+        map[board[key]] = map[board[key]] || {};
+        // eslint-disable-next-line no-param-reassign
+        map[board[key]][board[innerKey]] = { id, status, boardReference };
+        return map;
+    }, {})) || {};
+};
+const mapBoards = (boards) => {
+    const batch = boards.filter(board => (!board.taskId) && board.jobId);
+    const batchMap = _mapBoards(batch, 'jobId', 'nodeName');
+    const task = boards.filter(board => (board.taskId));
+    const taskMap = (_mapBoards(task, 'jobId', 'taskId'));
+    const node = boards.filter(board => (!board.jobId));
+    const nodeMap = _mapBoards(node, 'pipelineName', 'nodeName');
+    return { batchMap, taskMap, nodeMap };
+};
+
+const addHasMetricsToMap = async (nodeMap) => {
+    const pipelines = await storageManager.hkubeAlgoMetrics.listPipelines();
+    await Promise.all(pipelines.map(async pipeline => {
+        // eslint-disable-next-line no-param-reassign
+        nodeMap[pipeline] = nodeMap[pipeline] || {};
+        const nodes = await storageManager.hkubeAlgoMetrics.listNodes(pipeline);
+        nodes.forEach(nodeName => {
+            // eslint-disable-next-line no-param-reassign
+            nodeMap[pipeline][nodeName] = nodeMap[pipeline][nodeName] || {};
+            // eslint-disable-next-line no-param-reassign
+            nodeMap[pipeline][nodeName].hasMetrics = true;
+        });
+    }));
+};
+
+const getBoards = ({ node, job, batchMap, taskMap }) => {
+    if (node.batch && node.batch.some(task => (task.metricsPath && task.metricsPath.tensorboard.path))) {
+        return batchMap[job.key] && batchMap[job.key][node.nodeName] && [{ tensorboard: { board: batchMap[job.key][node.nodeName] } }];
+    }
+    if (node.metricsPath && node.metricsPath.tensorboard.path) {
+        return taskMap[job.key] && taskMap[job.key][node.taskId] && [{ tensorboard: { board: taskMap[job.key][node.taskId] } }];
+    }
+    return [];
+};
+
+module.exports = { mapBoards, getBoards, addHasMetricsToMap };

core/api-server/api/graphql/queries/dataSource-querier.js

@@ -0,0 +1,68 @@
+const { default: axios } = require('axios');
+const log = require('@hkube/logger').GetLogFromContainer();
+const component = require('../../../lib/consts/componentNames').GRAPHQL_SERVER;
+class DataSourceQuerier {
+    init(config) {
+        const { protocol, host, port, prefix } = config.dataSourceService;
+        this._baseUrl = `${protocol}://${host}:${port}/${prefix}`;
+        this.pathGenerator = {
+            dataSources: () => `${this._baseUrl}`,
+            dataSourceFiles: name => `${this._baseUrl}/${name}`,
+            versions: name => `${this._baseUrl}/${name}/versions`,
+            snapshot: name => `${this._baseUrl}/${name}/snapshot`,
+            // post
+            preview: id => `${this._baseUrl}/id/${id}/snapshot/preview`,
+        };
+    }
+
+    getDataSourcesList() {
+        return this._getRequest(this.pathGenerator.dataSources());
+    }
+
+    getDataSource(options) {
+        return this._getRequest(this.pathGenerator.dataSourceFiles(options.name), options);
+    }
+
+    getDataSourceVersions(options) {
+        return this._getRequest(this.pathGenerator.versions(options.name));
+    }
+
+    getDataSourceSnapshots(options) {
+        return this._getRequest(this.pathGenerator.snapshot(options.name));
+    }
+
+    postDataSourcePreviewQuery(options) {
+        return this._postRequest(this.pathGenerator.preview(options.id), options);
+    }
+
+    async getDataSourcesCount() {
+        const data = await this._getRequest(this.pathGenerator.dataSources());
+        if (data) {
+            return data.length;
+        }
+        return 0;
+    }
+
+    async _getRequest(url, options) {
+        try {
+            const { data } = await axios.get(url, { params: options });
+            return data;
+        }
+        catch (error) {
+            log.error(error.message, { component });
+        }
+        return null;
+    }
+
+    async _postRequest(url, options) {
+        try {
+            const { data } = await axios.post(url, options);
+            return data;
+        }
+        catch (error) {
+            log.error(error.message, { component });
+        }
+        return null;
+    }
+}
+module.exports = new DataSourceQuerier();