From 89cee2bd362c7952d404d7840f01ec209732910b Mon Sep 17 00:00:00 2001 From: Yehiyam Livneh Date: Thu, 17 Feb 2022 17:20:28 +0200 Subject: [PATCH] add exec jobs api --- .../api-server/api/rest-api/routes/v1/exec.js | 5 + core/api-server/api/rest-api/swagger.json | 98 +++++++++++++++++-- core/api-server/lib/service/execution.js | 6 ++ core/api-server/lib/state/state-manager.js | 7 +- core/api-server/tests/exec-jobs.js | 95 ++++++++++++++++++ core/openapi-spec/swagger.json | 98 +++++++++++++++++-- .../swagger/definitions/common/queryList.yaml | 4 +- .../swagger/definitions/exec/jobListItem.yaml | 6 ++ .../swagger/definitions/index.yaml | 2 + .../openapi-spec/swagger/paths/exec/jobs.yaml | 36 +++++++ core/openapi-spec/swagger/paths/index.yaml | 2 + 11 files changed, 341 insertions(+), 18 deletions(-) create mode 100644 core/api-server/tests/exec-jobs.js create mode 100644 core/openapi-spec/swagger/definitions/exec/jobListItem.yaml create mode 100644 core/openapi-spec/swagger/paths/exec/jobs.yaml diff --git a/core/api-server/api/rest-api/routes/v1/exec.js b/core/api-server/api/rest-api/routes/v1/exec.js index 544f7b4fe..ef8dbc72f 100644 --- a/core/api-server/api/rest-api/routes/v1/exec.js +++ b/core/api-server/api/rest-api/routes/v1/exec.js @@ -60,6 +60,11 @@ const routes = (options) => { const response = await Execution.getRunningPipelines(); res.json(response); }); + router.all('/jobs', methods(['GET']), async (req, res,) => { + const { status, raw } = req.query; + const response = await Execution.getActivePipelines({ status, raw }); + res.json(response); + }); router.all('/status/:jobId?', methods(['GET']), async (req, res) => { const { jobId } = req.params; const response = await Execution.getJobStatus({ jobId }); diff --git a/core/api-server/api/rest-api/swagger.json b/core/api-server/api/rest-api/swagger.json index face1ccd8..c8848caa6 100644 --- a/core/api-server/api/rest-api/swagger.json +++ b/core/api-server/api/rest-api/swagger.json @@ -3,7 +3,7 @@ "info": { "title": "HKube API", "description": "HKube RESTful API", - "version": "2.2.10", + "version": "2.2.11", "contact": { "email": "hkube.dev@gmail.com" }, @@ -3072,6 +3072,82 @@ } } }, + "/exec/jobs": { + "get": { + "tags": [ + "Execution" + ], + "summary": "Get jobs (active and pending)", + "description": "Returns a list of all jobs (active and pending)", + "parameters": [ + { + "name": "status", + "in": "query", + "description": "status of the jobs to return", + "required": false, + "schema": { + "type": "string", + "enum": [ + "active", + "pending" + ] + } + }, + { + "name": "raw", + "in": "query", + "description": "if true returns just the jobId", + "required": false, + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "jobs list", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "status": { + "type": "string" + } + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, "/exec/status/{jobId}": { "get": { "tags": [ @@ -18973,6 +19049,18 @@ "jobId" ] }, + "jobListItem": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "status": { + "type": "string" + } + } + }, "tag": { "type": "string", "description": "Unique identifier representing a specific tag", @@ -18983,8 +19071,7 @@ "type": "object", "properties": { "name": { - "type": "string", - "default": "" + "type": "string" }, "experimentName": { "type": "string", @@ -19008,10 +19095,7 @@ "minimum": 1, "maximum": 100000 } - }, - "required": [ - "name" - ] + } }, "queryRange": { "type": "object", diff --git a/core/api-server/lib/service/execution.js b/core/api-server/lib/service/execution.js index 9505a8138..95b5c6ab9 100644 --- a/core/api-server/lib/service/execution.js +++ b/core/api-server/lib/service/execution.js @@ -246,6 +246,12 @@ class ExecutionService { return list.map(l => ({ jobId: l.jobId, ...l.pipeline })); } + async getActivePipelines({ status, raw } = {}) { + const active = await stateManager.getRunningJobs({ status }); + + return raw === 'true' ? active.map(f => (f.jobId)) : active; + } + async stopJob(options) { validator.executions.validateStopPipeline(options); const { jobId, reason } = options; diff --git a/core/api-server/lib/state/state-manager.js b/core/api-server/lib/state/state-manager.js index dcc9649c5..d4805a091 100644 --- a/core/api-server/lib/state/state-manager.js +++ b/core/api-server/lib/state/state-manager.js @@ -3,7 +3,7 @@ const storageManager = require('@hkube/storage-manager'); const { tracer } = require('@hkube/metrics'); const dbConnect = require('@hkube/db'); const Logger = require('@hkube/logger'); -const { buildStatuses } = require('@hkube/consts'); +const { buildStatuses, pipelineStatuses } = require('@hkube/consts'); const component = require('../consts/componentNames').DB; class StateManager { @@ -274,6 +274,11 @@ class StateManager { return this._db.jobs.fetch({ jobId, fields }); } + async getRunningJobs({ status } = {}) { + const statuses = status ? [status] : [pipelineStatuses.ACTIVE, pipelineStatuses.PENDING]; + return this._db.jobs.search({ pipelineStatus: { $in: statuses }, fields: { jobId: true, status: 'status.status' } }); + } + async getStatus(status) { return this._db.jobs.fetchStatus(status); } diff --git a/core/api-server/tests/exec-jobs.js b/core/api-server/tests/exec-jobs.js new file mode 100644 index 000000000..60c904b11 --- /dev/null +++ b/core/api-server/tests/exec-jobs.js @@ -0,0 +1,95 @@ +const { expect } = require('chai'); +const HttpStatus = require('http-status-codes'); +const stateManager = require('../lib/state/state-manager'); +const { pipelines } = require('./mocks'); +const { request } = require('./utils'); +let restUrl; + +describe('Executions', () => { + before(() => { + restUrl = global.testParams.restUrl; + }); + describe('/exec/jobs', () => { + let restPath = null; + before(() => { + restPath = `${restUrl}/exec/jobs`; + }); + beforeEach(async () => { + const list = await stateManager.getRunningJobs(); + for (const job of list) { + await stateManager._db.jobs.delete({jobId: job.jobId}); + } + }); + + it('should return empty list', async () => { + const options = { + method: 'GET', + uri: restPath, + }; + const response = await request(options); + expect(response.body).to.be.empty + }); + + it('should return active list', async () => { + const options = { + method: 'GET', + uri: `${restPath}?status=active`, + }; + for (let i=0;i<2;i++) { + + await stateManager._db.jobs.create({jobId: `job_${i}`, status: {status:'active'}, type: 'stored'}); + } + for (let i=2;i<6;i++) { + await stateManager._db.jobs.create({jobId: `job_${i}`, status: {status:'pending'}, type: 'stored'}); + } + const response = await request(options); + expect(response.body).to.have.length(2) + }); + + it('should return pending list', async () => { + const options = { + method: 'GET', + uri: `${restPath}?status=pending`, + }; + for (let i=0;i<2;i++) { + await stateManager._db.jobs.create({jobId: `job_${i}`, status: {status:'active'}, type: 'stored'}); + } + for (let i=2;i<6;i++) { + await stateManager._db.jobs.create({jobId: `job_${i}`, status: {status:'pending'}, type: 'stored'}); + } + const response = await request(options); + expect(response.body).to.have.length(4) + }); + + it('should return all list', async () => { + const options = { + method: 'GET', + uri: `${restPath}`, + }; + for (let i=0;i<111;i++) { + await stateManager._db.jobs.create({jobId: `job_${i}`, status: {status:'active'}, type: 'stored'}); + } + for (let i=111;i<300;i++) { + await stateManager._db.jobs.create({jobId: `job_${i}`, status: {status:'pending'}, type: 'stored'}); + } + const response = await request(options); + expect(response.body).to.have.length(300) + }); + + it('should return all list just jobIds', async () => { + const options = { + method: 'GET', + uri: `${restPath}?raw=true`, + }; + for (let i=0;i<111;i++) { + await stateManager._db.jobs.create({jobId: `job_${i}`, status: {status:'active'}, type: 'stored'}); + } + for (let i=111;i<300;i++) { + await stateManager._db.jobs.create({jobId: `job_${i}`, status: {status:'pending'}, type: 'stored'}); + } + const response = await request(options); + expect(response.body).to.have.length(300); + expect(response.body[0]).to.eql('job_0'); + }); + }); +}); diff --git a/core/openapi-spec/swagger.json b/core/openapi-spec/swagger.json index face1ccd8..c8848caa6 100644 --- a/core/openapi-spec/swagger.json +++ b/core/openapi-spec/swagger.json @@ -3,7 +3,7 @@ "info": { "title": "HKube API", "description": "HKube RESTful API", - "version": "2.2.10", + "version": "2.2.11", "contact": { "email": "hkube.dev@gmail.com" }, @@ -3072,6 +3072,82 @@ } } }, + "/exec/jobs": { + "get": { + "tags": [ + "Execution" + ], + "summary": "Get jobs (active and pending)", + "description": "Returns a list of all jobs (active and pending)", + "parameters": [ + { + "name": "status", + "in": "query", + "description": "status of the jobs to return", + "required": false, + "schema": { + "type": "string", + "enum": [ + "active", + "pending" + ] + } + }, + { + "name": "raw", + "in": "query", + "description": "if true returns just the jobId", + "required": false, + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "jobs list", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "status": { + "type": "string" + } + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, "/exec/status/{jobId}": { "get": { "tags": [ @@ -18973,6 +19049,18 @@ "jobId" ] }, + "jobListItem": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "status": { + "type": "string" + } + } + }, "tag": { "type": "string", "description": "Unique identifier representing a specific tag", @@ -18983,8 +19071,7 @@ "type": "object", "properties": { "name": { - "type": "string", - "default": "" + "type": "string" }, "experimentName": { "type": "string", @@ -19008,10 +19095,7 @@ "minimum": 1, "maximum": 100000 } - }, - "required": [ - "name" - ] + } }, "queryRange": { "type": "object", diff --git a/core/openapi-spec/swagger/definitions/common/queryList.yaml b/core/openapi-spec/swagger/definitions/common/queryList.yaml index 20033691d..5adfff05e 100644 --- a/core/openapi-spec/swagger/definitions/common/queryList.yaml +++ b/core/openapi-spec/swagger/definitions/common/queryList.yaml @@ -2,7 +2,6 @@ type: object properties: name: type: string - default: "" experimentName: $ref: "#/components/schemas/experimentName" default: main @@ -17,5 +16,4 @@ properties: default: 1 minimum: 1 maximum: 100000 -required: - - name + diff --git a/core/openapi-spec/swagger/definitions/exec/jobListItem.yaml b/core/openapi-spec/swagger/definitions/exec/jobListItem.yaml new file mode 100644 index 000000000..e99ad7804 --- /dev/null +++ b/core/openapi-spec/swagger/definitions/exec/jobListItem.yaml @@ -0,0 +1,6 @@ +type: object +properties: + jobId: + $ref: "#/components/schemas/jobId" + status: + type: string \ No newline at end of file diff --git a/core/openapi-spec/swagger/definitions/index.yaml b/core/openapi-spec/swagger/definitions/index.yaml index be5260398..9d54bf3c1 100644 --- a/core/openapi-spec/swagger/definitions/index.yaml +++ b/core/openapi-spec/swagger/definitions/index.yaml @@ -75,6 +75,8 @@ jobId: $ref: ./common/jobId.yaml jobIdObject: $ref: ./common/jobIdObject.yaml +jobListItem: + $ref: ./exec/jobListItem.yaml tag: $ref: ./common/tag.yaml queryList: diff --git a/core/openapi-spec/swagger/paths/exec/jobs.yaml b/core/openapi-spec/swagger/paths/exec/jobs.yaml new file mode 100644 index 000000000..911cdb279 --- /dev/null +++ b/core/openapi-spec/swagger/paths/exec/jobs.yaml @@ -0,0 +1,36 @@ +get: + tags: + - Execution + summary: Get jobs (active and pending) + description: Returns a list of all jobs (active and pending) + parameters: + - name: status + in: query + description: status of the jobs to return + required: false + schema: + type: string + enum: + - active + - pending + - name: raw + in: query + description: if true returns just the jobId + required: false + schema: + type: boolean + responses: + "200": + description: jobs list + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/jobListItem" + default: + description: Unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/error" diff --git a/core/openapi-spec/swagger/paths/index.yaml b/core/openapi-spec/swagger/paths/index.yaml index bb4d4cbd1..5aba6f8f4 100644 --- a/core/openapi-spec/swagger/paths/index.yaml +++ b/core/openapi-spec/swagger/paths/index.yaml @@ -21,6 +21,8 @@ $ref: ./exec/pipelines_jobId.yaml /exec/pipeline/list: $ref: ./exec/pipelines_list.yaml +/exec/jobs: + $ref: ./exec/jobs.yaml /exec/status/{jobId}: $ref: ./exec/status.yaml /exec/results/{jobId}: