From 0da9f007a58f51d432c3562e4e2758c9f861211e Mon Sep 17 00:00:00 2001 From: yehiyam Date: Wed, 24 Nov 2021 10:40:24 +0200 Subject: [PATCH 1/4] update python wrapper --- .../environments/python/wrapper/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/algorithm-builder/environments/python/wrapper/requirements.txt b/core/algorithm-builder/environments/python/wrapper/requirements.txt index 891d2a41d..0c9e58d59 100644 --- a/core/algorithm-builder/environments/python/wrapper/requirements.txt +++ b/core/algorithm-builder/environments/python/wrapper/requirements.txt @@ -1 +1 @@ -hkube-python-wrapper==2.2.1 \ No newline at end of file +hkube-python-wrapper==2.2.2 \ No newline at end of file From 0e943dac1b24f6461c16d4b631c7b7d3e8b64b69 Mon Sep 17 00:00:00 2001 From: Yehiyam Livneh Date: Wed, 17 Nov 2021 10:25:02 +0200 Subject: [PATCH 2/4] disable datasources mount if service is disabled --- core/task-executor/lib/jobs/jobCreator.js | 10 +++++----- core/task-executor/tests/jobCreatorTests.js | 22 ++++++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/core/task-executor/lib/jobs/jobCreator.js b/core/task-executor/lib/jobs/jobCreator.js index a42340e62..bf652e5d7 100644 --- a/core/task-executor/lib/jobs/jobCreator.js +++ b/core/task-executor/lib/jobs/jobCreator.js @@ -187,11 +187,11 @@ const applyDevMode = (inputSpec, { algorithmOptions = {}, algorithmName, cluster return spec; }; -const applyDataSourcesVolumes = (inputSpec) => { +const applyDataSourcesVolumes = (inputSpec, clusterOptions) => { let spec = clonedeep(inputSpec); - // if (!clusterOptions.dataSourcesEnabled) { - // return spec; - // } + if (!clusterOptions?.datasourcesServiceEnabled) { + return spec; + } spec = applyVolumeMounts(spec, CONTAINERS.ALGORITHM, { name: 'datasources-storage', mountPath: '/hkube/datasources-storage' @@ -362,7 +362,7 @@ const createJobSpec = ({ kind, algorithmName, resourceRequests, workerImage, alg spec = applyJaeger(spec, CONTAINERS.WORKER, options); spec = applyJaeger(spec, CONTAINERS.ALGORITHM, options); spec = applyDevMode(spec, { options, algorithmOptions, clusterOptions, algorithmName }); - spec = applyDataSourcesVolumes(spec); + spec = applyDataSourcesVolumes(spec, clusterOptions); spec = applyMounts(spec, mounts); spec = applyImagePullSecret(spec, clusterOptions?.imagePullSecretName); diff --git a/core/task-executor/tests/jobCreatorTests.js b/core/task-executor/tests/jobCreatorTests.js index 029b604fa..519a9ee6c 100644 --- a/core/task-executor/tests/jobCreatorTests.js +++ b/core/task-executor/tests/jobCreatorTests.js @@ -163,7 +163,7 @@ describe('jobCreator', () => { expect(() => createJobSpec({ algorithmImage: 'myImage1', options })).to.throw('Unable to create job spec. algorithmName is required'); }); it('should apply all required properties', () => { - const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', options }); + const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', options, clusterOptions: { datasourcesServiceEnabled: true } }); expect(res).to.nested.include({ 'spec.template.spec.containers[1].image': 'myImage1' }); expect(res).to.nested.include({ 'metadata.labels.algorithm-name': 'myalgo1' }); expect(res).to.nested.include({ 'spec.template.spec.containers[0].image': 'hkube/worker:latest' }); @@ -347,13 +347,17 @@ describe('jobCreator', () => { ); }); it('should apply 0 mounts', () => { - const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', workerImage: 'workerImage2', options, mounts: [] }); + const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', workerImage: 'workerImage2', options, mounts: [], clusterOptions: { datasourcesServiceEnabled: true } }); expect(res.spec.template.spec.volumes).to.have.length(4) }); it('should apply no mounts', () => { - const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', workerImage: 'workerImage2', options }); + const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', workerImage: 'workerImage2', options, clusterOptions: { datasourcesServiceEnabled: true } }); expect(res.spec.template.spec.volumes).to.have.length(4) }); + it('should not apply datasources pvc if disabled', () => { + const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', workerImage: 'workerImage2', options, clusterOptions: { datasourcesServiceEnabled: false } }); + expect(res.spec.template.spec.volumes).to.have.length(3) + }); it('should apply opengl params', () => { const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', workerImage: 'workerImage2', options, algorithmOptions: { opengl: true } }); expect(res.spec.template.spec.containers[1].env).to.deep.include({ name: 'DISPLAY', value: ':0' }) @@ -422,19 +426,19 @@ describe('jobCreator', () => { ], environments: [ { - name: "env1", - value: "val1" + name: "env1", + value: "val1" }, { - name: "env2", - value: "val2" + name: "env2", + value: "val2" } - ] + ] }] }) after(() => { - globalSettings.sidecars=[] + globalSettings.sidecars = [] }); it('should not apply sidecar if not enabled', () => { const res = createJobSpec({ From c3ab5b0a46520e63186d2726e73c71a3458579de Mon Sep 17 00:00:00 2001 From: Yehiyam Livneh Date: Wed, 17 Nov 2021 11:48:29 +0200 Subject: [PATCH 3/4] apply resources to pipeline driver --- core/algorithm-operator/lib/jobs/jobCreator.js | 2 +- core/algorithm-operator/tests/pipeline-drivers.js | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/algorithm-operator/lib/jobs/jobCreator.js b/core/algorithm-operator/lib/jobs/jobCreator.js index 9d5341fe7..f3fc5023f 100644 --- a/core/algorithm-operator/lib/jobs/jobCreator.js +++ b/core/algorithm-operator/lib/jobs/jobCreator.js @@ -38,7 +38,7 @@ const createDriverJobSpec = ({ resourceRequests, image, inputEnv, clusterOptions spec = applyName(spec, CONTAINERS.PIPELINE_DRIVER); spec = applyPipelineDriverImage(spec, image); spec = applyEnvToContainer(spec, CONTAINERS.PIPELINE_DRIVER, inputEnv); - if (settings.applyResources) { + if (settings.applyResourceLimits) { spec = applyPipelineDriverResourceRequests(spec, resourceRequests); } spec = applyJaeger(spec, CONTAINERS.PIPELINE_DRIVER, options); diff --git a/core/algorithm-operator/tests/pipeline-drivers.js b/core/algorithm-operator/tests/pipeline-drivers.js index b72e4f7f7..d7d22b770 100644 --- a/core/algorithm-operator/tests/pipeline-drivers.js +++ b/core/algorithm-operator/tests/pipeline-drivers.js @@ -342,7 +342,10 @@ describe('bootstrap', () => { }); describe('createDriverJobSpec', () => { beforeEach(() => { - globalSettings.applyResources = false; + globalSettings.applyResourceLimits = false; + }); + after(() => { + globalSettings.applyResourceLimits = false; }); it('should throw if no image name', () => { expect(() => createDriverJobSpec({ options })).to.throw('Unable to create job spec. image is required'); @@ -358,7 +361,7 @@ describe('bootstrap', () => { expect(res.metadata.name).to.include(CONTAINERS.PIPELINE_DRIVER); }); it('should apply resources', () => { - globalSettings.applyResources = true; + globalSettings.applyResourceLimits = true; const res = createDriverJobSpec({ ...{ options }, image: 'myImage1', From e7239a50aea0a5acb2de3a144dd82d362797bc32 Mon Sep 17 00:00:00 2001 From: Yehiyam Livneh Date: Thu, 9 Dec 2021 11:36:50 +0200 Subject: [PATCH 4/4] fix test --- core/algorithm-builder/tests/test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/core/algorithm-builder/tests/test.js b/core/algorithm-builder/tests/test.js index 174798b9f..597416f28 100644 --- a/core/algorithm-builder/tests/test.js +++ b/core/algorithm-builder/tests/test.js @@ -49,6 +49,7 @@ describe('Test', function () { expect(build.progress).to.equal(100); }); it('nodejs: build', async function () { + this.timeout(20000) const env = 'nodejs'; const buildId = uuid(); await runBuild({ buildId, env });