updates to v2.1 (#1462)

* update python wrapper * disable datasources mount if service is disabled * apply resources to pipeline driver * fix test
kube-HPC · Dec 13, 2021 · 13a1eea · 13a1eea
1 parent 6aed3bd
commit 13a1eea
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 18 deletions.
diff --git a/core/algorithm-builder/environments/python/wrapper/requirements.txt b/core/algorithm-builder/environments/python/wrapper/requirements.txt
@@ -1 +1 @@
-hkube-python-wrapper==2.2.1
+hkube-python-wrapper==2.2.2
diff --git a/core/algorithm-builder/tests/test.js b/core/algorithm-builder/tests/test.js
@@ -49,6 +49,7 @@ describe('Test', function () {
             expect(build.progress).to.equal(100);
         });
         it('nodejs: build', async function () {
+            this.timeout(20000)
             const env = 'nodejs';
             const buildId = uuid();
             await runBuild({ buildId, env });

diff --git a/core/algorithm-operator/lib/jobs/jobCreator.js b/core/algorithm-operator/lib/jobs/jobCreator.js
@@ -38,7 +38,7 @@ const createDriverJobSpec = ({ resourceRequests, image, inputEnv, clusterOptions
     spec = applyName(spec, CONTAINERS.PIPELINE_DRIVER);
     spec = applyPipelineDriverImage(spec, image);
     spec = applyEnvToContainer(spec, CONTAINERS.PIPELINE_DRIVER, inputEnv);
-    if (settings.applyResources) {
+    if (settings.applyResourceLimits) {
         spec = applyPipelineDriverResourceRequests(spec, resourceRequests);
     }
     spec = applyJaeger(spec, CONTAINERS.PIPELINE_DRIVER, options);

diff --git a/core/algorithm-operator/tests/pipeline-drivers.js b/core/algorithm-operator/tests/pipeline-drivers.js
@@ -342,7 +342,10 @@ describe('bootstrap', () => {
         });
         describe('createDriverJobSpec', () => {
             beforeEach(() => {
-                globalSettings.applyResources = false;
+                globalSettings.applyResourceLimits = false;
+            });
+            after(() => {
+                globalSettings.applyResourceLimits = false;
             });
             it('should throw if no image name', () => {
                 expect(() => createDriverJobSpec({ options })).to.throw('Unable to create job spec. image is required');
@@ -358,7 +361,7 @@ describe('bootstrap', () => {
                 expect(res.metadata.name).to.include(CONTAINERS.PIPELINE_DRIVER);
             });
             it('should apply resources', () => {
-                globalSettings.applyResources = true;
+                globalSettings.applyResourceLimits = true;
                 const res = createDriverJobSpec({
                     ...{ options },
                     image: 'myImage1',

diff --git a/core/task-executor/lib/jobs/jobCreator.js b/core/task-executor/lib/jobs/jobCreator.js
@@ -187,11 +187,11 @@ const applyDevMode = (inputSpec, { algorithmOptions = {}, algorithmName, cluster
     return spec;
 };
 
-const applyDataSourcesVolumes = (inputSpec) => {
+const applyDataSourcesVolumes = (inputSpec, clusterOptions) => {
     let spec = clonedeep(inputSpec);
-    // if (!clusterOptions.dataSourcesEnabled) {
-    //     return spec;
-    // }
+    if (!clusterOptions?.datasourcesServiceEnabled) {
+        return spec;
+    }
     spec = applyVolumeMounts(spec, CONTAINERS.ALGORITHM, {
         name: 'datasources-storage',
         mountPath: '/hkube/datasources-storage'
@@ -362,7 +362,7 @@ const createJobSpec = ({ kind, algorithmName, resourceRequests, workerImage, alg
     spec = applyJaeger(spec, CONTAINERS.WORKER, options);
     spec = applyJaeger(spec, CONTAINERS.ALGORITHM, options);
     spec = applyDevMode(spec, { options, algorithmOptions, clusterOptions, algorithmName });
-    spec = applyDataSourcesVolumes(spec);
+    spec = applyDataSourcesVolumes(spec, clusterOptions);
     spec = applyMounts(spec, mounts);
     spec = applyImagePullSecret(spec, clusterOptions?.imagePullSecretName);
 

diff --git a/core/task-executor/tests/jobCreatorTests.js b/core/task-executor/tests/jobCreatorTests.js
@@ -163,7 +163,7 @@ describe('jobCreator', () => {
             expect(() => createJobSpec({ algorithmImage: 'myImage1', options })).to.throw('Unable to create job spec. algorithmName is required');
         });
         it('should apply all required properties', () => {
-            const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', options });
+            const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', options, clusterOptions: { datasourcesServiceEnabled: true } });
             expect(res).to.nested.include({ 'spec.template.spec.containers[1].image': 'myImage1' });
             expect(res).to.nested.include({ 'metadata.labels.algorithm-name': 'myalgo1' });
             expect(res).to.nested.include({ 'spec.template.spec.containers[0].image': 'hkube/worker:latest' });
@@ -347,13 +347,17 @@ describe('jobCreator', () => {
             );
         });
         it('should apply 0 mounts', () => {
-            const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', workerImage: 'workerImage2', options, mounts: [] });
+            const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', workerImage: 'workerImage2', options, mounts: [], clusterOptions: { datasourcesServiceEnabled: true } });
             expect(res.spec.template.spec.volumes).to.have.length(4)
         });
         it('should apply no mounts', () => {
-            const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', workerImage: 'workerImage2', options });
+            const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', workerImage: 'workerImage2', options, clusterOptions: { datasourcesServiceEnabled: true } });
             expect(res.spec.template.spec.volumes).to.have.length(4)
         });
+        it('should not apply datasources pvc if disabled', () => {
+            const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', workerImage: 'workerImage2', options, clusterOptions: { datasourcesServiceEnabled: false } });
+            expect(res.spec.template.spec.volumes).to.have.length(3)
+        });
         it('should apply opengl params', () => {
             const res = createJobSpec({ algorithmImage: 'myImage1', algorithmName: 'myalgo1', workerImage: 'workerImage2', options, algorithmOptions: { opengl: true } });
             expect(res.spec.template.spec.containers[1].env).to.deep.include({ name: 'DISPLAY', value: ':0' })
@@ -422,19 +426,19 @@ describe('jobCreator', () => {
                 ],
                 environments: [
                     {
-                      name: "env1",
-                      value: "val1"
+                        name: "env1",
+                        value: "val1"
                     },
                     {
-                      name: "env2",
-                      value: "val2"
+                        name: "env2",
+                        value: "val2"
                     }
-                  ]
+                ]
 
             }]
         })
         after(() => {
-            globalSettings.sidecars=[]
+            globalSettings.sidecars = []
         });
         it('should not apply sidecar if not enabled', () => {
             const res = createJobSpec({
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		hkube-python-wrapper==2.2.1
		hkube-python-wrapper==2.2.2