algorithm-deps-install (#1081)

* add dependencyInstallCmd option to algorithm add python support * add nodejs support * fix install deps script permission * add test
kube-HPC · Jan 5, 2021 · faf983d · faf983d
1 parent 3873023
commit faf983d
Show file tree

Hide file tree

Showing 21 changed files with 151 additions and 27 deletions.
diff --git a/core/algorithm-builder/.dockerignore b/core/algorithm-builder/.dockerignore
@@ -1,2 +1,3 @@
 builds
-uploads
+uploads
+node_modules
diff --git a/core/algorithm-builder/bootstrap.js b/core/algorithm-builder/bootstrap.js
@@ -41,8 +41,8 @@ class Bootstrap {
             const storageManager = require('@hkube/storage-manager');
             const fse = require('fs-extra');
             const { buildId } = mockBuild;
-            await stateManger.insertBuild(mockBuild);
-            await storageManager.hkubeBuilds.putStream({ buildId, data: fse.createReadStream(tar) });
+            const {path: filePath} = await storageManager.hkubeBuilds.putStream({ buildId, data: fse.createReadStream(tar) });
+            await stateManger.insertBuild({...mockBuild, filePath});
             config.buildId = buildId;
         }
     }

diff --git a/core/algorithm-builder/dockerfile/Dockerfile b/core/algorithm-builder/dockerfile/Dockerfile
@@ -1,3 +1,10 @@
+ARG BASE_PRIVATE_REGISTRY=""
+FROM ${BASE_PRIVATE_REGISTRY}node:14.5.0 as install
+ADD ./package-lock.json ./package.json /hkube/algorithm-builder/
+WORKDIR /hkube/algorithm-builder
+RUN npm ci --production
+RUN echo stam
+
 ARG BASE_PRIVATE_REGISTRY=""
 FROM ${BASE_PRIVATE_REGISTRY}hkube/base-node:v1.2.0
 LABEL maintainer="hkube.dev@gmail.com"
@@ -6,4 +13,5 @@ RUN apt update && apt install -y git gettext-base && rm -rf /var/lib/apt/lists/*
 RUN mkdir -p /hkube/algorithm-builder
 WORKDIR /hkube/algorithm-builder
 COPY . /hkube/algorithm-builder
+COPY --from=install /hkube/algorithm-builder/node_modules /hkube/algorithm-builder/node_modules
 CMD ["npm", "start"]
diff --git a/core/algorithm-builder/dockerfile/build.sh b/core/algorithm-builder/dockerfile/build.sh
@@ -1,9 +1,11 @@
 #!/usr/bin/env bash
 set -eo pipefail
 SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
-${SCRIPTPATH}/get-deps-python.sh
-${SCRIPTPATH}/get-deps-nodejs.sh
-${SCRIPTPATH}/get-deps-java.sh
+if [ -v $SKIP_DOWNLOAD_PACKAGES ]; then
+  ${SCRIPTPATH}/get-deps-python.sh
+  ${SCRIPTPATH}/get-deps-nodejs.sh
+  ${SCRIPTPATH}/get-deps-java.sh
+fi
 REPO_NAME=$1
 if [ -v PRIVATE_REGISTRY ]
 then

diff --git a/core/algorithm-builder/environments/nodejs/dockerfile/DockerfileTemplate b/core/algorithm-builder/environments/nodejs/dockerfile/DockerfileTemplate
@@ -8,6 +8,7 @@ WORKDIR /hkube/algorithm-runner/algorithm_unique_folder
 ENV packagesRegistry=${packagesRegistry}
 ENV packagesToken=${packagesToken}
 ENV packagesAuth=${packagesAuth}
+ENV dependency_install_cmd=${dependency_install_cmd}
 RUN ../dockerfile/requirements.sh
 
 # Second build

diff --git a/core/algorithm-builder/environments/nodejs/dockerfile/requirements.sh b/core/algorithm-builder/environments/nodejs/dockerfile/requirements.sh
@@ -2,11 +2,11 @@
 
 set -e
 
-PACKAGES_REGISTRY=${packagesRegistry}
+export PACKAGES_REGISTRY=${packagesRegistry}
 PACKAGES_REGISTRY_HOST=${PACKAGES_REGISTRY#http://}
-PACKAGES_REGISTRY_HOST=${PACKAGES_REGISTRY_HOST#https://}
-PACKAGES_TOKEN=${packagesToken}
-PACKAGES_AUTH=${packagesAuth}
+export PACKAGES_REGISTRY_HOST=${PACKAGES_REGISTRY_HOST#https://}
+export PACKAGES_TOKEN=${packagesToken}
+export PACKAGES_AUTH=${packagesAuth}
 
 if [ ! -z ${PACKAGES_REGISTRY} ]; then
     echo "found npm registry ${PACKAGES_REGISTRY}"
@@ -16,6 +16,14 @@ if [ ! -z ${PACKAGES_REGISTRY} ]; then
         echo "//${PACKAGES_REGISTRY_HOST}:_auth=${PACKAGES_AUTH}" > ${HOME}/.npmrc
         echo "//${PACKAGES_REGISTRY_HOST}:always-auth=true" >> ${HOME}/.npmrc
     fi
+fi
+if [ ! -z ${dependency_install_cmd} ]; then
+    echo "found dependency install script"
+    SCRIPT_CWD="${PWD}"
+    echo "running ${dependency_install_cmd} in folder ${SCRIPT_CWD}"
+    sh -c "cd ${SCRIPT_CWD} && sh ${dependency_install_cmd}"
+    echo "${dependency_install_cmd} execution done with code $?"
+elif [ ! -z ${PACKAGES_REGISTRY} ]; then
     npm install --registry=${PACKAGES_REGISTRY}
     rm -f .npmrc
 else

diff --git a/core/algorithm-builder/environments/python/dockerfile/DockerfileTemplate b/core/algorithm-builder/environments/python/dockerfile/DockerfileTemplate
@@ -7,7 +7,7 @@ COPY ./dockerfile/* /hkube/algorithm-runner/dockerfile/
 COPY ./packages/* /hkube/packages/
 COPY ./nodemon ./docker-entrypoint.sh /hkube/
 WORKDIR /hkube/algorithm-runner
-RUN ./dockerfile/requirements.sh ${packagesRegistry} ${packagesToken}
+RUN export dependency_install_cmd=${dependency_install_cmd} && ./dockerfile/requirements.sh ${packagesRegistry} ${packagesToken} 
 
 ENV PYTHONPATH=$PYTHONPATH:/hkube/algorithm-runner/algorithm_unique_folder
 ENTRYPOINT ["/hkube/docker-entrypoint.sh"]

diff --git a/core/algorithm-builder/environments/python/dockerfile/requirements.sh b/core/algorithm-builder/environments/python/dockerfile/requirements.sh
@@ -2,21 +2,29 @@
 
 set -e
 
-PACKAGES_REGISTRY=$1
-PACKAGES_TOKEN=$2
+export PACKAGES_REGISTRY=$1
+export PACKAGES_TOKEN=$2
 REQUIRMENTS=./algorithm_unique_folder/requirements.txt
 TRUSTED_HOST=pypi.python.org
+if [ ! -z ${PACKAGES_REGISTRY} ]; then
+  export PACKAGES_REGISTRY_HOST=$(echo $PACKAGES_REGISTRY | sed -e "s/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/")
+fi
 # install hkube + dependencies
 echo install hkube + dependencies
 if [ ! -z ${PACKAGES_REGISTRY} ]; then
-     PACKAGES_REGISTRY_HOST=$(echo $PACKAGES_REGISTRY | sed -e "s/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/")
      echo "found pip registry ${PACKAGES_REGISTRY}. Setting trusted host to ${PACKAGES_REGISTRY_HOST}"
      pip install --trusted-host "${PACKAGES_REGISTRY_HOST}" --index-url "${PACKAGES_REGISTRY}" --find-links /hkube/packages/ -r /hkube/algorithm-runner/requirements.txt
 else
   pip install --find-links /hkube/packages/ -r /hkube/algorithm-runner/requirements.txt
 fi
 
-if [ -f ${REQUIRMENTS} ]; then
+if [ ! -z ${dependency_install_cmd} ]; then
+     echo "found dependency install script"
+     SCRIPT_CWD="${PWD}/algorithm_unique_folder"
+     echo "running ${dependency_install_cmd} in folder ${SCRIPT_CWD}"
+     sh -c "cd ${SCRIPT_CWD} && sh ${dependency_install_cmd}"
+     echo "${dependency_install_cmd} execution done with code $?"
+elif [ -f ${REQUIRMENTS} ]; then
      echo "found requirements.txt"
      if [ ! -z ${PACKAGES_REGISTRY} ]; then
           PACKAGES_REGISTRY_HOST=$(echo $PACKAGES_REGISTRY | sed -e "s/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/")

diff --git a/core/algorithm-builder/lib/builds/build-utils.sh b/core/algorithm-builder/lib/builds/build-utils.sh
@@ -96,27 +96,28 @@ dockerBuildKaniko() {
   export insecure_pull=${INSECURE_PULL}
   export skip_tls_verify=${SKIP_TLS_VERIFY}
   export skip_tls_verify_pull=${SKIP_TLS_VERIFY_PULL}
+  export dependency_install_cmd=${DEPENDENCY_INSTALL_CMD}
 
   echo "Building image ${image}"
   echo copy context from ${buildPath} to ${workspace}
   cp -r ${buildPath}/* ${workspace}
 
   envsubst < ${workspace}/dockerfile/DockerfileTemplate > ${workspace}/dockerfile/Dockerfile
-  sed -i '/^ARG /d' ${workspace}/dockerfile/Dockerfile
 
   options=""
   if [[ $insecure == true ]]; then options="${options} --insecure"; fi
   if [[ $insecure_pull == true ]]; then options="${options} --insecure-pull"; fi
   if [[ $skip_tls_verify == true ]]; then options="${options} --skip-tls-verify"; fi
   if [[ $skip_tls_verify_pull == true ]]; then options="${options} --skip-tls-verify-pull"; fi
-
+  if [[ $NO_PUSH == true ]]; then options="${options} --no-push"; fi
   echo "/kaniko/executor \
     --dockerfile ./dockerfile/Dockerfile \
     ${options} --context dir:///workspace/ \
     --build-arg packagesRegistry=${packagesRegistry} \
     --build-arg packagesRegistryUser=${packagesRegistryUser} \
     --build-arg packagesToken=${packagesToken} \
     --build-arg baseImage=${baseImage} \
+    --build-arg dependency_install_cmd=${dependency_install_cmd} \
     --destination $image" > ${commands}/run
 
   chmod +x ${commands}/run

diff --git a/core/algorithm-builder/lib/builds/docker-builder.js b/core/algorithm-builder/lib/builds/docker-builder.js
@@ -381,7 +381,7 @@ const _overrideVersion = async (env, buildPath, version) => {
     }
 }
 
-const buildAlgorithmImage = async ({ buildMode, env, docker, algorithmName, imageTag, buildPath, rmi, baseImage, tmpFolder, packagesRepo, buildId }) => {
+const buildAlgorithmImage = async ({ buildMode, env, docker, algorithmName, imageTag, buildPath, rmi, baseImage, tmpFolder, packagesRepo, buildId, dependencyInstallCmd }) => {
     const pushRegistry = _createURL(docker.push);
     const algorithmImage = `${path.join(pushRegistry, algorithmName)}:v${imageTag}`;
     const packages = packagesRepo[env];
@@ -393,6 +393,7 @@ const buildAlgorithmImage = async ({ buildMode, env, docker, algorithmName, imag
     _envsHelper(envs, 'REMOVE_IMAGE', rmi);
     _envsHelper(envs, 'BUILD_PATH', buildPath);
     _envsHelper(envs, 'BASE_IMAGE', baseImageName);
+    _envsHelper(envs, 'DEPENDENCY_INSTALL_CMD', dependencyInstallCmd);
     _envsHelper(envs, 'BUILD_ID', buildId);
     _envsHelper(envs, 'WRAPPER_VERSION', wrapperVersion);
 
@@ -475,7 +476,7 @@ const runBuild = async (options) => {
         await _setBuildStatus({ buildId, progress, status: STATES.ACTIVE });
 
         const overwrite = true;
-        const { env, imageTag, fileExt, filePath, baseImage, type, gitRepository } = build;
+        const { env, imageTag, fileExt, filePath, baseImage, type, gitRepository, dependencyInstallCmd } = build;
         const { docker, buildDirs, tmpFolder, packagesRepo } = options;
         buildMode = options.buildMode;
         algorithmName = build.algorithmName;
@@ -495,7 +496,7 @@ const runBuild = async (options) => {
         }
         await _prepareBuild({ buildPath, env, dest, overwrite });
         await _setBuildStatus({ buildId, progress, status: STATES.ACTIVE });
-        result = await buildAlgorithmImage({ buildMode, env, docker, algorithmName, imageTag, buildPath, rmi: 'True', baseImage, tmpFolder, packagesRepo, buildId });
+        result = await buildAlgorithmImage({ buildMode, env, docker, algorithmName, imageTag, buildPath, rmi: 'True', baseImage, tmpFolder, packagesRepo, buildId, dependencyInstallCmd });
     }
     catch (e) {
         error = e.message;

diff --git a/core/algorithm-builder/tests/mocks/java/build.json b/core/algorithm-builder/tests/mocks/java/build.json
@@ -3,6 +3,7 @@
     "algorithmName": "sort-alg",
     "env": "java",
     "version": "1.0.0",
+    "imageTag": "1.0.0",
     "fileExt": "gz",
     "status": "pending",
     "baseImage": "",
@@ -11,5 +12,6 @@
     "result": null,
     "progress": 0,
     "startTime": 1556799896273,
-    "endTime": null
+    "endTime": null,
+    "dependencyInstallCmd": "./install.sh"
 }
diff --git a/core/algorithm-builder/tests/mocks/nodejs/alg.tar.gz b/core/algorithm-builder/tests/mocks/nodejs/alg.tar.gz
diff --git a/core/algorithm-builder/tests/mocks/nodejs/build.json b/core/algorithm-builder/tests/mocks/nodejs/build.json
@@ -3,12 +3,14 @@
     "algorithmName": "sort-alg",
     "env": "nodejs",
     "version": "5.0.0",
+    "imageTag": "5.0.0",
     "fileExt": "gz",
     "status": "pending",
     "error": null,
     "stack": null,
     "result": null,
     "progress": 0,
     "startTime": 1556799896273,
-    "endTime": null
+    "endTime": null,
+    "dependencyInstallCmd": "./install.sh"
 }
diff --git a/core/algorithm-builder/tests/mocks/python/alg.tar.gz b/core/algorithm-builder/tests/mocks/python/alg.tar.gz
diff --git a/core/algorithm-builder/tests/mocks/python/build.json b/core/algorithm-builder/tests/mocks/python/build.json
@@ -3,6 +3,7 @@
     "algorithmName": "sort-alg",
     "env": "python",
     "version": "1.0.0",
+    "imageTag": "1.0.0",
     "fileExt": "gz",
     "status": "pending",
     "baseImage": "",
@@ -11,5 +12,6 @@
     "result": null,
     "progress": 0,
     "startTime": 1556799896273,
-    "endTime": null
+    "endTime": null,
+    "dependencyInstallCmd": "./install.sh"
 }
diff --git a/core/api-server/api/rest-api/swagger/definitions/algorithm.yaml b/core/api-server/api/rest-api/swagger/definitions/algorithm.yaml
@@ -97,6 +97,12 @@ properties:
       - url
   entryPoint:
     type: string
+  dependencyInstallCmd:
+    type: string
+    description: |
+      Command to run to install algorithm dependencies. CWD is the algorithm root folder
+      Defaults to language specific defaults.
+      e.g. For python: pip install -r requirements.txt
   baseImage:
     type: string
     description: Custom docker image to be used as base to the newly built algorithm image

diff --git a/core/api-server/lib/consts/builds.js b/core/api-server/lib/consts/builds.js
@@ -1,4 +1,4 @@
-const BUILD_TRIGGERS = ['checksum', 'env', 'commit', 'baseImage'];
+const BUILD_TRIGGERS = ['checksum', 'env', 'commit', 'baseImage', 'dependencyInstallCmd'];
 
 const BUILD_GUIDE = 'use the Hkube dashboard or the Hkube API to follow the build progress';
 

diff --git a/core/api-server/lib/service/build.js b/core/api-server/lib/service/build.js
@@ -13,6 +13,7 @@ class Build {
         this.gitRepository = options.gitRepository;
         this.type = options.type;
         this.baseImage = options.baseImage;
+        this.dependencyInstallCmd = options.dependencyInstallCmd;
     }
 
     _createBuildID(algorithmName) {

diff --git a/core/api-server/lib/service/builds.js b/core/api-server/lib/service/builds.js
@@ -136,7 +136,8 @@ class Builds {
                     algorithmName: newAlgorithm.name,
                     gitRepository: newAlgorithm.gitRepository,
                     type: newAlgorithm.type,
-                    baseImage: newAlgorithm.baseImage
+                    baseImage: newAlgorithm.baseImage,
+                    dependencyInstallCmd: newAlgorithm.dependencyInstallCmd
                 });
                 buildId = build.buildId;
                 if (fileInfo && !fileInfo.path) {
@@ -221,10 +222,10 @@ class Builds {
     }
 
     _formatDiff(algorithm) {
-        const { fileInfo, env, baseImage, gitRepository } = algorithm;
+        const { fileInfo, env, baseImage, gitRepository, dependencyInstallCmd } = algorithm;
         const checksum = fileInfo?.checksum;
         const commit = gitRepository?.commit?.id;
-        return { checksum, env, commit, baseImage };
+        return { checksum, env, commit, baseImage, dependencyInstallCmd };
     }
 }
 

diff --git a/core/api-server/tests/algorithms-store.js b/core/api-server/tests/algorithms-store.js
@@ -1109,6 +1109,32 @@ describe('Store/Algorithms', () => {
                 expect(response2.body).to.have.property('buildId');
                 expect(response2.body.messages[0]).to.contains('a build was triggered due to change in checksum');
             });
+            it('should succeed to apply algorithm with buildId due to change in dependencyInstallCmd', async () => {
+                const body1 = {
+                    name: `my-alg-${uuid()}`,
+                    env: 'python'
+                }
+                const body2 = {
+                    ...body1,
+                    dependencyInstallCmd: './foo'
+                }
+                const formData1 = {
+                    payload: JSON.stringify(body1),
+                    file: fse.createReadStream('tests/mocks/algorithm.tar.gz')
+                };
+                const formData2 = {
+                    payload: JSON.stringify(body2),
+                    file: fse.createReadStream('tests/mocks/algorithm.tar.gz')
+                };
+                const uri = applyPath;
+                const options1 = { uri, formData: formData1 };
+                const options2 = { uri, formData: formData2 };
+                const response1 = await request(options1);
+                const response2 = await request(options2)
+                expect(response1.body).to.have.property('buildId');
+                expect(response2.body).to.have.property('buildId');
+                expect(response2.body.messages[0]).to.contains('a build was triggered due to change in dependencyInstallCmd');
+            });
             it('should succeed to apply with build due to change in baseImage', async () => {
                 const body1 = {
                     name: `my-alg-${uuid()}`,

diff --git a/core/api-server/tests/builds.js b/core/api-server/tests/builds.js
@@ -116,6 +116,60 @@ describe('Builds', () => {
             expect(response.response.statusCode).to.equal(HttpStatus.OK);
             expect(response.body.baseImage).to.equal('userOwnBaseImage');
         });
+        it('should succeed to get dependencyInstallCmd', async () => {
+            const payload = {
+                name: `my-alg-${uuid()}`,
+                mem: "50Mi",
+                cpu: 1,
+                version: '1.9.0',
+                env: 'nodejs',
+                baseImage: 'userOwnBaseImage',
+                dependencyInstallCmd: 'install.sh'
+            }
+            const formData = {
+                payload: JSON.stringify(payload),
+                file: fse.createReadStream('tests/mocks/algorithm.tar.gz')
+            };
+            const opt = {
+                uri: restUrl + '/store/algorithms/apply',
+                formData
+            };
+            const res = await request(opt);
+
+            const options = {
+                uri: restPath + `/${res.body.buildId}`,
+                method: 'GET'
+            };
+            const response = await request(options);
+            expect(response.response.statusCode).to.equal(HttpStatus.OK);
+            expect(response.body.dependencyInstallCmd).to.equal('install.sh');
+        });
+        it('should work without dependencyInstallCmd', async () => {
+            const payload = {
+                name: `my-alg-${uuid()}`,
+                mem: "50Mi",
+                cpu: 1,
+                version: '1.9.0',
+                env: 'nodejs',
+            }
+            const formData = {
+                payload: JSON.stringify(payload),
+                file: fse.createReadStream('tests/mocks/algorithm.tar.gz')
+            };
+            const opt = {
+                uri: restUrl + '/store/algorithms/apply',
+                formData
+            };
+            const res = await request(opt);
+
+            const options = {
+                uri: restPath + `/${res.body.buildId}`,
+                method: 'GET'
+            };
+            const response = await request(options);
+            expect(response.response.statusCode).to.equal(HttpStatus.OK);
+            expect(response.body.dependencyInstallCmd).to.not.exist;
+        });
     });
     describe('stop', () => {
         let restPath = null;