From 34034d018a03c0e6c1c43ea6be35ae7c1d8f3796 Mon Sep 17 00:00:00 2001 From: Yehiyam Livneh Date: Thu, 8 Oct 2020 15:42:27 +0300 Subject: [PATCH] add warning for algorithm-queue not created --- core/algorithm-operator/lib/helpers/etcd.js | 8 ++- core/algorithm-operator/lib/operator.js | 13 +++-- core/algorithm-operator/package-lock.json | 56 ++++++++++----------- core/algorithm-operator/package.json | 2 +- core/algorithm-operator/tests/test.js | 19 +++++++ 5 files changed, 64 insertions(+), 34 deletions(-) diff --git a/core/algorithm-operator/lib/helpers/etcd.js b/core/algorithm-operator/lib/helpers/etcd.js index 3efcee9b1..7f447603b 100644 --- a/core/algorithm-operator/lib/helpers/etcd.js +++ b/core/algorithm-operator/lib/helpers/etcd.js @@ -29,8 +29,12 @@ class Etcd extends EventEmitter { return this._etcd.algorithms.store.get({ name }); } - getAlgorithmTemplates() { - return this._etcd.algorithms.store.list(); + async getAlgorithmTemplates() { + const [algorithms, count] = await Promise.all([ + this._etcd.algorithms.store.list(), + this._etcd.algorithms.store.count() + ]); + return { algorithms, count }; } removeAlgorithmData(name) { diff --git a/core/algorithm-operator/lib/operator.js b/core/algorithm-operator/lib/operator.js index 9d708d11a..d60729768 100644 --- a/core/algorithm-operator/lib/operator.js +++ b/core/algorithm-operator/lib/operator.js @@ -44,12 +44,12 @@ class Operator { try { log.debug('Reconcile interval.', { component }); const configMap = await kubernetes.getVersionsConfigMap(); - const algorithms = await etcd.getAlgorithmTemplates(); + const { algorithms, count } = await etcd.getAlgorithmTemplates(); await Promise.all([ this._algorithmBuilds({ ...configMap }, options), this._tenosrboards({ ...configMap, boardTimeOut: this._boardTimeOut }, options), this._algorithmDebug(configMap, algorithms, options), - this._algorithmQueue({ ...configMap, resources: options.resources.algorithmQueue }, algorithms, options), + this._algorithmQueue({ ...configMap, resources: options.resources.algorithmQueue }, algorithms, options, count), ]); } catch (e) { @@ -121,7 +121,8 @@ class Operator { }); } - async _algorithmQueue({ versions, registry, clusterOptions, resources }, algorithms, options) { + async _algorithmQueue({ versions, registry, clusterOptions, resources }, algorithms, options, count) { + this._logAlgorithmCountError(algorithms, count); const deployments = await kubernetes.getDeployments({ labelSelector: `type=${CONTAINERS.ALGORITHM_QUEUE}` }); await algorithmQueueReconciler.reconcile({ deployments, @@ -133,6 +134,12 @@ class Operator { options }); } + + _logAlgorithmCountError(algorithms, count) { + if (algorithms.length < count) { + log.throttle.error(`Only ${algorithms.length} algorithm-queue's can be created, but ${count} algorithms are defined. Please delete unused algorithms`, { component }); + } + } } module.exports = new Operator(); diff --git a/core/algorithm-operator/package-lock.json b/core/algorithm-operator/package-lock.json index 569066306..5defbd92a 100644 --- a/core/algorithm-operator/package-lock.json +++ b/core/algorithm-operator/package-lock.json @@ -204,9 +204,9 @@ "integrity": "sha512-Q9I77WXvJn1mg/ovC+aLWJgZ5iN8McU4pAD5ol2wfcwXlrl7pO6dtLpv+Vdea9j2xCJAO2h6iugYmlm/+Szi9Q==" }, "@hkube/etcd": { - "version": "5.0.37", - "resolved": "https://registry.npmjs.org/@hkube/etcd/-/etcd-5.0.37.tgz", - "integrity": "sha512-ZX0wW+Y7Gfx1jArXkLMr9fUbVfurn5xqLk73ZKHZj05GZrH16KWspZ4FIefUE64lo8zVVPQSP/BEyqeHiHCiiQ==", + "version": "5.0.38", + "resolved": "https://registry.npmjs.org/@hkube/etcd/-/etcd-5.0.38.tgz", + "integrity": "sha512-6trSYyMJCpg6AgQU7PYVwziJx8CoGzAY3tS4TtE+xkz6rWSXdZ1Phyziw6GWDVtcfb7AVnc95aiKpU6tvs8kYQ==", "requires": { "@hkube/uid": "^1.0.4", "ajv": "^6.10.2", @@ -310,9 +310,9 @@ }, "dependencies": { "uuid": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.0.tgz", - "integrity": "sha512-fX6Z5o4m6XsXBdli9g7DtWgAx+osMsRRZFKma1mIUsLCz6vRvv+pz5VNbyu9UEDzpMWulZfvpgb/cmDXVulYFQ==" + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz", + "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg==" } } }, @@ -422,7 +422,8 @@ "@types/color-name": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", - "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==" + "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==", + "dev": true }, "@types/json5": { "version": "0.0.29", @@ -436,9 +437,9 @@ "integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==" }, "@types/node": { - "version": "12.12.62", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.62.tgz", - "integrity": "sha512-qAfo81CsD7yQIM9mVyh6B/U47li5g7cfpVQEDMfQeF8pSZVwzbhwU3crc0qG4DmpsebpJPR49AKOExQyJ05Cpg==" + "version": "12.12.64", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.64.tgz", + "integrity": "sha512-UV1/ZJMC+HcP902wWdpC43cAcGu0IQk/I5bXjP2aSuCjsk3cE74mDvFrLKga7oDC170ugOAYBwfT4DSQW3akDA==" }, "abort-controller": { "version": "3.0.0", @@ -667,9 +668,9 @@ } }, "bignumber.js": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", - "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==" + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.1.tgz", + "integrity": "sha512-IdZR9mh6ahOBv/hYGiXyVuyCetmGJhtYkqLBpTStdhEGjegpPlUawydyaF3pbIOFynJTpllEs+NP+CS9jKFLjA==" }, "bluebird": { "version": "3.7.2", @@ -2207,9 +2208,9 @@ "dev": true }, "google-auth-library": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.1.0.tgz", - "integrity": "sha512-GbalszIADE1YPWhUyfFMrkLhFHnlAgoRcqGVW+MsLDPsuaOB5MRPk7NNafPDv9SherNE4EKzcYuxMJjaxzXMOw==", + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.1.1.tgz", + "integrity": "sha512-0WfExOx3FrLYnY88RICQxvpaNzdwjz44OsHqHkIoAJfjY6Jck6CZRl1ASWadk+wbJ0LhkQ8rNY4zZebKml4Ghg==", "requires": { "arrify": "^2.0.0", "base64-js": "^1.3.0", @@ -2217,7 +2218,7 @@ "fast-text-encoding": "^1.0.0", "gaxios": "^3.0.0", "gcp-metadata": "^4.1.0", - "gtoken": "^5.0.0", + "gtoken": "^5.0.4", "jws": "^4.0.0", "lru-cache": "^6.0.0" }, @@ -2289,12 +2290,12 @@ "dev": true }, "gtoken": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.0.3.tgz", - "integrity": "sha512-Nyd1wZCMRc2dj/mAD0LlfQLcAO06uKdpKJXvK85SGrF5+5+Bpfil9u/2aw35ltvEHjvl0h5FMKN5knEU+9JrOg==", + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.0.4.tgz", + "integrity": "sha512-U9wnSp4GZ7ov6zRdPuRHG4TuqEWqRRgT1gfXGNArhzBUn9byrPeH8uTmBWU/ZiWJJvTEmkjhDIC3mqHWdVi3xQ==", "requires": { "gaxios": "^3.0.0", - "google-p12-pem": "^3.0.0", + "google-p12-pem": "^3.0.3", "jws": "^4.0.0", "mime": "^2.2.0" }, @@ -4130,9 +4131,9 @@ }, "dependencies": { "@types/node": { - "version": "13.13.21", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.21.tgz", - "integrity": "sha512-tlFWakSzBITITJSxHV4hg4KvrhR/7h3xbJdSFbYJBVzKubrASbnnIFuSgolUh7qKGo/ZeJPKUfbZ0WS6Jp14DQ==" + "version": "13.13.23", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.23.tgz", + "integrity": "sha512-L31WmMJYKb15PDqFWutn8HNwrNK6CE6bkWgSB0dO1XpNoHrszVKV1Clcnfgd6c/oG54TVF8XQEvY2gQrW8K6Mw==" } } }, @@ -5122,11 +5123,10 @@ "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==" }, "ansi-styles": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", - "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "requires": { - "@types/color-name": "^1.1.1", "color-convert": "^2.0.1" } }, diff --git a/core/algorithm-operator/package.json b/core/algorithm-operator/package.json index 2094a81c5..a8b01c715 100644 --- a/core/algorithm-operator/package.json +++ b/core/algorithm-operator/package.json @@ -16,7 +16,7 @@ "dependencies": { "@hkube/config": "^2.0.7", "@hkube/consts": "^1.0.21", - "@hkube/etcd": "^5.0.37", + "@hkube/etcd": "^5.0.38", "@hkube/healthchecks": "^1.0.0", "@hkube/kubernetes-client": "^1.0.32", "@hkube/logger": "^1.0.25", diff --git a/core/algorithm-operator/tests/test.js b/core/algorithm-operator/tests/test.js index 6b5155b19..78219ae80 100644 --- a/core/algorithm-operator/tests/test.js +++ b/core/algorithm-operator/tests/test.js @@ -21,6 +21,9 @@ describe('bootstrap', () => { mockery.disable(); decache('../bootstrap'); }); + beforeEach(async () => { + await etcd._etcd._client.delete('/algorithms/store', { isPrefix: true }) + }); it('should init without error', async () => { }); @@ -29,4 +32,20 @@ describe('bootstrap', () => { const template = await etcd.getAlgorithmTemplate({ name: 'algo2' }); expect(template).to.eql(templateStoreStub[1]); }); + + it('should list template store', async () => { + await Promise.all(templateStoreStub.map(a => etcd._etcd.algorithms.store.set(a))); + const { algorithms, count } = await etcd.getAlgorithmTemplates(); + expect(algorithms).to.deep.include(templateStoreStub[1]); + expect(algorithms.length).to.eql(count); + }); + it('should list template store above limit', async () => { + + await Promise.all([...Array(110).keys()].map(i => etcd._etcd.algorithms.store.set({ + name: `key-${i}`, + }))); + const { algorithms, count } = await etcd.getAlgorithmTemplates(); + expect(algorithms).to.deep.include({ name: 'key-99' }); + expect(algorithms.length).to.not.eql(count); + }); });