From 6f0220f443eaabe43ead90899d295b7c63af9a33 Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Mon, 15 Apr 2019 14:54:10 +0300 Subject: [PATCH] feat: App multi-tenancy support in single ServerCore instance --- packages/cubejs-api-gateway/index.js | 28 ++++- .../cubejs-playground/config-overrides.js | 43 ++++++- packages/cubejs-playground/package.json | 5 +- packages/cubejs-playground/yarn.lock | 30 +++-- packages/cubejs-server-core/core/index.js | 66 ++++++++-- packages/cubejs-server-core/yarn.lock | 73 ++++------- packages/cubejs-serverless/Handlers.js | 90 ++++++++++++++ packages/cubejs-serverless/index.js | 92 +------------- packages/cubejs-serverless/yarn.lock | 117 ++++++++++-------- 9 files changed, 327 insertions(+), 217 deletions(-) create mode 100644 packages/cubejs-serverless/Handlers.js diff --git a/packages/cubejs-api-gateway/index.js b/packages/cubejs-api-gateway/index.js index 996934830766c..7271aa71d3391 100644 --- a/packages/cubejs-api-gateway/index.js +++ b/packages/cubejs-api-gateway/index.js @@ -221,8 +221,8 @@ class ApiGateway { }); const normalizedQuery = normalizeQuery(query); const [compilerSqlResult, metaConfigResult] = await Promise.all([ - this.compilerApi.getSql(coerceForSqlQuery(normalizedQuery, req)), - this.compilerApi.metaConfig() + this.getCompilerApi(req).getSql(coerceForSqlQuery(normalizedQuery, req)), + this.getCompilerApi(req).metaConfig() ]); const sqlQuery = compilerSqlResult; const metaConfig = metaConfigResult; @@ -231,7 +231,7 @@ class ApiGateway { const toExecute = { ...sqlQuery, query: sqlQuery.sql[0], values: sqlQuery.sql[1], continueWait: true }; - const response = await this.adapterApi.executeQuery(toExecute); + const response = await this.getAdapterApi(req).executeQuery(toExecute); this.log(req, { type: 'Load Request Success', query: req.query.query, @@ -255,7 +255,7 @@ class ApiGateway { try { const query = JSON.parse(req.query.query); const normalizedQuery = normalizeQuery(query); - const sqlQuery = await this.compilerApi.getSql(coerceForSqlQuery(normalizedQuery, req)); + const sqlQuery = await this.getCompilerApi(req).getSql(coerceForSqlQuery(normalizedQuery, req)); res.json({ sql: sqlQuery }); @@ -266,7 +266,7 @@ class ApiGateway { app.get(`${this.basePath}/v1/meta`, this.checkAuthMiddleware, (async (req, res) => { try { - const metaConfig = await this.compilerApi.metaConfig(); + const metaConfig = await this.getCompilerApi(req).metaConfig(); const cubes = metaConfig.map(c => c.config); res.json({ cubes }); } catch (e) { @@ -275,6 +275,24 @@ class ApiGateway { })); } + getCompilerApi(req) { + if (typeof this.compilerApi === 'function') { + return this.compilerApi(this.contextByReq(req)); + } + return this.compilerApi; + } + + getAdapterApi(req) { + if (typeof this.adapterApi === 'function') { + return this.adapterApi(this.contextByReq(req)); + } + return this.adapterApi; + } + + contextByReq(req) { + return { authInfo: req.authInfo }; + } + handleError(e, req, res) { if (e instanceof UserError) { this.log(req, { diff --git a/packages/cubejs-playground/config-overrides.js b/packages/cubejs-playground/config-overrides.js index 23ee8ce4fe8a8..71f4b560c140f 100644 --- a/packages/cubejs-playground/config-overrides.js +++ b/packages/cubejs-playground/config-overrides.js @@ -1,4 +1,45 @@ +const TerserPlugin = require('terser-webpack-plugin'); +const webpack = require('webpack'); + module.exports = function override(config, env) { - config.optimization = { minimize: false }; + config.optimization = { + minimizer: [ + new TerserPlugin({ + cache: true, + parallel: true, + chunkFilter: (chunk) => chunk.name.indexOf('babel') === -1 // && chunk.name.indexOf('vendors') === -1 + }) + ], + splitChunks: { + chunks: 'all', + minSize: 30000, + maxSize: 0, + minChunks: 1, + maxAsyncRequests: 5, + maxInitialRequests: 3, + automaticNameDelimiter: '~', + name: true, + cacheGroups: { + babel: { + test: /babel/, + priority: -5 + }, + vendors: { + test: /[\\/]node_modules[\\/]/, + priority: -10 + }, + default: { + priority: -20 + } + } + } + }; + config.stats = 'verbose'; + config.plugins = config.plugins.concat([ + new webpack.ProgressPlugin() + ]); + if (env === 'production') { + config.devtool = false; + } return config; }; diff --git a/packages/cubejs-playground/package.json b/packages/cubejs-playground/package.json index 844a9a8d82467..723a27f83d5ed 100644 --- a/packages/cubejs-playground/package.json +++ b/packages/cubejs-playground/package.json @@ -34,7 +34,7 @@ }, "scripts": { "start": "react-app-rewired start", - "build": "react-app-rewired build && cp -R build/ ../cubejs-server-core/playground/", + "build": "react-app-rewired build && rm -Rf ../cubejs-server-core/playground/** && cp -R build/ ../cubejs-server-core/playground/", "test": "react-app-rewired test", "eject": "react-scripts eject" }, @@ -59,6 +59,7 @@ "eslint-plugin-import": "^2.16.0", "eslint-plugin-jsx-a11y": "^6.2.1", "eslint-plugin-react": "^7.12.4", - "react-app-rewired": "^2.1.0" + "react-app-rewired": "^2.1.0", + "terser-webpack-plugin": "^1.2.3" } } diff --git a/packages/cubejs-playground/yarn.lock b/packages/cubejs-playground/yarn.lock index 569aabff7d920..17e9c9d932e27 100755 --- a/packages/cubejs-playground/yarn.lock +++ b/packages/cubejs-playground/yarn.lock @@ -1063,10 +1063,10 @@ resolved "https://registry.yarnpkg.com/@csstools/convert-colors/-/convert-colors-1.4.0.tgz#ad495dc41b12e75d588c6db8b9834f08fa131eb7" integrity sha512-5a6wqoJV/xEdbRNKVo6I4hO3VjyDq//8q2f9I6PBAvMesJHFauXDorcNCsr9RzvsZnaWi5NYCcfyqP1QeFHFbw== -"@cubejs-client/core@^0.4.5": - version "0.4.5" - resolved "https://registry.yarnpkg.com/@cubejs-client/core/-/core-0.4.5.tgz#6df4b6c15fbf678323d14bbaae83607ca1ebe437" - integrity sha512-EGitKr6zPtsOcMyFSDe6fLKS7nzFhMIptnbMgEPbvfhlqw4CyvSACSBoThMC4Sa3NVOjpmGOTnzKrglzegE3sw== +"@cubejs-client/core@^0.6.0": + version "0.6.0" + resolved "https://registry.yarnpkg.com/@cubejs-client/core/-/core-0.6.0.tgz#86b36526936a569e9de0b61bab4aecf9f9bb7584" + integrity sha512-z23Hghs6QH5E1/8nNAzqjWRtlmyqIsIFM8oSjj+/5npjwVl63RgHILex+nNzlTCanYZrHtkC+Fnta4ThmG9O5g== dependencies: "@babel/runtime" "^7.1.2" core-js "^2.5.3" @@ -1075,10 +1075,10 @@ ramda "^0.25.0" whatwg-fetch "^3.0.0" -"@cubejs-client/react@^0.4.5": - version "0.4.5" - resolved "https://registry.yarnpkg.com/@cubejs-client/react/-/react-0.4.5.tgz#4cc4dbf86865a53a37d63f8ee09ff1f5653cf078" - integrity sha512-hryecjoLOqFs1fEHtu0ErZZidbDGQIw6CJiUVXzQbqt7jhh7/GaD3jVcbshm0UDfH9sAPDKoYT0+jZv54o2lbA== +"@cubejs-client/react@^0.6.0": + version "0.6.0" + resolved "https://registry.yarnpkg.com/@cubejs-client/react/-/react-0.6.0.tgz#31516d96467a3d09fb018fb00b9f619cb465b7e1" + integrity sha512-9xwVp7ankCrF4wVdEbnYLNYNHMInavJCGnKObKhpzpAV0gqD3sGZHPAzmS2bf+WzxF9PWDkIJiooUCiTlJT9pQ== dependencies: "@babel/runtime" "^7.1.2" prop-types "^15.6.2" @@ -10816,6 +10816,20 @@ terser-webpack-plugin@1.2.2, terser-webpack-plugin@^1.1.0: webpack-sources "^1.1.0" worker-farm "^1.5.2" +terser-webpack-plugin@^1.2.3: + version "1.2.3" + resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-1.2.3.tgz#3f98bc902fac3e5d0de730869f50668561262ec8" + integrity sha512-GOK7q85oAb/5kE12fMuLdn2btOS9OBZn4VsecpHDywoUC/jLhSAKOiYo0ezx7ss2EXPMzyEWFoE0s1WLE+4+oA== + dependencies: + cacache "^11.0.2" + find-cache-dir "^2.0.0" + schema-utils "^1.0.0" + serialize-javascript "^1.4.0" + source-map "^0.6.1" + terser "^3.16.1" + webpack-sources "^1.1.0" + worker-farm "^1.5.2" + terser@^3.16.1: version "3.16.1" resolved "https://registry.yarnpkg.com/terser/-/terser-3.16.1.tgz#5b0dd4fa1ffd0b0b43c2493b2c364fd179160493" diff --git a/packages/cubejs-server-core/core/index.js b/packages/cubejs-server-core/core/index.js index b1fa610c2315d..943314f393ed2 100644 --- a/packages/cubejs-server-core/core/index.js +++ b/packages/cubejs-server-core/core/index.js @@ -66,6 +66,15 @@ class CubejsServerCore { } }); this.repository = new FileRepository(this.schemaPath); + this.repositoryFactory = options.repositoryFactory || (() => this.repository); + this.contextToDbType = typeof options.dbType === 'function' ? options.dbType : () => options.dbType; + this.appIdToCompilerApi = {}; + this.appIdToOrchestratorApi = {}; + this.contextToAppId = options.contextToAppId || (() => process.env.CUBEJS_APP || 'STANDALONE'); + this.orchestratorOptions = + typeof options.orchestratorOptions === 'function' ? + options.orchestratorOptions : + () => options.orchestratorOptions; const Analytics = require('analytics-node'); const client = new Analytics('dSR8JiNYIGKyQHKid9OaLYugXLao18hA', { flushInterval: 100 }); @@ -148,12 +157,10 @@ class CubejsServerCore { async initApp(app) { checkEnvForPlaceholders(); - this.compilerApi = this.createCompilerApi(this.repository); - this.orchestratorApi = this.createOrchestratorApi(); const apiGateway = new ApiGateway( this.apiSecret, - this.compilerApi, - this.orchestratorApi, + this.getCompilerApi.bind(this), + this.getOrchestratorApi.bind(this), this.logger, { basePath: this.options.basePath, checkAuthMiddleware: this.options.checkAuthMiddleware @@ -165,18 +172,55 @@ class CubejsServerCore { } } - createCompilerApi(repository) { - return new CompilerApi(repository, this.dbType, { - schemaVersion: this.options.schemaVersion, + getCompilerApi(context) { + const appId = this.contextToAppId(context); + if (!this.appIdToCompilerApi[appId]) { + this.appIdToCompilerApi[appId] = this.createCompilerApi( + this.repositoryFactory(context), { + dbType: this.contextToDbType(context), + schemaVersion: this.options.schemaVersion && (() => this.options.schemaVersion(context)) + } + ); + } + return this.appIdToCompilerApi[appId]; + } + + getOrchestratorApi(context) { + const appId = this.contextToAppId(context); + if (!this.appIdToOrchestratorApi[appId]) { + let driverPromise; + this.appIdToOrchestratorApi[appId] = this.createOrchestratorApi({ + getDriver: () => { + if (!driverPromise) { + const driver = this.driverFactory(context); + driverPromise = driver.testConnection().then(() => driver).catch(e => { + driverPromise = null; + throw e; + }); + } + return driverPromise; + }, + redisPrefix: appId, + orchestratorOptions: this.orchestratorOptions(context) + }); + } + return this.appIdToOrchestratorApi[appId]; + } + + createCompilerApi(repository, options) { + options = options || {}; + return new CompilerApi(repository, options.dbType || this.dbType, { + schemaVersion: options.schemaVersion || this.options.schemaVersion, devServer: this.options.devServer, logger: this.logger }); } - createOrchestratorApi() { - return new OrchestratorApi(() => this.getDriver(), this.logger, { - redisPrefix: process.env.CUBEJS_APP, - ...this.options.orchestratorOptions + createOrchestratorApi(options) { + options = options || {}; + return new OrchestratorApi(options.getDriver || this.getDriver.bind(this), this.logger, { + redisPrefix: options.redisPrefix || process.env.CUBEJS_APP, + ...(options.orchestratorOptions || this.options.orchestratorOptions) }); } diff --git a/packages/cubejs-server-core/yarn.lock b/packages/cubejs-server-core/yarn.lock index 2461228f10238..07f944de8fe65 100644 --- a/packages/cubejs-server-core/yarn.lock +++ b/packages/cubejs-server-core/yarn.lock @@ -2,29 +2,30 @@ # yarn lockfile v1 -"@cubejs-backend/api-gateway@^0.4.5": - version "0.4.5" - resolved "https://registry.yarnpkg.com/@cubejs-backend/api-gateway/-/api-gateway-0.4.5.tgz#a4dbf10a39cd1225c97e6c86e8913d7de8325256" - integrity sha512-E3NNqQs4oq76n2e1G9yv0q6NdxWPT/MiFVwA83Zzg0hcM/V4QFj5d8f+x2Hx0iTJ+EYSPZ6UFLtJO7ef2xasgg== +"@cubejs-backend/api-gateway@^0.6.2": + version "0.6.2" + resolved "https://registry.yarnpkg.com/@cubejs-backend/api-gateway/-/api-gateway-0.6.2.tgz#64a8f6f1975863907d6e38300160ca0fe63ad3f5" + integrity sha512-Dtv6GblNvNYK1y3Gb6S3lG1woULVgEWQpYttrKs3x8grUSF4AP7y+vpOpgq3abMaJhEG3FMR8hrbp308tUlq7A== dependencies: + chrono-node "^1.3.11" joi "^14.0.6" jsonwebtoken "^8.3.0" moment "^2.24.0" ramda "^0.25.0" -"@cubejs-backend/query-orchestrator@^0.4.4": - version "0.4.4" - resolved "https://registry.yarnpkg.com/@cubejs-backend/query-orchestrator/-/query-orchestrator-0.4.4.tgz#92d2eeb9c85547d6b2ebbb698b3c8d1b05e6dfbd" - integrity sha512-GQHYCS3TcWk5jzoWTC9rPQG4Xw/atIok8GlQJtTgG1LRvKhldvKfveLZCKb7vqt//eqQri72/MLTmiJ4LrQ4tQ== +"@cubejs-backend/query-orchestrator@^0.6.0": + version "0.6.0" + resolved "https://registry.yarnpkg.com/@cubejs-backend/query-orchestrator/-/query-orchestrator-0.6.0.tgz#5650556f9d9397777cc44384412dae68a3e6bb8d" + integrity sha512-Z04QvH+ZFHmztClHcnf4xA7LRxPcYHAI+c+6OcBzlNaz6GrIXyljlfhLqL904XrRm3ykuuW4WSp2HZ6pg9qJ9g== dependencies: ramda "^0.24.1" redis "^2.8.0" util-promisifyall "^1.0.4" -"@cubejs-backend/schema-compiler@^0.4.4": - version "0.4.4" - resolved "https://registry.yarnpkg.com/@cubejs-backend/schema-compiler/-/schema-compiler-0.4.4.tgz#a51dacf5e0abfd3d049e7c7645b734dd09078583" - integrity sha512-4oDV+K0vckD8jiLrldfist6a+LaUq1dirQHksckM7nXZJTMzGrmgqdpKMZWqVxjFbepqPzQLXEf4JhCIEZoSUg== +"@cubejs-backend/schema-compiler@^0.6.0": + version "0.6.0" + resolved "https://registry.yarnpkg.com/@cubejs-backend/schema-compiler/-/schema-compiler-0.6.0.tgz#6c8a5c0cb2ec50e32bd66c19e3df4a08c4b26b2d" + integrity sha512-Rm76ice0kmPfocs1qbK70RvUCdCsqs/2vI/D/0hgqpxYCw8vxxUN/3aZE23673QA8f4yMdBT0Kk3zAS39dXEuA== dependencies: babel-generator "^6.25.0" babel-traverse "^6.25.0" @@ -32,7 +33,7 @@ babylon "^6.17.4" humps "^2.0.1" inflection "^1.12.0" - joi "^10.6.0" + joi "^14.3.1" moment-range "^4.0.1" moment-timezone "^0.5.13" node-dijkstra "^2.5.0" @@ -311,6 +312,13 @@ charenc@~0.0.1: resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" integrity sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc= +chrono-node@^1.3.11: + version "1.3.11" + resolved "https://registry.yarnpkg.com/chrono-node/-/chrono-node-1.3.11.tgz#b86a26b7e3157edcc4fe3374e1b6f90caedc8e39" + integrity sha512-jDWRnY6nYvzfV3HPYBqo+tot7tcsUs9i3arGbMdI0TouPSXP2C2y/Ctp27rxKTQDi6yuTxAB2cw+Q6igGhOhdQ== + dependencies: + moment "2.21.0" + circular-json@^0.3.1: version "0.3.3" resolved "https://registry.yarnpkg.com/circular-json/-/circular-json-0.3.3.tgz#815c99ea84f6809529d2f45791bdf82711352d66" @@ -799,11 +807,6 @@ he@1.1.1: resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd" integrity sha1-k0EP0hsAlzUVH4howvJx80J+I/0= -hoek@4.x.x: - version "4.2.1" - resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.1.tgz#9634502aa12c445dd5a7c5734b572bb8738aacbb" - integrity sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA== - hoek@6.x.x: version "6.1.2" resolved "https://registry.yarnpkg.com/hoek/-/hoek-6.1.2.tgz#99e6d070561839de74ee427b61aa476bd6bddfd6" @@ -923,11 +926,6 @@ isarray@~1.0.0: resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= -isemail@2.x.x: - version "2.2.1" - resolved "https://registry.yarnpkg.com/isemail/-/isemail-2.2.1.tgz#0353d3d9a62951080c262c2aa0a42b8ea8e9e2a6" - integrity sha1-A1PT2aYpUQgMJiwqoKQrjqjp4qY= - isemail@3.x.x: version "3.2.0" resolved "https://registry.yarnpkg.com/isemail/-/isemail-3.2.0.tgz#59310a021931a9fb06bbb51e155ce0b3f236832c" @@ -949,22 +947,7 @@ istextorbinary@2.2.1: editions "^1.3.3" textextensions "2" -items@2.x.x: - version "2.1.2" - resolved "https://registry.yarnpkg.com/items/-/items-2.1.2.tgz#0849354595805d586dac98e7e6e85556ea838558" - integrity sha512-kezcEqgB97BGeZZYtX/MA8AG410ptURstvnz5RAgyFZ8wQFPMxHY8GpTq+/ZHKT3frSlIthUq7EvLt9xn3TvXg== - -joi@^10.6.0: - version "10.6.0" - resolved "https://registry.yarnpkg.com/joi/-/joi-10.6.0.tgz#52587f02d52b8b75cdb0c74f0b164a191a0e1fc2" - integrity sha512-hBF3LcqyAid+9X/pwg+eXjD2QBZI5eXnBFJYaAkH4SK3mp9QSRiiQnDYlmlz5pccMvnLcJRS4whhDOTCkmsAdQ== - dependencies: - hoek "4.x.x" - isemail "2.x.x" - items "2.x.x" - topo "2.x.x" - -joi@^14.0.6: +joi@^14.0.6, joi@^14.3.1: version "14.3.1" resolved "https://registry.yarnpkg.com/joi/-/joi-14.3.1.tgz#164a262ec0b855466e0c35eea2a885ae8b6c703c" integrity sha512-LQDdM+pkOrpAn4Lp+neNIFV3axv1Vna3j38bisbQhETPMANYRbFJFUyOZcOClYvM/hppMhGWuKSFEK9vjrB+bQ== @@ -1187,6 +1170,11 @@ moment-timezone@^0.5.13: dependencies: moment ">= 2.9.0" +moment@2.21.0: + version "2.21.0" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.21.0.tgz#2a114b51d2a6ec9e6d83cf803f838a878d8a023a" + integrity sha512-TCZ36BjURTeFTM/CwRcViQlfkMvL1/vFISuNLO5GkcVm1+QHfbSiNqZuWeMFjj1/3+uAjXswgRk30j1kkLYJBQ== + "moment@>= 2.9.0": version "2.23.0" resolved "https://registry.yarnpkg.com/moment/-/moment-2.23.0.tgz#759ea491ac97d54bac5ad776996e2a58cc1bc225" @@ -1709,13 +1697,6 @@ to-fast-properties@^1.0.3: resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-1.0.3.tgz#b83571fa4d8c25b82e231b06e3a3055de4ca1a47" integrity sha1-uDVx+k2MJbguIxsG46MFXeTKGkc= -topo@2.x.x: - version "2.0.2" - resolved "https://registry.yarnpkg.com/topo/-/topo-2.0.2.tgz#cd5615752539057c0dc0491a621c3bc6fbe1d182" - integrity sha1-zVYVdSU5BXwNwEkaYhw7xvvh0YI= - dependencies: - hoek "4.x.x" - topo@3.x.x: version "3.0.3" resolved "https://registry.yarnpkg.com/topo/-/topo-3.0.3.tgz#d5a67fb2e69307ebeeb08402ec2a2a6f5f7ad95c" diff --git a/packages/cubejs-serverless/Handlers.js b/packages/cubejs-serverless/Handlers.js new file mode 100644 index 0000000000000..97a6846a9b473 --- /dev/null +++ b/packages/cubejs-serverless/Handlers.js @@ -0,0 +1,90 @@ +const aws = require('aws-sdk'); +const express = require('serverless-express/express'); +const handler = require('serverless-express/handler'); +const cors = require('cors'); +const ServerCore = require('@cubejs-backend/server-core'); + +const sns = new aws.SNS(); + +const topicArn = (topic) => `arn:aws:sns:${process.env.AWS_REGION}:${process.env.AWS_ACCOUNT_ID}:${topic}`; +const sendSnsMessage = async (message, type, context) => { + const params = { + Message: JSON.stringify({ message, type, context }), + TopicArn: topicArn(`${process.env.CUBEJS_APP || 'cubejs'}-process`) + }; + await sns.publish(params).promise(); +}; + +const processHandlers = { + queryProcess: async (queryKey, orchestrator) => { + const queue = orchestrator.queryCache.getQueue(); + await queue.processQuery(queryKey); + }, + queryCancel: async (query, orchestrator) => { + const queue = orchestrator.queryCache.getQueue(); + await queue.processCancel(query); + }, + preAggregationProcess: async (queryKey, orchestrator) => { + const queue = orchestrator.preAggregations.getQueue(); + await queue.processQuery(queryKey); + }, + preAggregationCancel: async (query, orchestrator) => { + const queue = orchestrator.preAggregations.getQueue(); + await queue.processCancel(query); + } +}; + +class Handlers { + constructor(options) { + options = { + ...options, + orchestratorOptions: (context) => ({ + queryCacheOptions: { + queueOptions: { + sendProcessMessageFn: async (queryKey) => sendSnsMessage(queryKey, 'queryProcess', context), + sendCancelMessageFn: async (query) => sendSnsMessage(query, 'queryCancel', context) + } + }, + preAggregationsOptions: { + queueOptions: { + sendProcessMessageFn: async (queryKey) => sendSnsMessage(queryKey, 'preAggregationProcess', context), + sendCancelMessageFn: async (query) => sendSnsMessage(query, 'preAggregationCancel', context) + } + } + }) + }; + this.serverCore = new ServerCore(options); + } + + getApiHandler() { + if (!this.apiHandler) { + const app = express(); + app.use(cors()); + this.serverCore.initApp(app); + this.apiHandler = handler(app); + } + return this.apiHandler; + } + + api(event, context) { + return this.getApiHandler()(event, context); + } + + async process(event) { + await Promise.all(event.Records.map(async record => { + const message = JSON.parse(record.Sns.Message); + const processFn = processHandlers[message.type]; + if (!processFn) { + throw new Error(`Unrecognized message type: ${message.type}`); + } + const orchestratorApi = this.serverCore.getOrchestratorApi(message.context); + await processFn(message.message, orchestratorApi.orchestrator); + })); + + return { + statusCode: 200 + }; + } +} + +module.exports = Handlers; diff --git a/packages/cubejs-serverless/index.js b/packages/cubejs-serverless/index.js index ce13322ea32ef..21eef534d5032 100644 --- a/packages/cubejs-serverless/index.js +++ b/packages/cubejs-serverless/index.js @@ -1,91 +1,3 @@ -const aws = require('aws-sdk'); -const sns = new aws.SNS(); +const Handlers = require('./Handlers'); -const topicArn = (topic) => `arn:aws:sns:${process.env.AWS_REGION}:${process.env.AWS_ACCOUNT_ID}:${topic}`; -const sendSnsMessage = async (message, type) => { - const params = { - Message: JSON.stringify({ message, type }), - TopicArn: topicArn(`${process.env.CUBEJS_APP || 'cubejs'}-process`) - }; - await sns.publish(params).promise(); -}; - -exports.serverCore = require('@cubejs-backend/server-core').create({ - orchestratorOptions: { - queryCacheOptions: { - queueOptions: { - sendProcessMessageFn: async (queryKey) => sendSnsMessage(queryKey, 'queryProcess'), - sendCancelMessageFn: async (query) => sendSnsMessage(query, 'queryCancel') - } - }, - preAggregationsOptions: { - queueOptions: { - sendProcessMessageFn: async (queryKey) => sendSnsMessage(queryKey, 'preAggregationProcess'), - sendCancelMessageFn: async (query) => sendSnsMessage(query, 'preAggregationCancel') - } - } - } -}); - -const getHandler = () => { - if (!exports.apiHandler) { - const express = require('serverless-express/express'); - const app = express(); - app.use(require('cors')()); - exports.serverCore.initApp(app); - const handler = require('serverless-express/handler'); - exports.apiHandler = handler(app); - } - return exports.apiHandler; -}; - - -exports.api = (event, context) => getHandler()(event, context); - -let orchestratorApi; - -const getOrchestratorApi = async () => { - if (!orchestratorApi) { - orchestratorApi = await exports.serverCore.createOrchestratorApi(); - } - return orchestratorApi; -}; - -const handlers = { - queryProcess: async (queryKey, orchestrator) => { - const queue = orchestrator.queryCache.getQueue(); - await queue.processQuery(queryKey); - }, - queryCancel: async (query, orchestrator) => { - const queue = orchestrator.queryCache.getQueue(); - await queue.processCancel(query); - }, - preAggregationProcess: async (queryKey, orchestrator) => { - const queue = orchestrator.preAggregations.getQueue(); - await queue.processQuery(queryKey); - }, - preAggregationCancel: async (query, orchestrator) => { - const queue = orchestrator.preAggregations.getQueue(); - await queue.processCancel(query); - } -}; - -const processMessage = async (event) => { - await Promise.all(event.Records.map(async record => { - const message = JSON.parse(record.Sns.Message); - let processFn = handlers[message.type]; - if (!processFn) { - throw new Error(`Unrecognized message type: ${message.type}`); - } - const orchestratorApi = await getOrchestratorApi(); - await processFn(message.message, orchestratorApi.orchestrator); - })); - - return { - statusCode: 200 - } -}; - -exports.process = async (event) => { - return processMessage(event); -}; \ No newline at end of file +module.exports = new Handlers(); diff --git a/packages/cubejs-serverless/yarn.lock b/packages/cubejs-serverless/yarn.lock index b5e819f1ae471..8289e69ccc779 100644 --- a/packages/cubejs-serverless/yarn.lock +++ b/packages/cubejs-serverless/yarn.lock @@ -2,29 +2,30 @@ # yarn lockfile v1 -"@cubejs-backend/api-gateway@0.0.26": - version "0.0.26" - resolved "https://registry.yarnpkg.com/@cubejs-backend/api-gateway/-/api-gateway-0.0.26.tgz#9ba5b35b613d524b934072c1d14bb7b717af2103" - integrity sha512-3hJgzweTedq9NlLECZCLhLUjigbVO7IXTmb+9jaZsGdypbmT4jrQT91d+uvQCkQdsO9QS7erPyTumJs0kZ/iPA== +"@cubejs-backend/api-gateway@^0.6.2": + version "0.6.2" + resolved "https://registry.yarnpkg.com/@cubejs-backend/api-gateway/-/api-gateway-0.6.2.tgz#64a8f6f1975863907d6e38300160ca0fe63ad3f5" + integrity sha512-Dtv6GblNvNYK1y3Gb6S3lG1woULVgEWQpYttrKs3x8grUSF4AP7y+vpOpgq3abMaJhEG3FMR8hrbp308tUlq7A== dependencies: + chrono-node "^1.3.11" joi "^14.0.6" jsonwebtoken "^8.3.0" moment "^2.24.0" ramda "^0.25.0" -"@cubejs-backend/query-orchestrator@0.0.26": - version "0.0.26" - resolved "https://registry.yarnpkg.com/@cubejs-backend/query-orchestrator/-/query-orchestrator-0.0.26.tgz#3cb5f1d8b32f70355ee357a22d3a31f630fc44b2" - integrity sha512-hDxSbwmAaVMtsXfmNgB6DMa/E9zUkBuebdrfb8sM8/vJnXhYqg3KnHyQvVpotnEdm8kWOaKYuOPXvik52nJriA== +"@cubejs-backend/query-orchestrator@^0.6.0": + version "0.6.0" + resolved "https://registry.yarnpkg.com/@cubejs-backend/query-orchestrator/-/query-orchestrator-0.6.0.tgz#5650556f9d9397777cc44384412dae68a3e6bb8d" + integrity sha512-Z04QvH+ZFHmztClHcnf4xA7LRxPcYHAI+c+6OcBzlNaz6GrIXyljlfhLqL904XrRm3ykuuW4WSp2HZ6pg9qJ9g== dependencies: ramda "^0.24.1" redis "^2.8.0" util-promisifyall "^1.0.4" -"@cubejs-backend/schema-compiler@0.0.20": - version "0.0.20" - resolved "https://registry.yarnpkg.com/@cubejs-backend/schema-compiler/-/schema-compiler-0.0.20.tgz#8dbc06f3d7c38de0f2349c22fad809b726cb99bd" - integrity sha512-I5CO/fHvjyTQKmZNw/f8lsDoR9NFkafLK9MMKKxsSil7XqkJIj7AP6MtwTJgKVaz/PlJ9kmE/xa8zMwZcodTaQ== +"@cubejs-backend/schema-compiler@^0.6.0": + version "0.6.0" + resolved "https://registry.yarnpkg.com/@cubejs-backend/schema-compiler/-/schema-compiler-0.6.0.tgz#6c8a5c0cb2ec50e32bd66c19e3df4a08c4b26b2d" + integrity sha512-Rm76ice0kmPfocs1qbK70RvUCdCsqs/2vI/D/0hgqpxYCw8vxxUN/3aZE23673QA8f4yMdBT0Kk3zAS39dXEuA== dependencies: babel-generator "^6.25.0" babel-traverse "^6.25.0" @@ -32,28 +33,30 @@ babylon "^6.17.4" humps "^2.0.1" inflection "^1.12.0" - joi "^10.6.0" + joi "^14.3.1" moment-range "^4.0.1" moment-timezone "^0.5.13" node-dijkstra "^2.5.0" ramda "^0.24.1" syntax-error "^1.3.0" -"@cubejs-backend/server-core@0.0.26": - version "0.0.26" - resolved "https://registry.yarnpkg.com/@cubejs-backend/server-core/-/server-core-0.0.26.tgz#cc3473db0f59596101a7a90012a693e577687747" - integrity sha512-iKCCrsM7eNVAbSKKdWwTGFBhKJSBp2z+G7pFZh2j8jJgO9KttJmzCYBncm0mtV+XDuLVa+bl1PKayD9jjyIzew== +"@cubejs-backend/server-core@^0.6.2": + version "0.6.2" + resolved "https://registry.yarnpkg.com/@cubejs-backend/server-core/-/server-core-0.6.2.tgz#f759644d1aaad10c5ee972190cab60c1e938150b" + integrity sha512-sZFR9SUJH6pqA+kWKPed4fQSamTW6zgpQwkpqIJvu/+CgyniAiJApAcaThEYtRBs2UREpDH6DmQ+CXsmyfbd0g== dependencies: - "@cubejs-backend/api-gateway" "0.0.26" - "@cubejs-backend/query-orchestrator" "0.0.26" - "@cubejs-backend/schema-compiler" "0.0.20" + "@cubejs-backend/api-gateway" "^0.6.2" + "@cubejs-backend/query-orchestrator" "^0.6.0" + "@cubejs-backend/schema-compiler" "^0.6.0" analytics-node "^3.3.0" codesandbox-import-utils "^1.3.8" + cross-spawn "^6.0.5" fs-extra "^7.0.1" jsonwebtoken "^8.4.0" node-machine-id "^1.1.10" promise-timeout "^1.3.0" ramda "^0.25.0" + serve-static "^1.13.2" "@segment/loosely-validate-event@^2.0.0": version "2.0.0" @@ -410,6 +413,13 @@ charenc@~0.0.1: resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" integrity sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc= +chrono-node@^1.3.11: + version "1.3.11" + resolved "https://registry.yarnpkg.com/chrono-node/-/chrono-node-1.3.11.tgz#b86a26b7e3157edcc4fe3374e1b6f90caedc8e39" + integrity sha512-jDWRnY6nYvzfV3HPYBqo+tot7tcsUs9i3arGbMdI0TouPSXP2C2y/Ctp27rxKTQDi6yuTxAB2cw+Q6igGhOhdQ== + dependencies: + moment "2.21.0" + circular-json@^0.3.1: version "0.3.3" resolved "https://registry.yarnpkg.com/circular-json/-/circular-json-0.3.3.tgz#815c99ea84f6809529d2f45791bdf82711352d66" @@ -535,6 +545,17 @@ cross-spawn@^5.1.0: shebang-command "^1.2.0" which "^1.2.9" +cross-spawn@^6.0.5: + version "6.0.5" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" + integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ== + dependencies: + nice-try "^1.0.4" + path-key "^2.0.1" + semver "^5.5.0" + shebang-command "^1.2.0" + which "^1.2.9" + crypt@~0.0.1: version "0.0.2" resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b" @@ -974,11 +995,6 @@ he@1.1.1: resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd" integrity sha1-k0EP0hsAlzUVH4howvJx80J+I/0= -hoek@4.x.x: - version "4.2.1" - resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.1.tgz#9634502aa12c445dd5a7c5734b572bb8738aacbb" - integrity sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA== - hoek@6.x.x: version "6.1.2" resolved "https://registry.yarnpkg.com/hoek/-/hoek-6.1.2.tgz#99e6d070561839de74ee427b61aa476bd6bddfd6" @@ -1120,11 +1136,6 @@ isarray@^1.0.0, isarray@~1.0.0: resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= -isemail@2.x.x: - version "2.2.1" - resolved "https://registry.yarnpkg.com/isemail/-/isemail-2.2.1.tgz#0353d3d9a62951080c262c2aa0a42b8ea8e9e2a6" - integrity sha1-A1PT2aYpUQgMJiwqoKQrjqjp4qY= - isemail@3.x.x: version "3.2.0" resolved "https://registry.yarnpkg.com/isemail/-/isemail-3.2.0.tgz#59310a021931a9fb06bbb51e155ce0b3f236832c" @@ -1146,27 +1157,12 @@ istextorbinary@2.2.1: editions "^1.3.3" textextensions "2" -items@2.x.x: - version "2.1.2" - resolved "https://registry.yarnpkg.com/items/-/items-2.1.2.tgz#0849354595805d586dac98e7e6e85556ea838558" - integrity sha512-kezcEqgB97BGeZZYtX/MA8AG410ptURstvnz5RAgyFZ8wQFPMxHY8GpTq+/ZHKT3frSlIthUq7EvLt9xn3TvXg== - jmespath@0.15.0: version "0.15.0" resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= -joi@^10.6.0: - version "10.6.0" - resolved "https://registry.yarnpkg.com/joi/-/joi-10.6.0.tgz#52587f02d52b8b75cdb0c74f0b164a191a0e1fc2" - integrity sha512-hBF3LcqyAid+9X/pwg+eXjD2QBZI5eXnBFJYaAkH4SK3mp9QSRiiQnDYlmlz5pccMvnLcJRS4whhDOTCkmsAdQ== - dependencies: - hoek "4.x.x" - isemail "2.x.x" - items "2.x.x" - topo "2.x.x" - -joi@^14.0.6: +joi@^14.0.6, joi@^14.3.1: version "14.3.1" resolved "https://registry.yarnpkg.com/joi/-/joi-14.3.1.tgz#164a262ec0b855466e0c35eea2a885ae8b6c703c" integrity sha512-LQDdM+pkOrpAn4Lp+neNIFV3axv1Vna3j38bisbQhETPMANYRbFJFUyOZcOClYvM/hppMhGWuKSFEK9vjrB+bQ== @@ -1416,6 +1412,11 @@ moment-timezone@^0.5.13: dependencies: moment ">= 2.9.0" +moment@2.21.0: + version "2.21.0" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.21.0.tgz#2a114b51d2a6ec9e6d83cf803f838a878d8a023a" + integrity sha512-TCZ36BjURTeFTM/CwRcViQlfkMvL1/vFISuNLO5GkcVm1+QHfbSiNqZuWeMFjj1/3+uAjXswgRk30j1kkLYJBQ== + "moment@>= 2.9.0", moment@^2.24.0: version "2.24.0" resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b" @@ -1451,6 +1452,11 @@ next-tick@1: resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c" integrity sha1-yobR/ogoFpsBICCOPchCS524NCw= +nice-try@^1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" + integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== + node-dijkstra@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/node-dijkstra/-/node-dijkstra-2.5.0.tgz#0feb76c5a05f35b56e786de6df4d3364af28d4e8" @@ -1524,6 +1530,11 @@ path-is-inside@^1.0.2: resolved "https://registry.yarnpkg.com/path-is-inside/-/path-is-inside-1.0.2.tgz#365417dede44430d1c11af61027facf074bdfc53" integrity sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM= +path-key@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" + integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A= + path-parse@^1.0.6: version "1.0.6" resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" @@ -1755,6 +1766,11 @@ semver@^5.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-5.6.0.tgz#7e74256fbaa49c75aa7c7a205cc22799cac80004" integrity sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg== +semver@^5.5.0: + version "5.7.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.0.tgz#790a7cf6fea5459bac96110b29b60412dc8ff96b" + integrity sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA== + send@0.16.2: version "0.16.2" resolved "https://registry.yarnpkg.com/send/-/send-0.16.2.tgz#6ecca1e0f8c156d141597559848df64730a6bbc1" @@ -1774,7 +1790,7 @@ send@0.16.2: range-parser "~1.2.0" statuses "~1.4.0" -serve-static@1.13.2: +serve-static@1.13.2, serve-static@^1.13.2: version "1.13.2" resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.13.2.tgz#095e8472fd5b46237db50ce486a43f4b86c6cec1" integrity sha512-p/tdJrO4U387R9oMjb1oj7qSMaMfmOyd4j9hOFoxZe2baQszgHcSWjuya/CiT5kgZZKRudHNOA0pYXOl8rQ5nw== @@ -1985,13 +2001,6 @@ to-fast-properties@^1.0.3: resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-1.0.3.tgz#b83571fa4d8c25b82e231b06e3a3055de4ca1a47" integrity sha1-uDVx+k2MJbguIxsG46MFXeTKGkc= -topo@2.x.x: - version "2.0.2" - resolved "https://registry.yarnpkg.com/topo/-/topo-2.0.2.tgz#cd5615752539057c0dc0491a621c3bc6fbe1d182" - integrity sha1-zVYVdSU5BXwNwEkaYhw7xvvh0YI= - dependencies: - hoek "4.x.x" - topo@3.x.x: version "3.0.3" resolved "https://registry.yarnpkg.com/topo/-/topo-3.0.3.tgz#d5a67fb2e69307ebeeb08402ec2a2a6f5f7ad95c"