From 7379b84b4cf4dae49c4c2f22f77d6340158608ff Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Fri, 3 Apr 2020 15:48:44 -0700 Subject: [PATCH] fix: TypeError: date.match is not a function at BaseTimeDimension.formatFromDate --- packages/cubejs-schema-compiler/package.json | 2 +- .../test/MySqlPreAggregationsTest.js | 42 +++++++- packages/cubejs-schema-compiler/yarn.lock | 101 +++++------------- .../core/RefreshScheduler.js | 6 +- 4 files changed, 74 insertions(+), 77 deletions(-) diff --git a/packages/cubejs-schema-compiler/package.json b/packages/cubejs-schema-compiler/package.json index fbf0fc14872cf..db8d596b8ad5e 100644 --- a/packages/cubejs-schema-compiler/package.json +++ b/packages/cubejs-schema-compiler/package.json @@ -41,7 +41,7 @@ "istanbul": "^0.4.5", "mocha": "^3.4.2", "mssql": "^6.1.0", - "mysql": "^2.17.1", + "mysql": "^2.18.1", "nyc": "^15.0.0", "pg-promise": "^7.3.2", "request": "^2.88.0", diff --git a/packages/cubejs-schema-compiler/test/MySqlPreAggregationsTest.js b/packages/cubejs-schema-compiler/test/MySqlPreAggregationsTest.js index 1894a366ac916..d2028a31fae66 100644 --- a/packages/cubejs-schema-compiler/test/MySqlPreAggregationsTest.js +++ b/packages/cubejs-schema-compiler/test/MySqlPreAggregationsTest.js @@ -76,7 +76,8 @@ describe('MySqlPreAggregations', function test() { dimensionReferences: [source], timeDimensionReference: createdAt, granularity: 'day', - partitionGranularity: 'month' + partitionGranularity: 'month', + scheduledRefresh: true }, googleRollup: { type: 'rollup', @@ -171,6 +172,45 @@ describe('MySqlPreAggregations', function test() { }); }); + it('partitioned scheduled refresh', () => { + return compiler.compile().then(async () => { + const query = new MySqlQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitors.count' + ], + dimensions: [ + 'visitors.source' + ], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '', + timeDimensions: [{ + dimension: 'visitors.createdAt', + granularity: 'day', + dateRange: ['2016-12-30', '2017-01-30'] + }], + order: [{ + id: 'visitors.createdAt' + }], + }); + + const preAggregations = cubeEvaluator.scheduledPreAggregations(); + const partitionedPreAgg = + preAggregations.find(p => p.preAggregationName === 'partitioned' && p.cube === 'visitors'); + + const minMaxQueries = query.preAggregationStartEndQueries('visitors', partitionedPreAgg.preAggregation); + + console.log(minMaxQueries); + + const res = await dbRunner.testQueries(minMaxQueries); + + console.log(res); + + res.should.be.deepEqual( + [{ 'max(`visitors`.created_at)': '2017-01-07T08:00:00.000Z' }] + ); + }); + }); + it('segment', () => compiler.compile().then(() => { const query = new MySqlQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ diff --git a/packages/cubejs-schema-compiler/yarn.lock b/packages/cubejs-schema-compiler/yarn.lock index 48cd9158b9476..2a8312ab11571 100644 --- a/packages/cubejs-schema-compiler/yarn.lock +++ b/packages/cubejs-schema-compiler/yarn.lock @@ -51,7 +51,7 @@ invariant "^2.2.4" semver "^5.5.0" -"@babel/core@^7.7.5", "@babel/core@^7.9.0": +"@babel/core@^7.7.5": version "7.9.0" resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.9.0.tgz#ac977b538b77e132ff706f3b8a4dbad09c03c56e" integrity sha512-kWc7L0fw1xwvI0zi8OKVBuxRVefwGOrKSQMvrQ3dW+bIIavBY3/NpXmpjMy7bQnLgwgzWQZ8TlM57YHpHNHz4w== @@ -742,17 +742,6 @@ levenary "^1.1.1" semver "^5.5.0" -"@babel/register@^7.9.0": - version "7.9.0" - resolved "https://registry.yarnpkg.com/@babel/register/-/register-7.9.0.tgz#02464ede57548bddbb5e9f705d263b7c3f43d48b" - integrity sha512-Tv8Zyi2J2VRR8g7pC5gTeIN8Ihultbmk0ocyNz8H2nEZbmhp1N6q0A1UGsQbDvGP/sNinQKUHf3SqXwqjtFv4Q== - dependencies: - find-cache-dir "^2.0.0" - lodash "^4.17.13" - make-dir "^2.1.0" - pirates "^4.0.0" - source-map-support "^0.5.16" - "@babel/standalone@^7.3.4": version "7.8.6" resolved "https://registry.yarnpkg.com/@babel/standalone/-/standalone-7.8.6.tgz#1364534775c83bf7b7988e4ca98823bef56a0a53" @@ -1118,10 +1107,10 @@ bcrypt-pbkdf@^1.0.0: dependencies: tweetnacl "^0.14.3" -bignumber.js@7.2.1: - version "7.2.1" - resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-7.2.1.tgz#80c048759d826800807c4bfd521e50edbba57a5f" - integrity sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ== +bignumber.js@9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.0.0.tgz#805880f84a329b5eac6e7cb6f8274b6d82bdf075" + integrity sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A== bl@^1.0.0: version "1.2.2" @@ -1914,15 +1903,6 @@ file-entry-cache@^5.0.1: dependencies: flat-cache "^2.0.1" -find-cache-dir@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-2.1.0.tgz#8d0f94cd13fe43c6c7c261a0d86115ca918c05f7" - integrity sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ== - dependencies: - commondir "^1.0.1" - make-dir "^2.0.0" - pkg-dir "^3.0.0" - find-cache-dir@^3.2.0: version "3.3.1" resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-3.3.1.tgz#89b33fad4a4670daa94f855f7fbe31d6d84fe880" @@ -2745,14 +2725,6 @@ loose-envify@^1.0.0: dependencies: js-tokens "^3.0.0 || ^4.0.0" -make-dir@^2.0.0, make-dir@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5" - integrity sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA== - dependencies: - pify "^4.0.1" - semver "^5.6.0" - make-dir@^3.0.0, make-dir@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.0.2.tgz#04a1acbf22221e1d6ef43559f43e05a90dbb4392" @@ -2879,13 +2851,13 @@ mute-stream@0.0.8: resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d" integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA== -mysql@^2.17.1: - version "2.17.1" - resolved "https://registry.yarnpkg.com/mysql/-/mysql-2.17.1.tgz#62bba4a039a9b2f73638cd1652ce50fc6f682899" - integrity sha512-7vMqHQ673SAk5C8fOzTG2LpPcf3bNt0oL3sFpxPEEFp1mdlDcrLK0On7z8ZYKaaHrHwNcQ/MTUz7/oobZ2OyyA== +mysql@^2.18.1: + version "2.18.1" + resolved "https://registry.yarnpkg.com/mysql/-/mysql-2.18.1.tgz#2254143855c5a8c73825e4522baf2ea021766717" + integrity sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig== dependencies: - bignumber.js "7.2.1" - readable-stream "2.3.6" + bignumber.js "9.0.0" + readable-stream "2.3.7" safe-buffer "5.1.2" sqlstring "2.3.1" @@ -2924,11 +2896,6 @@ node-duration@^1.0.4: resolved "https://registry.yarnpkg.com/node-duration/-/node-duration-1.0.4.tgz#3e94ecc0e473691c89c4560074503362071cecac" integrity sha512-eUXYNSY7DL53vqfTosggWkvyIW3bhAcqBDIlolgNYlZhianXTrCL50rlUJWD1eRqkIxMppXTfiFbp+9SjpPrgA== -node-modules-regexp@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/node-modules-regexp/-/node-modules-regexp-1.0.0.tgz#8d9dbe28964a4ac5712e9131642107c71e90ec40" - integrity sha1-jZ2+KJZKSsVxLpExZCEHxx6Q7EA= - node-preload@^0.2.0: version "0.2.1" resolved "https://registry.yarnpkg.com/node-preload/-/node-preload-0.2.1.tgz#c03043bb327f417a18fee7ab7ee57b408a144301" @@ -3265,18 +3232,6 @@ pify@^2.0.0: resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" integrity sha1-7RQaasBDqEnqWISY59yosVMw6Qw= -pify@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/pify/-/pify-4.0.1.tgz#4b2cd25c50d598735c50292224fd8c6df41e3231" - integrity sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g== - -pirates@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.1.tgz#643a92caf894566f91b2b986d2c66950a8e2fb87" - integrity sha512-WuNqLTbMI3tmfef2TKxlQmAiLHKtFhlsCZnPIpuv2Ow0RDVO8lfy1Opf4NUzlMXLjPl+Men7AuVdX6TA+s+uGA== - dependencies: - node-modules-regexp "^1.0.0" - pkg-dir@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-2.0.0.tgz#f6d5d1109e19d63edf428e0bd57e12777615334b" @@ -3284,13 +3239,6 @@ pkg-dir@^2.0.0: dependencies: find-up "^2.1.0" -pkg-dir@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-3.0.0.tgz#2749020f239ed990881b1f71210d51eb6523bea3" - integrity sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw== - dependencies: - find-up "^3.0.0" - pkg-dir@^4.1.0: version "4.2.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" @@ -3400,7 +3348,20 @@ read-pkg@^2.0.0: normalize-package-data "^2.3.2" path-type "^2.0.0" -readable-stream@2.3.6, readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.5: +readable-stream@2.3.7: + version "2.3.7" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" + integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + +readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.5: version "2.3.6" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw== @@ -3647,7 +3608,7 @@ semver@7.0.0: resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== -semver@^5.4.1, semver@^5.6.0: +semver@^5.4.1: version "5.7.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== @@ -3744,20 +3705,12 @@ slice-ansi@^2.1.0: astral-regex "^1.0.0" is-fullwidth-code-point "^2.0.0" -source-map-support@^0.5.16: - version "0.5.16" - resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.16.tgz#0ae069e7fe3ba7538c64c98515e35339eac5a042" - integrity sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ== - dependencies: - buffer-from "^1.0.0" - source-map "^0.6.0" - source-map@^0.5.0: version "0.5.7" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w= -source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.1: +source-map@^0.6.1, source-map@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== diff --git a/packages/cubejs-server-core/core/RefreshScheduler.js b/packages/cubejs-server-core/core/RefreshScheduler.js index 833ded1d2d2d2..9e2dbca90c107 100644 --- a/packages/cubejs-server-core/core/RefreshScheduler.js +++ b/packages/cubejs-server-core/core/RefreshScheduler.js @@ -25,7 +25,11 @@ class RefreshScheduler { })) ); - const extractDate = ({ data }) => data[0] && data[0][Object.keys(data[0])[0]].toString(); + const extractDate = ({ data }) => { + // TODO some backends return dates as objects here. Use ApiGateway data transformation ? + data = JSON.parse(JSON.stringify(data)); + return data[0] && data[0][Object.keys(data[0])[0]]; + }; const baseQuery = { ...queryingOptions,