feat: updateWindow property for incremental partitioned rollup refr…

…eshKey
cube-js · Jan 21, 2020 · 09c0a86 · 09c0a86
1 parent 6242b36
commit 09c0a86
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 5 deletions.
diff --git a/packages/cubejs-schema-compiler/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/adapter/BaseQuery.js
@@ -1479,15 +1479,17 @@ class BaseQuery {
     return `FLOOR(${numeric})`;
   }
 
-  incrementalRefreshKey(query, originalRefreshKey) {
+  incrementalRefreshKey(query, originalRefreshKey, options) {
+    options = options || {};
+    const updateWindow = options.window;
     return query.evaluateSql(
       null,
       (FILTER_PARAMS) => query.caseWhenStatement([{
         sql: FILTER_PARAMS[
           query.timeDimensions[0].path()[0]
         ][
           query.timeDimensions[0].path()[1]
-        ].filter((from, to) => `${query.nowTimestampSql()} < ${this.timeStampCast(to)}`),
+        ].filter((from, to) => `${query.nowTimestampSql()} < ${updateWindow ? this.addInterval(this.timeStampCast(to), updateWindow) : this.timeStampCast(to)}`),
         label: originalRefreshKey
       }])
     );
@@ -1514,7 +1516,11 @@ class BaseQuery {
         if (!preAggregation.partitionGranularity) {
           throw new UserError(`Incremental refresh key can only be used for partitioned pre-aggregations`);
         }
-        refreshKey = this.incrementalRefreshKey(preAggregationQueryForSql, refreshKey);
+        refreshKey = this.incrementalRefreshKey(
+          preAggregationQueryForSql,
+          refreshKey,
+          { window: preAggregation.refreshKey.updateWindow }
+        );
       }
       if (preAggregation.refreshKey.every || preAggregation.refreshKey.incremental) {
         return {

diff --git a/packages/cubejs-schema-compiler/compiler/CubeValidator.js b/packages/cubejs-schema-compiler/compiler/CubeValidator.js
@@ -67,7 +67,8 @@ const BasePreAggregation = {
     }),
     Joi.object().keys({
       every: everyInterval,
-      incremental: Joi.boolean()
+      incremental: Joi.boolean(),
+      updateWindow: timeInterval
     })
   ),
   useOriginalSqlPreAggregations: Joi.boolean(),

diff --git a/packages/cubejs-schema-compiler/test/PreAggregationsTest.js b/packages/cubejs-schema-compiler/test/PreAggregationsTest.js
@@ -204,7 +204,8 @@ describe('PreAggregations', function test() {
           partitionGranularity: 'month',
           refreshKey: {
             every: '1 hour',
-            incremental: true
+            incremental: true,
+            updateWindow: '1 day'
           }
         }
       }