From c6ac87363d7ce89b50319304196b01f6b9c026a5 Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Fri, 11 Oct 2019 10:53:39 -0700 Subject: [PATCH] feat: `ungrouped` queries support --- .../@cubejs-backend-server-core.md | 5 ++ docs/Cube.js-Frontend/Query-Format.md | 3 + packages/cubejs-api-gateway/index.js | 3 +- .../adapter/BaseQuery.js | 26 +++++++- .../test/SQLGenerationTest.js | 64 +++++++++++++++++++ .../cubejs-server-core/core/CompilerApi.js | 4 +- packages/cubejs-server-core/core/index.js | 3 +- 7 files changed, 104 insertions(+), 4 deletions(-) diff --git a/docs/Cube.js-Backend/@cubejs-backend-server-core.md b/docs/Cube.js-Backend/@cubejs-backend-server-core.md index e3f0992d0dfd7..9c68d8b2a5ffc 100644 --- a/docs/Cube.js-Backend/@cubejs-backend-server-core.md +++ b/docs/Cube.js-Backend/@cubejs-backend-server-core.md @@ -55,6 +55,7 @@ Both [CubejsServerCore](@cubejs-backend-server-core) and [CubejsServer](@cubejs- preAggregationsSchema: String | (context: RequestContext) => String, schemaVersion: (context: RequestContext) => String, telemetry: Boolean, + allowUngroupedWithoutPrimaryKey: Boolean, orchestratorOptions: { redisPrefix: String, queryCacheOptions: { @@ -252,6 +253,10 @@ CubejsServerCore.create({ }); ``` +### allowUngroupedWithoutPrimaryKey + +Providing `allowUngroupedWithoutPrimaryKey: true` disables primary key inclusion check for `ungrouped` queries. + ### telemetry Cube.js collects high-level anonymous usage statistics for servers started in development mode. It doesn't track any credentials, schema contents or queries issued. This statistics is used solely for the purpose of constant cube.js improvement. diff --git a/docs/Cube.js-Frontend/Query-Format.md b/docs/Cube.js-Frontend/Query-Format.md index e0d64e2e3e693..5038c668ad93c 100644 --- a/docs/Cube.js-Frontend/Query-Format.md +++ b/docs/Cube.js-Frontend/Query-Format.md @@ -30,6 +30,9 @@ Query has the following properties: fields to order is based on the order of the keys in the object. - `timezone`: All time based calculations performed within Cube.js are timezone-aware. Using this property you can set your desired timezone in [TZ Database Name](https://en.wikipedia.org/wiki/Tz_database) format, e.g.: `America/Los_Angeles`. The default value is `UTC`. - `renewQuery`: If `renewQuery` is set to `true`, query will always refresh cache and return the latest data from the database. The default value is `false`. +- `ungrouped`: If `ungrouped` is set to `true` no `GROUP BY` statement will be added to the query and raw results after filtering and joining will be returned. +By default `ungrouped` query requires to pass primary key as a dimension of every cube involved in query for security purpose. +To disable this behavior please see [allowUngroupedWithoutPrimaryKey](@cubejs-backend-server-core#allow-ungrouped-without-primary-key) server option. ```js { diff --git a/packages/cubejs-api-gateway/index.js b/packages/cubejs-api-gateway/index.js index 000f8ace5a4c0..0ef4708dac754 100644 --- a/packages/cubejs-api-gateway/index.js +++ b/packages/cubejs-api-gateway/index.js @@ -130,7 +130,8 @@ const querySchema = Joi.object().keys({ timezone: Joi.string(), limit: Joi.number().integer().min(1).max(50000), offset: Joi.number().integer().min(0), - renewQuery: Joi.boolean() + renewQuery: Joi.boolean(), + ungrouped: Joi.boolean() }); const normalizeQuery = (query) => { diff --git a/packages/cubejs-schema-compiler/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/adapter/BaseQuery.js index c25db807c8c3b..b3b3f34c75627 100644 --- a/packages/cubejs-schema-compiler/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/adapter/BaseQuery.js @@ -78,6 +78,27 @@ class BaseQuery { } this.externalQueryClass = this.options.externalQueryClass; + this.initUngrouped(); + } + + initUngrouped() { + this.ungrouped = this.options.ungrouped; + if (this.ungrouped) { + if (!this.options.allowUngroupedWithoutPrimaryKey) { + const cubes = R.uniq([this.join.root].concat(this.join.joins.map(j => j.originalTo))); + const primaryKeyNames = cubes.map(c => this.primaryKeyName(c)); + const missingPrimaryKeys = primaryKeyNames.filter(key => !this.dimensions.find(d => d.dimension === key)); + if (missingPrimaryKeys.length) { + throw new UserError(`Ungrouped query requires primary keys to be present in dimensions: ${missingPrimaryKeys.map(k => `'${k}'`).join(', ')}. Pass allowUngroupedWithoutPrimaryKey option to disable this check.`); + } + } + if (this.measures.length) { + throw new UserError(`Measures aren't allowed in ungrouped query`); + } + if (this.measureFilters.length) { + throw new UserError(`Measure filters aren't allowed in ungrouped query`); + } + } } get subQueryDimensions() { @@ -162,7 +183,7 @@ class BaseQuery { } buildParamAnnotatedSql() { - if (!this.options.preAggregationQuery) { + if (!this.options.preAggregationQuery && !this.ungrouped) { const preAggregationForQuery = this.preAggregations.findPreAggregationForQuery(); if (preAggregationForQuery) { return this.preAggregations.rollupPreAggregation(preAggregationForQuery); @@ -740,6 +761,9 @@ class BaseQuery { } groupByClause() { + if (this.ungrouped) { + return ''; + } const dimensionColumns = this.dimensionColumns(); return dimensionColumns.length ? ` GROUP BY ${dimensionColumns.map((c, i) => `${i + 1}`).join(', ')}` : ''; } diff --git a/packages/cubejs-schema-compiler/test/SQLGenerationTest.js b/packages/cubejs-schema-compiler/test/SQLGenerationTest.js index 54cefc1e034dc..8a4fc39406f20 100644 --- a/packages/cubejs-schema-compiler/test/SQLGenerationTest.js +++ b/packages/cubejs-schema-compiler/test/SQLGenerationTest.js @@ -1,3 +1,4 @@ +/* globals it, describe, after */ /* eslint-disable quote-props */ const UserError = require('../compiler/UserError'); const PostgresQuery = require('../adapter/PostgresQuery'); @@ -1336,4 +1337,67 @@ describe('SQL Generation', function test() { } ]) ); + + it('ungrouped', () => runQueryTest({ + measures: [], + dimensions: [ + 'visitors.id' + ], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2016-01-09', '2017-01-10'] + }], + order: [{ + id: 'visitors.created_at' + }], + timezone: 'America/Los_Angeles', + ungrouped: true + }, [{ + "visitors__id": 6, + "visitors__created_at_date": "2016-09-06T00:00:00.000Z" + }, { + "visitors__id": 1, + "visitors__created_at_date": "2017-01-02T00:00:00.000Z" + }, { + "visitors__id": 2, + "visitors__created_at_date": "2017-01-04T00:00:00.000Z" + }, { + "visitors__id": 3, + "visitors__created_at_date": "2017-01-05T00:00:00.000Z" + }, { + "visitors__id": 4, + "visitors__created_at_date": "2017-01-06T00:00:00.000Z" + }, { + "visitors__id": 5, + "visitors__created_at_date": "2017-01-06T00:00:00.000Z" + }])); + + it('ungrouped without id', () => runQueryTest({ + measures: [], + dimensions: [], + timeDimensions: [{ + dimension: 'visitors.created_at', + granularity: 'date', + dateRange: ['2016-01-09', '2017-01-10'] + }], + order: [{ + id: 'visitors.created_at' + }], + timezone: 'America/Los_Angeles', + ungrouped: true, + allowUngroupedWithoutPrimaryKey: true + }, [{ + "visitors__created_at_date": "2016-09-06T00:00:00.000Z" + }, { + "visitors__created_at_date": "2017-01-02T00:00:00.000Z" + }, { + "visitors__created_at_date": "2017-01-04T00:00:00.000Z" + }, { + "visitors__created_at_date": "2017-01-05T00:00:00.000Z" + }, { + "visitors__created_at_date": "2017-01-06T00:00:00.000Z" + }, { + "visitors__created_at_date": "2017-01-06T00:00:00.000Z" + }])); }); diff --git a/packages/cubejs-server-core/core/CompilerApi.js b/packages/cubejs-server-core/core/CompilerApi.js index 871240319a403..8b519a87df74a 100644 --- a/packages/cubejs-server-core/core/CompilerApi.js +++ b/packages/cubejs-server-core/core/CompilerApi.js @@ -10,6 +10,7 @@ class CompilerApi { this.allowNodeRequire = options.allowNodeRequire == null ? true : options.allowNodeRequire; this.logger = this.options.logger; this.preAggregationsSchema = this.options.preAggregationsSchema; + this.allowUngroupedWithoutPrimaryKey = this.options.allowUngroupedWithoutPrimaryKey; } async getCompilers() { @@ -39,7 +40,8 @@ class CompilerApi { this.dbType, { ...query, externalDbType: this.options.externalDbType, - preAggregationsSchema: this.preAggregationsSchema + preAggregationsSchema: this.preAggregationsSchema, + allowUngroupedWithoutPrimaryKey: this.allowUngroupedWithoutPrimaryKey } ); return (await this.getCompilers()).compiler.withQuery(sqlGenerator, () => ({ diff --git a/packages/cubejs-server-core/core/index.js b/packages/cubejs-server-core/core/index.js index 10806399896cb..ffa8d0893cfe8 100644 --- a/packages/cubejs-server-core/core/index.js +++ b/packages/cubejs-server-core/core/index.js @@ -262,7 +262,8 @@ class CubejsServerCore { devServer: this.options.devServer, logger: this.logger, externalDbType: options.externalDbType, - preAggregationsSchema: options.preAggregationsSchema + preAggregationsSchema: options.preAggregationsSchema, + allowUngroupedWithoutPrimaryKey: this.options.allowUngroupedWithoutPrimaryKey }); }