Skip to content

Commit

Permalink
feat(mssql-driver): add readonly aggregation for mssql sources (#920)…
Browse files Browse the repository at this point in the history 
… Thanks to @JoshMentzer!

Co-authored-by: Joshua D. Mentzer <mentzerj@trinity-health.org>
  • Loading branch information
@JoshMentzer @mentzerj
JoshMentzer and mentzerj authored Aug 11, 2020
1 parent 92cd0b3 commit dfeccca
Show file tree
Hide file tree
Showing 3 changed files with 181 additions and 141 deletions.
20 changes: 19 additions & 1 deletion packages/cubejs-mssql-driver/driver/MSSqlDriver.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ class MSSqlDriver extends BaseDriver {
process.env.CUBEJS_DB_DOMAIN : undefined,
requestTimeout: 10 * 60 * 1000, // 10 minutes
options: {
encrypt: !!process.env.CUBEJS_DB_SSL || false
encrypt: !!process.env.CUBEJS_DB_SSL || false,
useUTC: false
},
pool: {
max: 8,
Expand Down Expand Up @@ -72,6 +73,23 @@ class MSSqlDriver extends BaseDriver {
return null;
});
}

async downloadQueryResults(query, values) {
const result = await this.query(query, values);
const types = Object.keys(result.columns).map((key) => ({
name: result.columns[key].name,
type: this.toGenericType(result.columns[key].type.declaration),
}));

return {
rows: result,
types,
};
}

readOnly() {
return !!this.config.readOnly;
}
}

module.exports = MSSqlDriver;
14 changes: 7 additions & 7 deletions packages/cubejs-schema-compiler/adapter/MssqlQuery.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,13 @@ class MssqlParamAllocator extends ParamAllocator {
}

const GRANULARITY_TO_INTERVAL = {
day: (date) => `FORMAT(${date}, 'yyyy-MM-ddT00:00:00.000')`,
week: (date) => `FORMAT(dateadd(week, DATEDIFF(week, '1900-01-01', ${date}), '1900-01-01'), 'yyyy-MM-ddT00:00:00.000')`,
hour: (date) => `FORMAT(${date}, 'yyyy-MM-ddTHH:00:00.000')`,
minute: (date) => `FORMAT(${date}, 'yyyy-MM-ddTHH:mm:00.000')`,
second: (date) => `FORMAT(${date}, 'yyyy-MM-ddTHH:mm:ss.000')`,
month: (date) => `FORMAT(${date}, 'yyyy-MM-01T00:00:00.000')`,
year: (date) => `FORMAT(${date}, 'yyyy-01-01T00:00:00.000')`
day: (date) => `dateadd(day, DATEDIFF(day, 0, ${date}), 0)`,
week: (date) => `dateadd(week, DATEDIFF(week, 0, ${date}), 0)`,
hour: (date) => `dateadd(hour, DATEDIFF(hour, 0, ${date}), 0)`,
minute: (date) => `dateadd(minute, DATEDIFF(minute, 0, ${date}), 0)`,
second: (date) => `CAST(FORMAT(${date}, 'yyyy-MM-ddTHH:mm:ss.000') AS datetime)`, // until SQL 2016, this causes an int overflow; in SQL 2016 these calls can be changed to DATEDIFF_BIG
month: (date) => `dateadd(month, DATEDIFF(month, 0, ${date}), 0)`,
year: (date) => `dateadd(year, DATEDIFF(year, 0, ${date}), 0)`,
};

class MssqlFilter extends BaseFilter {
Expand Down
288 changes: 155 additions & 133 deletions packages/cubejs-schema-compiler/test/MSSqlPreAggregationsTest.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,155 +178,177 @@ describe('MSSqlPreAggregations', function test() {
preAggregation = Array.isArray(preAggregation) ? preAggregation : [preAggregation];
return [
preAggregation.reduce(
(replacedQuery, desc) => replacedQuery.replace(new RegExp(desc.tableName, 'g'), `##${desc.tableName}_${suffix}`), toReplace
(replacedQuery, desc) =>
replacedQuery.replace(new RegExp(desc.tableName, 'g'), `##${desc.tableName}_${suffix}`),
toReplace
),
params
params,
];
}

function tempTablePreAggregations(preAggregationsDescriptions) {
return R.unnest(preAggregationsDescriptions.map(
desc => desc.invalidateKeyQueries.concat([[desc.loadSql[0], desc.loadSql[1]]])
));
return R.unnest(
preAggregationsDescriptions.map((desc) => desc.invalidateKeyQueries.concat([[desc.loadSql[0], desc.loadSql[1]]]))
);
}

it('simple pre-aggregation', () => compiler.compile().then(() => {
const query = new MSSqlQuery({ joinGraph, cubeEvaluator, compiler }, {
measures: [
'visitors.count'
],
timeDimensions: [{
dimension: 'visitors.createdAt',
granularity: 'day',
dateRange: ['2017-01-01', '2017-01-30']
}],
timezone: 'UTC',
order: [{
id: 'visitors.createdAt'
}],
preAggregationsSchema: ''
});
it('simple pre-aggregation', () =>
compiler.compile().then(() => {
const query = new MSSqlQuery(
{ joinGraph, cubeEvaluator, compiler },
{
measures: ['visitors.count'],
timeDimensions: [
{
dimension: 'visitors.createdAt',
granularity: 'day',
dateRange: ['2017-01-01', '2017-01-30'],
},
],
timezone: 'UTC',
order: [
{
id: 'visitors.createdAt',
},
],
preAggregationsSchema: '',
}
);

const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
console.log(preAggregationsDescription);
const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
console.log(preAggregationsDescription);

return dbRunner.testQueries(tempTablePreAggregations(preAggregationsDescription).concat([
query.buildSqlAndParams()
]).map(q => replaceTableName(q, preAggregationsDescription, 1))).then(res => {
res.should.be.deepEqual(
[
{
"visitors__created_at_day": "2017-01-03T00:00:00.000",
"visitors__count": 1
},
{
"visitors__created_at_day": "2017-01-05T00:00:00.000",
"visitors__count": 1
},
{
"visitors__created_at_day": "2017-01-06T00:00:00.000",
"visitors__count": 1
},
{
"visitors__created_at_day": "2017-01-07T00:00:00.000",
"visitors__count": 2
}
]
return dbRunner
.testQueries(
tempTablePreAggregations(preAggregationsDescription)
.concat([query.buildSqlAndParams()])
.map((q) => replaceTableName(q, preAggregationsDescription, 1))
)
.then((res) => {
res.should.be.deepEqual([
{
visitors__created_at_day: new Date('2017-01-03T00:00:00.000Z'),
visitors__count: 1,
},
{
visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z'),
visitors__count: 1,
},
{
visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z'),
visitors__count: 1,
},
{
visitors__created_at_day: new Date('2017-01-07T00:00:00.000Z'),
visitors__count: 2,
},
]);
});
}));

it('leaf measure pre-aggregation', () =>
compiler.compile().then(() => {
const query = new MSSqlQuery(
{ joinGraph, cubeEvaluator, compiler },
{
measures: ['visitors.ratio'],
timeDimensions: [
{
dimension: 'visitors.createdAt',
granularity: 'day',
dateRange: ['2017-01-01', '2017-01-30'],
},
],
timezone: 'UTC',
order: [
{
id: 'visitors.createdAt',
},
],
preAggregationsSchema: '',
}
);
});
}));

it('leaf measure pre-aggregation', () => compiler.compile().then(() => {
const query = new MSSqlQuery({ joinGraph, cubeEvaluator, compiler }, {
measures: [
'visitors.ratio'
],
timeDimensions: [{
dimension: 'visitors.createdAt',
granularity: 'day',
dateRange: ['2017-01-01', '2017-01-30']
}],
timezone: 'UTC',
order: [{
id: 'visitors.createdAt'
}],
preAggregationsSchema: ''
});
const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
console.log(preAggregationsDescription);
preAggregationsDescription[0].loadSql[0].should.match(/visitors_ratio/);

const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
console.log(preAggregationsDescription);
preAggregationsDescription[0].loadSql[0].should.match(/visitors_ratio/);
return dbRunner
.testQueries(
tempTablePreAggregations(preAggregationsDescription)
.concat([query.buildSqlAndParams()])
.map((q) => replaceTableName(q, preAggregationsDescription, 10))
)
.then((res) => {
res.should.be.deepEqual([
{
visitors__created_at_day: new Date('2017-01-03T00:00:00.000Z'),
visitors__ratio: 0.333333333333,
},
{
visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z'),
visitors__ratio: 0.5,
},
{
visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z'),
visitors__ratio: 1,
},
{
visitors__created_at_day: new Date('2017-01-07T00:00:00.000Z'),
visitors__ratio: null,
},
]);
});
}));

return dbRunner.testQueries(tempTablePreAggregations(preAggregationsDescription).concat([
query.buildSqlAndParams()
]).map(q => replaceTableName(q, preAggregationsDescription, 10))).then(res => {
res.should.be.deepEqual(
[
{
"visitors__created_at_day": "2017-01-03T00:00:00.000",
"visitors__ratio": 0.333333333333
},
{
"visitors__created_at_day": "2017-01-05T00:00:00.000",
"visitors__ratio": 0.5
},
{
"visitors__created_at_day": "2017-01-06T00:00:00.000",
"visitors__ratio": 1
},
{
"visitors__created_at_day": "2017-01-07T00:00:00.000",
"visitors__ratio": null
}
]
it('segment', () =>
compiler.compile().then(() => {
const query = new MSSqlQuery(
{ joinGraph, cubeEvaluator, compiler },
{
measures: ['visitors.checkinsTotal'],
dimensions: [],
segments: ['visitors.google'],
timezone: 'UTC',
preAggregationsSchema: '',
timeDimensions: [
{
dimension: 'visitors.createdAt',
granularity: 'day',
dateRange: ['2016-12-30', '2017-01-06'],
},
],
order: [
{
id: 'visitors.createdAt',
},
],
}
);
});
}));

it('segment', () => compiler.compile().then(() => {
const query = new MSSqlQuery({ joinGraph, cubeEvaluator, compiler }, {
measures: [
'visitors.checkinsTotal'
],
dimensions: [],
segments: ['visitors.google'],
timezone: 'UTC',
preAggregationsSchema: '',
timeDimensions: [{
dimension: 'visitors.createdAt',
granularity: 'day',
dateRange: ['2016-12-30', '2017-01-06']
}],
order: [{
id: 'visitors.createdAt'
}],
});

const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
console.log(preAggregationsDescription);
const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
console.log(preAggregationsDescription);

const queries = tempTablePreAggregations(preAggregationsDescription);
const queries = tempTablePreAggregations(preAggregationsDescription);

console.log(JSON.stringify(queries.concat(queryAndParams)));
console.log(JSON.stringify(queries.concat(queryAndParams)));

return dbRunner.testQueries(
queries.concat([queryAndParams]).map(q => replaceTableName(q, preAggregationsDescription, 142))
).then(res => {
console.log(JSON.stringify(res));
res.should.be.deepEqual(
[
{
"visitors__created_at_day": "2017-01-06T00:00:00.000",
"visitors__checkins_total": 1
}
]
);
});
}));
return dbRunner
.testQueries(queries.concat([queryAndParams]).map((q) => replaceTableName(q, preAggregationsDescription, 142)))
.then((res) => {
console.log(JSON.stringify(res));
res.should.be.deepEqual([
{
visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z'),
visitors__checkins_total: 1,
},
]);
});
}));
});

0 comments on commit dfeccca

Please sign in to comment.