Skip to content
This repository has been archived by the owner on Dec 11, 2022. It is now read-only.

Commit

Permalink
Handle time range filtering issue for both partitioned and non-partit…
Browse files Browse the repository at this point in the history 
…ioned (fixes #321, #325, #334, #362)
  • Loading branch information
@ofir5300
ofir5300 committed Nov 21, 2021
1 parent 370b61a commit 08d8282
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 22 deletions.
137 changes: 122 additions & 15 deletions src/bigquery_query.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,19 +378,44 @@ export default class BigQueryQuery {
return query;
}

// public getPartitionFieldType() { // TODO - impklement if possible
// // const sqlWithNoVariables = this.templateSrv.replace(tmpQ, options.scopedVars, this.interpolateVariable);
// const [project, dataset, table] = BigQueryDatasource._extractFromClause(this.target.from);
// const datasource = new BigQueryDatasource()
// this.getDateFields(project, dataset, table)
// .then(dateFields => {
// const tm = BigQueryDatasource._FindTimeField(tmpQ, dateFields);
// this.queryModel.target.timeColumn = tm.text;
// this.queryModel.target.timeColumnType = tm.value;
// this.queryModel.target.table = table;
// })
// .catch((err) => {
// console.log(err);
// });
// }

// Detects either date or timestamp, only if not comment out
static hasDateFilter(whereClause) {
return whereClause.match(/((?<!--.*)([1-2]\d{3}-[0-1]\d-[0-3]\d)|([\D](\d{13})[\D]).*\n)/gi);
}

public buildWhereClause() {
let query = '';
let query = '', hasMacro = false, hasDateFilter = false;
const conditions = _.map(this.target.where, (tag, index) => {
switch (tag.type) {
case 'macro':
hasMacro = true;
return tag.name + '(' + this.target.timeColumn + ')';
case 'expression':
return tag.params.join(' ');
const expression = tag.params.join(' ');
hasDateFilter = BigQueryQuery.hasDateFilter(expression) ? true : hasDateFilter;
return expression;
}
});
const hasTimeFilter = !!(hasMacro || hasDateFilter);
if (this.target.partitioned) {
const partitionedField = this.target.partitionedField ? this.target.partitionedField : '_PARTITIONTIME';
if (this.target.timeColumn !== partitionedField) {
if (this.target.timeColumn !== partitionedField && !hasTimeFilter) {
if (this.templateSrv.timeRange && this.templateSrv.timeRange.from) {
const from = `${partitionedField} >= '${BigQueryQuery.formatDateToString(
this.templateSrv.timeRange.from._d,
Expand Down Expand Up @@ -536,11 +561,12 @@ export default class BigQueryQuery {
public expend_macros(options) {
if (this.target.rawSql) {
let q = this.target.rawSql;
q = BigQueryQuery.replaceTimeShift(q);
q = this.replaceTimeFilters(q, options);
q = this.replacetimeGroupAlias(q, true, options);
q = this.replacetimeGroupAlias(q, false, options);
return q;
let hasTimeFilter, hasTimeGroup, hasTimeGroupAlias = false;
q = BigQueryQuery.replaceTimeShift(q); //
[q, hasTimeFilter] = this.replaceTimeFilters(q, options);
[q, hasTimeGroup] = this.replacetimeGroupAlias(q, true, options);
[q, hasTimeGroupAlias] = this.replacetimeGroupAlias(q, false, options);
return [q, hasTimeFilter || hasTimeGroup || hasTimeGroupAlias];
}
}
public replaceTimeFilters(q, options) {
Expand All @@ -564,26 +590,27 @@ export default class BigQueryQuery {
const range = BigQueryQuery.quoteFiledName(this.target.timeColumn) + ' BETWEEN ' + from + ' AND ' + to;
const fromRange = BigQueryQuery.quoteFiledName(this.target.timeColumn) + ' > ' + from + ' ';
const toRange = BigQueryQuery.quoteFiledName(this.target.timeColumn) + ' < ' + to + ' ';
const hasMacro = q.match(/(\b__timeFilter\b)|(\b__timeFrom\b)|(\b__timeTo\b)|(\b__millisTimeTo\b)|(\b__millisTimeFrom\b)/g)
q = q.replace(/\$__timeFilter\((.*?)\)/g, range);
q = q.replace(/\$__timeFrom\(([\w_.]+)\)/g, fromRange);
q = q.replace(/\$__timeTo\(([\w_.]+)\)/g, toRange);
q = q.replace(/\$__millisTimeTo\(([\w_.]+)\)/g, to);
q = q.replace(/\$__millisTimeFrom\(([\w_.]+)\)/g, from);
return q;
return [q, hasMacro];
}

public replacetimeGroupAlias(q, alias: boolean, options) {
const res = BigQueryQuery._getInterval(q, alias);
const interval = res[0];
const mininterval = res[1];
if (!interval) {
return q;
return [q, false];
}
const intervalStr = this.getIntervalStr(interval, mininterval, options);
if (alias) {
return q.replace(/\$__timeGroupAlias\(([\w_.]+,+[a-zA-Z0-9_ ]+.*\))/g, intervalStr);
return [q.replace(/\$__timeGroupAlias\(([\w_.]+,+[a-zA-Z0-9_ ]+.*\))/g, intervalStr), q.match(/(\b__timeGroupAlias\b)/g)];
} else {
return q.replace(/\$__timeGroup\(([\w_.]+,+[a-zA-Z0-9_ ]+.*\))/g, intervalStr);
return [q.replace(/\$__timeGroup\(([\w_.]+,+[a-zA-Z0-9_ ]+.*\))/g, intervalStr), q.match(/(\b__timeGroup\b)/g)];
}
}

Expand All @@ -599,11 +626,91 @@ export default class BigQueryQuery {
}
private _getDateRangePart(part) {
if (this.target.timeColumnType === 'DATE') {
return "'" + BigQueryQuery.formatDateToString(part, '-') + "'";
return "'" + BigQueryQuery.formatDateToString(part, '-') + "'"; // to = "'2021-01-31'"
} else if (this.target.timeColumnType === 'DATETIME') {
return "'" + BigQueryQuery.formatDateToString(part, '-', true) + "'";
return "'" + BigQueryQuery.formatDateToString(part, '-', true) + "'"; // "'2021-01-31 19:41:45'"
} else {
return 'TIMESTAMP_MILLIS (' + part.valueOf().toString() + ')';
return 'TIMESTAMP_MILLIS (' + part.valueOf().toString() + ')'; // "TIMESTAMP_MILLIS (1612056873199)"
}
}
}




/*
//TODO remove before commit
// public getPartitionFieldType() { // TODO - impklement if possible
// // const sqlWithNoVariables = this.templateSrv.replace(tmpQ, options.scopedVars, this.interpolateVariable);
// const [project, dataset, table] = BigQueryDatasource._extractFromClause(this.target.from);
// const datasource = new BigQueryDatasource()
// this.getDateFields(project, dataset, table)
// .then(dateFields => {
// const tm = BigQueryDatasource._FindTimeField(tmpQ, dateFields);
// this.queryModel.target.timeColumn = tm.text;
// this.queryModel.target.timeColumnType = tm.value;
// this.queryModel.target.table = table;
// })
// .catch((err) => {
// console.log(err);
// });
// }
// Detects either date or timestamp, only if not comment out
static hasDateFilter(whereClause) {
return whereClause.match(/((?<!--.*)([1-2]\d{3}-[0-1]\d-[0-3]\d)|([\D](\d{13})[\D]).*\n)/gi);
}
public buildWhereClause() {
let query = '', hasMacro = false, hasDateFilter = false;
const conditions = _.map(this.target.where, (tag, index) => {
switch (tag.type) {
case 'macro':
hasMacro = true;
return tag.name + '(' + this.target.timeColumn + ')';
case 'expression':
const expression = tag.params.join(' ');
hasDateFilter = BigQueryQuery.hasDateFilter(expression) ? true : hasDateFilter;
return expression;
}
});
const hasTimeFilter = !!(hasMacro || hasDateFilter);
if (this.target.partitioned) {
const partitionedField = this.target.partitionedField ? this.target.partitionedField : '_PARTITIONTIME';
if (this.target.timeColumn !== partitionedField && !hasTimeFilter) {
if (this.templateSrv.timeRange && this.templateSrv.timeRange.from) {
const from = `${partitionedField} >= ${this.target.timeColumnType}('${BigQueryQuery.formatDateToString(
// const from = `${partitionedField} >= TIMESTAMP('${BigQueryQuery.formatDateToString(
this.templateSrv.timeRange.from._d,
'-',
true
)}')`;
conditions.push(from);
}
if (this.templateSrv.timeRange && this.templateSrv.timeRange.to) {
// const to = `${partitionedField} < '${BigQueryQuery.formatDateToString(
const to = `${partitionedField} < TIMESTAMP('${BigQueryQuery.formatDateToString(
this.templateSrv.timeRange.to._d,
'-',
true
)}')`;
conditions.push(to);
}
}
}
if (this.target.sharded) {
const from = BigQueryQuery.formatDateToString(this.templateSrv.timeRange.from._d);
const to = BigQueryQuery.formatDateToString(this.templateSrv.timeRange.to._d);
const sharded = "_TABLE_SUFFIX BETWEEN '" + from + "' AND '" + to + "' ";
conditions.push(sharded);
}
if (conditions.length > 0) {
query = '\nWHERE\n ' + conditions.join(' AND\n ');
}
return query;
}
*/
15 changes: 8 additions & 7 deletions src/datasource.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ export class BigQueryDatasource {
});
this.queryModel.target.rawSql = query.rawSql;
modOptions = BigQueryDatasource._setupTimeShiftQuery(query, options);
const q = this.setUpQ(modOptions, options, query);
const q = this.setUpQ(modOptions, options, query); // TODO hanle raw sql WHERE clause!
console.log(q);
return this.doQuery(q, options.panelId + query.refId, query.queryPriority).then((response) => {
return ResponseParser.parseDataQuery(response, query.format);
Expand Down Expand Up @@ -426,7 +426,7 @@ export class BigQueryDatasource {
refId: options.annotation.name,
};
this.queryModel.target.rawSql = query.rawSql;
query.rawSql = this.queryModel.expend_macros(options);
[query.rawSql,] = this.queryModel.expend_macros(options);
return this.backendSrv
.datasourceRequest({
data: {
Expand All @@ -444,9 +444,9 @@ export class BigQueryDatasource {
.then((data) => this.responseParser.transformAnnotationResponse(options, data));
}
private setUpQ(modOptions, options, query) {
let q = this.queryModel.expend_macros(modOptions);
let [q, hasMacro] = this.queryModel.expend_macros(modOptions);
if (q) {
q = this.setUpPartition(q, query.partitioned, query.partitionedField, modOptions);
q = this.setUpPartition(q, query.partitioned, query.partitionedField, modOptions, hasMacro);
q = BigQueryDatasource._updatePartition(q, modOptions);
q = BigQueryDatasource._updateTableSuffix(q, modOptions);
if (query.refId.search(Shifted) > -1) {
Expand All @@ -466,15 +466,16 @@ export class BigQueryDatasource {
return q;
}
/**
* Add partition to query unless it has one
* Add partition to query unless it has one OR already being ranged by other condition
* @param query
* @param isPartitioned
* @param partitionedField
* @param options
*/
private setUpPartition(query, isPartitioned, partitionedField, options) {
private setUpPartition(query, isPartitioned, partitionedField, options, hasMacro = false) {
partitionedField = partitionedField ? partitionedField : '_PARTITIONTIME';
if (isPartitioned && !query.match(new RegExp(partitionedField, "i"))) {
const hasTimeFilter = !!(BigQueryQuery.hasDateFilter(query.split(/where/gi)[1] || "") || hasMacro);
if (isPartitioned && !hasTimeFilter && !query.match(new RegExp(partitionedField, "i"))) {
const fromD = BigQueryQuery.convertToUtc(options.range.from._d);
const toD = BigQueryQuery.convertToUtc(options.range.to._d);
const from = `${partitionedField} >= '${BigQueryQuery.formatDateToString(fromD, '-', true)}'`;
Expand Down

0 comments on commit 08d8282

Please sign in to comment.