diff --git a/.pylintrc b/.pylintrc index ad87fcc21c076..4117d685bd3db 100644 --- a/.pylintrc +++ b/.pylintrc @@ -99,7 +99,7 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme [BASIC] # Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_,d,e,v,o,l,x +good-names=i,j,k,ex,Run,_,d,e,v,o,l,x,ts # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata diff --git a/superset/__init__.py b/superset/__init__.py index 5f324f6e946b2..71cd4549f8261 100644 --- a/superset/__init__.py +++ b/superset/__init__.py @@ -14,6 +14,7 @@ from flask_appbuilder import SQLA, AppBuilder, IndexView from flask_appbuilder.baseviews import expose from flask_migrate import Migrate +from flask_wtf.csrf import CSRFProtect from werkzeug.contrib.fixers import ProxyFix from superset.connectors.connector_registry import ConnectorRegistry @@ -50,6 +51,8 @@ db = SQLA(app) +if conf.get('WTF_CSRF_ENABLED'): + csrf = CSRFProtect(app) utils.pessimistic_connection_handling(db.engine.pool) diff --git a/superset/assets/javascripts/SqlLab/actions.js b/superset/assets/javascripts/SqlLab/actions.js index 3a3f64e15d31f..ca12ac8be0c31 100644 --- a/superset/assets/javascripts/SqlLab/actions.js +++ b/superset/assets/javascripts/SqlLab/actions.js @@ -1,3 +1,4 @@ +/* global notify */ import shortid from 'shortid'; import { now } from '../modules/dates'; const $ = require('jquery'); @@ -33,11 +34,25 @@ export const QUERY_FAILED = 'QUERY_FAILED'; export const CLEAR_QUERY_RESULTS = 'CLEAR_QUERY_RESULTS'; export const REMOVE_DATA_PREVIEW = 'REMOVE_DATA_PREVIEW'; export const CHANGE_DATA_PREVIEW_ID = 'CHANGE_DATA_PREVIEW_ID'; +export const SAVE_QUERY = 'SAVE_QUERY'; export function resetState() { return { type: RESET_STATE }; } +export function saveQuery(query) { + const url = '/savedqueryviewapi/api/create'; + $.ajax({ + type: 'POST', + url, + data: query, + success: () => notify.success('Your query was saved'), + error: () => notify.error('Your query could not be saved'), + dataType: 'json', + }); + return { type: SAVE_QUERY }; +} + export function startQuery(query) { Object.assign(query, { id: query.id ? query.id : shortid.generate(), @@ -328,6 +343,27 @@ export function popStoredQuery(urlId) { }; dispatch(addQueryEditor(queryEditorProps)); }, + error: () => notify.error("The query couldn't be loaded"), + }); + }; +} +export function popSavedQuery(saveQueryId) { + return function (dispatch) { + $.ajax({ + type: 'GET', + url: `/savedqueryviewapi/api/get/${saveQueryId}`, + success: (data) => { + const sq = data.result; + const queryEditorProps = { + title: sq.label, + dbId: sq.db_id, + schema: sq.schema, + autorun: false, + sql: sq.sql, + }; + dispatch(addQueryEditor(queryEditorProps)); + }, + error: () => notify.error("The query couldn't be loaded"), }); }; } diff --git a/superset/assets/javascripts/SqlLab/components/Alerts.jsx b/superset/assets/javascripts/SqlLab/components/Alerts.jsx deleted file mode 100644 index c635b5ae5aab3..0000000000000 --- a/superset/assets/javascripts/SqlLab/components/Alerts.jsx +++ /dev/null @@ -1,34 +0,0 @@ -import React from 'react'; -import { Alert } from 'react-bootstrap'; - -class Alerts extends React.PureComponent { - removeAlert(alert) { - this.props.actions.removeAlert(alert); - } - render() { - const alerts = this.props.alerts.map((alert) => - - {alert.msg} - - - ); - return ( -
{alerts}
- ); - } -} - -Alerts.propTypes = { - alerts: React.PropTypes.array, - actions: React.PropTypes.object, -}; - -export default Alerts; diff --git a/superset/assets/javascripts/SqlLab/components/AlertsWrapper.jsx b/superset/assets/javascripts/SqlLab/components/AlertsWrapper.jsx new file mode 100644 index 0000000000000..529c2a10244ee --- /dev/null +++ b/superset/assets/javascripts/SqlLab/components/AlertsWrapper.jsx @@ -0,0 +1,18 @@ +import React from 'react'; +import AlertContainer from 'react-alert'; + +export default class AlertsWrapper extends React.PureComponent { + render() { + return ( + { + global.notify = ref; + }} + offset={14} + position="top right" + theme="dark" + time={5000} + transition="fade" + />); + } +} diff --git a/superset/assets/javascripts/SqlLab/components/App.jsx b/superset/assets/javascripts/SqlLab/components/App.jsx index 7c97f004832e0..81736acd7f8c4 100644 --- a/superset/assets/javascripts/SqlLab/components/App.jsx +++ b/superset/assets/javascripts/SqlLab/components/App.jsx @@ -5,7 +5,7 @@ import React from 'react'; import TabbedSqlEditors from './TabbedSqlEditors'; import QueryAutoRefresh from './QueryAutoRefresh'; import QuerySearch from './QuerySearch'; -import Alerts from './Alerts'; +import AlertsWrapper from './AlertsWrapper'; import { bindActionCreators } from 'redux'; import { connect } from 'react-redux'; @@ -64,7 +64,7 @@ class App extends React.PureComponent { } return (
- +
{content}
diff --git a/superset/assets/javascripts/SqlLab/components/RunQueryActionButton.jsx b/superset/assets/javascripts/SqlLab/components/RunQueryActionButton.jsx index 9d0865c136092..579fc487ff114 100644 --- a/superset/assets/javascripts/SqlLab/components/RunQueryActionButton.jsx +++ b/superset/assets/javascripts/SqlLab/components/RunQueryActionButton.jsx @@ -31,7 +31,7 @@ export default function RunQueryActionButton(props) { onClick={() => props.runQuery(false)} key="run-btn" > - {runBtnText} + {runBtnText} ); diff --git a/superset/assets/javascripts/SqlLab/components/SaveQuery.jsx b/superset/assets/javascripts/SqlLab/components/SaveQuery.jsx new file mode 100644 index 0000000000000..e32d23a4d4e6e --- /dev/null +++ b/superset/assets/javascripts/SqlLab/components/SaveQuery.jsx @@ -0,0 +1,131 @@ +/* global notify */ +import React from 'react'; +import { FormControl, FormGroup, Overlay, Popover, Row, Col } from 'react-bootstrap'; +import Button from '../../components/Button'; + +const propTypes = { + defaultLabel: React.PropTypes.string, + sql: React.PropTypes.string, + schema: React.PropTypes.string, + dbId: React.PropTypes.number, + animation: React.PropTypes.bool, + onSave: React.PropTypes.func, +}; +const defaultProps = { + defaultLabel: 'Undefined', + animation: true, + onSave: () => {}, +}; + +class SaveQuery extends React.PureComponent { + constructor(props) { + super(props); + this.state = { + description: '', + label: props.defaultLabel, + showSave: false, + }; + this.toggleSave = this.toggleSave.bind(this); + this.onSave = this.onSave.bind(this); + this.onCancel = this.onCancel.bind(this); + this.onLabelChange = this.onLabelChange.bind(this); + this.onDescriptionChange = this.onDescriptionChange.bind(this); + } + onSave() { + const query = { + label: this.state.label, + description: this.state.description, + db_id: this.props.dbId, + schema: this.props.schema, + sql: this.props.sql, + }; + this.props.onSave(query); + this.setState({ showSave: false }); + } + onCancel() { + this.setState({ showSave: false }); + } + onLabelChange(e) { + this.setState({ label: e.target.value }); + } + onDescriptionChange(e) { + this.setState({ description: e.target.value }); + } + renderPopover() { + return ( + + + + + + + + + + +
+ + + + + + + + +
+ + + + + + + + + ); + } + toggleSave(e) { + this.setState({ target: e.target, showSave: !this.state.showSave }); + } + render() { + return ( + + + {this.renderPopover()} + + + + ); + } +} +SaveQuery.propTypes = propTypes; +SaveQuery.defaultProps = defaultProps; + +export default SaveQuery; diff --git a/superset/assets/javascripts/SqlLab/components/SqlEditor.jsx b/superset/assets/javascripts/SqlLab/components/SqlEditor.jsx index 86ca358a9d149..e1537d59ae526 100644 --- a/superset/assets/javascripts/SqlLab/components/SqlEditor.jsx +++ b/superset/assets/javascripts/SqlLab/components/SqlEditor.jsx @@ -15,6 +15,7 @@ import { import Button from '../../components/Button'; import SouthPane from './SouthPane'; +import SaveQuery from './SaveQuery'; import Timer from '../../components/Timer'; import SqlEditorLeftBar from './SqlEditorLeftBar'; import AceEditorWrapper from './AceEditorWrapper'; @@ -101,6 +102,7 @@ class SqlEditor extends React.PureComponent { } render() { + const qe = this.props.queryEditor; let limitWarning = null; if (this.props.latestQuery && this.props.latestQuery.limit_reached) { const tooltip = ( @@ -149,12 +151,19 @@ class SqlEditor extends React.PureComponent {
+ {ctasControls}
diff --git a/superset/assets/javascripts/SqlLab/components/TabbedSqlEditors.jsx b/superset/assets/javascripts/SqlLab/components/TabbedSqlEditors.jsx index a6eea4bceba27..984f9ea98390d 100644 --- a/superset/assets/javascripts/SqlLab/components/TabbedSqlEditors.jsx +++ b/superset/assets/javascripts/SqlLab/components/TabbedSqlEditors.jsx @@ -6,7 +6,7 @@ import * as Actions from '../actions'; import SqlEditor from './SqlEditor'; import CopyQueryTabUrl from './CopyQueryTabUrl'; import { areArraysShallowEqual } from '../../reduxUtils'; -import { getParamFromQuery } from '../../../utils/common'; +import URI from 'urijs'; const propTypes = { actions: React.PropTypes.object.isRequired, @@ -35,19 +35,19 @@ class TabbedSqlEditors extends React.PureComponent { }; } componentDidMount() { - const search = window.location.search; - if (search) { - const queryString = search.substring(1); - const urlId = getParamFromQuery(queryString, 'id'); - if (urlId) { - this.props.actions.popStoredQuery(urlId); - } else { - let dbId = getParamFromQuery(queryString, 'dbid'); + const query = URI(window.location).search(true); + if (query.id || query.sql || query.savedQueryId) { + if (query.id) { + this.props.actions.popStoredQuery(query.id); + } else if (query.savedQueryId) { + this.props.actions.popSavedQuery(query.savedQueryId); + } else if (query.sql) { + let dbId = query.dbid; if (dbId) { dbId = parseInt(dbId, 10); } else { const databases = this.props.databases; - const dbName = getParamFromQuery(queryString, 'dbname'); + const dbName = query.dbname; if (dbName) { Object.keys(databases).forEach((db) => { if (databases[db].database_name === dbName) { @@ -57,11 +57,11 @@ class TabbedSqlEditors extends React.PureComponent { } } const newQueryEditor = { - title: getParamFromQuery(queryString, 'title'), + title: query.title, dbId, - schema: getParamFromQuery(queryString, 'schema'), - autorun: getParamFromQuery(queryString, 'autorun'), - sql: getParamFromQuery(queryString, 'sql'), + schema: query.schema, + autorun: query.autorun, + sql: query.sql, }; this.props.actions.addQueryEditor(newQueryEditor); } diff --git a/superset/assets/javascripts/SqlLab/index.jsx b/superset/assets/javascripts/SqlLab/index.jsx index b58d3fc16030e..4a78952745798 100644 --- a/superset/assets/javascripts/SqlLab/index.jsx +++ b/superset/assets/javascripts/SqlLab/index.jsx @@ -6,6 +6,7 @@ import React from 'react'; import { render } from 'react-dom'; import { getInitialState, sqlLabReducer } from './reducers'; import { initEnhancer } from '../reduxUtils'; +import { initJQueryAjaxCSRF } from '../modules/utils'; import { createStore, compose, applyMiddleware } from 'redux'; import { Provider } from 'react-redux'; import thunkMiddleware from 'redux-thunk'; @@ -14,6 +15,7 @@ import App from './components/App'; require('./main.css'); +initJQueryAjaxCSRF(); const appContainer = document.getElementById('app'); const bootstrapData = JSON.parse(appContainer.getAttribute('data-bootstrap')); diff --git a/superset/assets/javascripts/index.jsx b/superset/assets/javascripts/index.jsx deleted file mode 100644 index 89cee75e736c9..0000000000000 --- a/superset/assets/javascripts/index.jsx +++ /dev/null @@ -1,14 +0,0 @@ -import React from 'react'; -import { render } from 'react-dom'; -import { Jumbotron } from 'react-bootstrap'; - -function App() { - return ( - -

Superset

-

Extensible visualization tool for exploring data from any database.

- - ); -} - -render(, document.getElementById('app')); diff --git a/superset/assets/javascripts/modules/utils.js b/superset/assets/javascripts/modules/utils.js index d518b45524732..92bc931e4f612 100644 --- a/superset/assets/javascripts/modules/utils.js +++ b/superset/assets/javascripts/modules/utils.js @@ -196,3 +196,18 @@ export function getTextWidth(text, fontDetails) { const metrics = context.measureText(text); return metrics.width; } + +export function initJQueryAjaxCSRF() { + // Works in conjunction with a Flask-WTF token as described here: + // http://flask-wtf.readthedocs.io/en/stable/csrf.html#javascript-requests + const token = $('input#csrf_token').val(); + if (token) { + $.ajaxSetup({ + beforeSend(xhr, settings) { + if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) { + xhr.setRequestHeader('X-CSRFToken', token); + } + }, + }); + } +} diff --git a/superset/assets/package.json b/superset/assets/package.json index 420e4248f2db7..d1e313138782f 100644 --- a/superset/assets/package.json +++ b/superset/assets/package.json @@ -66,7 +66,9 @@ "nvd3": "1.8.5", "react": "^15.3.2", "react-ace": "^4.1.5", + "react-addons-css-transition-group": "^15.4.2", "react-addons-shallow-compare": "^15.4.2", + "react-alert": "^1.0.14", "react-bootstrap": "^0.30.3", "react-bootstrap-table": "^2.3.8", "react-dom": "^15.3.2", @@ -88,6 +90,7 @@ "style-loader": "^0.13.0", "supercluster": "https://github.com/georgeke/supercluster/tarball/ac3492737e7ce98e07af679623aad452373bbc40", "topojson": "^1.6.22", + "urijs": "^1.18.10", "victory": "^0.17.0", "viewport-mercator-project": "^2.1.0" }, diff --git a/superset/assets/spec/javascripts/sqllab/AlertsWrapper_spec.jsx b/superset/assets/spec/javascripts/sqllab/AlertsWrapper_spec.jsx new file mode 100644 index 0000000000000..064fe8cf116df --- /dev/null +++ b/superset/assets/spec/javascripts/sqllab/AlertsWrapper_spec.jsx @@ -0,0 +1,11 @@ +import React from 'react'; +import AlertsWrapper from '../../../javascripts/SqlLab/components/AlertsWrapper'; +import { describe, it } from 'mocha'; +import { expect } from 'chai'; + + +describe('AlertsWrapper', () => { + it('is valid', () => { + expect(React.isValidElement()).to.equal(true); + }); +}); diff --git a/superset/assets/spec/javascripts/sqllab/Alerts_spec.jsx b/superset/assets/spec/javascripts/sqllab/Alerts_spec.jsx deleted file mode 100644 index 19f456f765d26..0000000000000 --- a/superset/assets/spec/javascripts/sqllab/Alerts_spec.jsx +++ /dev/null @@ -1,24 +0,0 @@ -import React from 'react'; -import Alerts from '../../../javascripts/SqlLab/components/Alerts'; -import { Alert } from 'react-bootstrap'; -import { shallow } from 'enzyme'; -import { describe, it } from 'mocha'; -import { expect } from 'chai'; -import { alert } from './fixtures'; - - -describe('Alerts', () => { - const mockedProps = { - alerts: [alert], - }; - it('is valid', () => { - expect(React.isValidElement()).to.equal(true); - }); - it('is valid with props', () => { - expect(React.isValidElement()).to.equal(true); - }); - it('renders an Alert', () => { - const wrapper = shallow(); - expect(wrapper.find(Alert)).to.have.length(1); - }); -}); diff --git a/superset/assets/spec/javascripts/sqllab/SaveQuery_spec.jsx b/superset/assets/spec/javascripts/sqllab/SaveQuery_spec.jsx new file mode 100644 index 0000000000000..43371e6fc4776 --- /dev/null +++ b/superset/assets/spec/javascripts/sqllab/SaveQuery_spec.jsx @@ -0,0 +1,48 @@ +import React from 'react'; +import SaveQuery from '../../../javascripts/SqlLab/components/SaveQuery'; +import { Overlay, Popover, FormControl } from 'react-bootstrap'; +import { shallow, mount } from 'enzyme'; +import { describe, it } from 'mocha'; +import { expect } from 'chai'; + + +describe('SavedQuery', () => { + const mockedProps = { + dbId: 1, + schema: 'main', + sql: 'SELECT * FROM t', + defaultLabel: 'untitled', + animation: false, + }; + it('is valid', () => { + expect( + React.isValidElement() + ).to.equal(true); + }); + it('is valid with props', () => { + expect( + React.isValidElement() + ).to.equal(true); + }); + it('has an Overlay and a Popover', () => { + const wrapper = shallow(); + expect(wrapper.find(Overlay)).to.have.length(1); + expect(wrapper.find(Popover)).to.have.length(1); + }); + it('pops and hides', () => { + const wrapper = mount(); + expect(wrapper.state().showSave).to.equal(false); + wrapper.find('.toggleSave').simulate('click'); + expect(wrapper.state().showSave).to.equal(true); + wrapper.find('.toggleSave').simulate('click'); + expect(wrapper.state().showSave).to.equal(false); + }); + it('has a cancel button', () => { + const wrapper = shallow(); + expect(wrapper.find('.cancelQuery')).to.have.length(1); + }); + it('has 2 FormControls', () => { + const wrapper = shallow(); + expect(wrapper.find(FormControl)).to.have.length(2); + }); +}); diff --git a/superset/config.py b/superset/config.py index 32041bb982a88..2e981921df980 100644 --- a/superset/config.py +++ b/superset/config.py @@ -54,7 +54,7 @@ QUERY_SEARCH_LIMIT = 1000 # Flask-WTF flag for CSRF -CSRF_ENABLED = True +WTF_CSRF_ENABLED = True # Whether to run the web server in debug mode or not DEBUG = False diff --git a/superset/migrations/versions/2fcdcb35e487_saved_queries.py b/superset/migrations/versions/2fcdcb35e487_saved_queries.py new file mode 100644 index 0000000000000..43aa277c55055 --- /dev/null +++ b/superset/migrations/versions/2fcdcb35e487_saved_queries.py @@ -0,0 +1,40 @@ +"""saved_queries + +Revision ID: 2fcdcb35e487 +Revises: a6c18f869a4e +Create Date: 2017-03-29 15:04:35.734190 + +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '2fcdcb35e487' +down_revision = 'a6c18f869a4e' + + +def upgrade(): + op.create_table( + 'saved_query', + sa.Column('created_on', sa.DateTime(), nullable=True), + sa.Column('changed_on', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('db_id', sa.Integer(), nullable=True), + sa.Column('label', sa.String(256), nullable=True), + sa.Column('schema', sa.String(128), nullable=True), + sa.Column('sql', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('changed_by_fk', sa.Integer(), nullable=True), + sa.Column('created_by_fk', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['changed_by_fk'], ['ab_user.id'], ), + sa.ForeignKeyConstraint(['created_by_fk'], ['ab_user.id'], ), + sa.ForeignKeyConstraint(['user_id'], [u'ab_user.id'], ), + sa.ForeignKeyConstraint(['db_id'], [u'dbs.id'], ), + sa.PrimaryKeyConstraint('id') + ) + + +def downgrade(): + op.drop_table('saved_query') diff --git a/superset/models/__init__.py b/superset/models/__init__.py index 2a1dbbc06ff9e..bed8c30ede7c1 100644 --- a/superset/models/__init__.py +++ b/superset/models/__init__.py @@ -1 +1,2 @@ from . import core # noqa +from . import sql_lab # noqa diff --git a/superset/models/core.py b/superset/models/core.py index 184eb72fe98da..e1a372268310e 100644 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -9,7 +9,6 @@ import logging import numpy import pickle -import re import textwrap from future.standard_library import install_aliases from copy import copy @@ -26,10 +25,10 @@ from sqlalchemy import ( Column, Integer, String, ForeignKey, Text, Boolean, - DateTime, Date, Table, Numeric, + DateTime, Date, Table, create_engine, MetaData, select ) -from sqlalchemy.orm import backref, relationship +from sqlalchemy.orm import relationship from sqlalchemy.orm.session import make_transient from sqlalchemy.sql import text from sqlalchemy.sql.expression import TextAsFrom @@ -797,105 +796,6 @@ class FavStar(Model): dttm = Column(DateTime, default=datetime.utcnow) -class Query(Model): - - """ORM model for SQL query""" - - __tablename__ = 'query' - id = Column(Integer, primary_key=True) - client_id = Column(String(11), unique=True, nullable=False) - - database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False) - - # Store the tmp table into the DB only if the user asks for it. - tmp_table_name = Column(String(256)) - user_id = Column( - Integer, ForeignKey('ab_user.id'), nullable=True) - status = Column(String(16), default=QueryStatus.PENDING) - tab_name = Column(String(256)) - sql_editor_id = Column(String(256)) - schema = Column(String(256)) - sql = Column(Text) - # Query to retrieve the results, - # used only in case of select_as_cta_used is true. - select_sql = Column(Text) - executed_sql = Column(Text) - # Could be configured in the superset config. - limit = Column(Integer) - limit_used = Column(Boolean, default=False) - select_as_cta = Column(Boolean) - select_as_cta_used = Column(Boolean, default=False) - - progress = Column(Integer, default=0) # 1..100 - # # of rows in the result set or rows modified. - rows = Column(Integer) - error_message = Column(Text) - # key used to store the results in the results backend - results_key = Column(String(64), index=True) - - # Using Numeric in place of DateTime for sub-second precision - # stored as seconds since epoch, allowing for milliseconds - start_time = Column(Numeric(precision=3)) - start_running_time = Column(Numeric(precision=3)) - end_time = Column(Numeric(precision=3)) - changed_on = Column( - DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=True) - - database = relationship( - 'Database', - foreign_keys=[database_id], - backref=backref('queries', cascade='all, delete-orphan') - ) - user = relationship( - 'User', - backref=backref('queries', cascade='all, delete-orphan'), - foreign_keys=[user_id]) - - __table_args__ = ( - sqla.Index('ti_user_id_changed_on', user_id, changed_on), - ) - - @property - def limit_reached(self): - return self.rows == self.limit if self.limit_used else False - - def to_dict(self): - return { - 'changedOn': self.changed_on, - 'changed_on': self.changed_on.isoformat(), - 'dbId': self.database_id, - 'db': self.database.database_name, - 'endDttm': self.end_time, - 'errorMessage': self.error_message, - 'executedSql': self.executed_sql, - 'id': self.client_id, - 'limit': self.limit, - 'progress': self.progress, - 'rows': self.rows, - 'schema': self.schema, - 'ctas': self.select_as_cta, - 'serverId': self.id, - 'sql': self.sql, - 'sqlEditorId': self.sql_editor_id, - 'startDttm': self.start_time, - 'state': self.status.lower(), - 'tab': self.tab_name, - 'tempTable': self.tmp_table_name, - 'userId': self.user_id, - 'user': self.user.username, - 'limit_reached': self.limit_reached, - 'resultsKey': self.results_key, - } - - @property - def name(self): - ts = datetime.now().isoformat() - ts = ts.replace('-', '').replace(':', '').split('.')[0] - tab = self.tab_name.replace(' ', '_').lower() if self.tab_name else 'notab' - tab = re.sub(r'\W+', '', tab) - return "sqllab_{tab}_{ts}".format(**locals()) - - class DatasourceAccessRequest(Model, AuditMixinNullable): """ORM model for the access requests for datasources and dbs.""" __tablename__ = 'access_request' diff --git a/superset/models/sql_lab.py b/superset/models/sql_lab.py new file mode 100644 index 0000000000000..18c67f4561249 --- /dev/null +++ b/superset/models/sql_lab.py @@ -0,0 +1,165 @@ +"""A collection of ORM sqlalchemy models for SQL Lab""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import re +from datetime import datetime + +from future.standard_library import install_aliases + +from flask import Markup + +from flask_appbuilder import Model + +import sqlalchemy as sqla +from sqlalchemy import ( + Column, Integer, String, ForeignKey, Text, Boolean, + DateTime, Numeric, +) +from sqlalchemy.orm import backref, relationship + +from superset.utils import QueryStatus +from superset.models.helpers import AuditMixinNullable + +install_aliases() + + +class Query(Model): + + """ORM model for SQL query""" + + __tablename__ = 'query' + id = Column(Integer, primary_key=True) + client_id = Column(String(11), unique=True, nullable=False) + + database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False) + + # Store the tmp table into the DB only if the user asks for it. + tmp_table_name = Column(String(256)) + user_id = Column( + Integer, ForeignKey('ab_user.id'), nullable=True) + status = Column(String(16), default=QueryStatus.PENDING) + tab_name = Column(String(256)) + sql_editor_id = Column(String(256)) + schema = Column(String(256)) + sql = Column(Text) + # Query to retrieve the results, + # used only in case of select_as_cta_used is true. + select_sql = Column(Text) + executed_sql = Column(Text) + # Could be configured in the superset config. + limit = Column(Integer) + limit_used = Column(Boolean, default=False) + select_as_cta = Column(Boolean) + select_as_cta_used = Column(Boolean, default=False) + + progress = Column(Integer, default=0) # 1..100 + # # of rows in the result set or rows modified. + rows = Column(Integer) + error_message = Column(Text) + # key used to store the results in the results backend + results_key = Column(String(64), index=True) + + # Using Numeric in place of DateTime for sub-second precision + # stored as seconds since epoch, allowing for milliseconds + start_time = Column(Numeric(precision=3)) + start_running_time = Column(Numeric(precision=3)) + end_time = Column(Numeric(precision=3)) + changed_on = Column( + DateTime, + default=datetime.utcnow, + onupdate=datetime.utcnow, + nullable=True) + + database = relationship( + 'Database', + foreign_keys=[database_id], + backref=backref('queries', cascade='all, delete-orphan') + ) + user = relationship( + 'User', + foreign_keys=[user_id]) + + __table_args__ = ( + sqla.Index('ti_user_id_changed_on', user_id, changed_on), + ) + + @property + def limit_reached(self): + return self.rows == self.limit if self.limit_used else False + + def to_dict(self): + return { + 'changedOn': self.changed_on, + 'changed_on': self.changed_on.isoformat(), + 'dbId': self.database_id, + 'db': self.database.database_name, + 'endDttm': self.end_time, + 'errorMessage': self.error_message, + 'executedSql': self.executed_sql, + 'id': self.client_id, + 'limit': self.limit, + 'progress': self.progress, + 'rows': self.rows, + 'schema': self.schema, + 'ctas': self.select_as_cta, + 'serverId': self.id, + 'sql': self.sql, + 'sqlEditorId': self.sql_editor_id, + 'startDttm': self.start_time, + 'state': self.status.lower(), + 'tab': self.tab_name, + 'tempTable': self.tmp_table_name, + 'userId': self.user_id, + 'user': self.user.username, + 'limit_reached': self.limit_reached, + 'resultsKey': self.results_key, + } + + @property + def name(self): + """Name property""" + ts = datetime.now().isoformat() + ts = ts.replace('-', '').replace(':', '').split('.')[0] + tab = ( + self.tab_name.replace(' ', '_').lower() + if self.tab_name + else 'notab' + ) + tab = re.sub(r'\W+', '', tab) + return "sqllab_{tab}_{ts}".format(**locals()) + + +class SavedQuery(Model, AuditMixinNullable): + + """ORM model for SQL query""" + + __tablename__ = 'saved_query' + id = Column(Integer, primary_key=True) + user_id = Column( + Integer, ForeignKey('ab_user.id'), nullable=True) + db_id = Column( + Integer, ForeignKey('dbs.id'), nullable=True) + schema = Column(String(128)) + label = Column(String(256)) + description = Column(Text) + sql = Column(Text) + user = relationship( + 'User', + backref=backref('saved_queries', cascade='all, delete-orphan'), + foreign_keys=[user_id]) + database = relationship( + 'Database', + foreign_keys=[db_id], + backref=backref('saved_queries', cascade='all, delete-orphan') + ) + + @property + def pop_tab_link(self): + return Markup(""" + + + + """.format(**locals())) diff --git a/superset/sql_lab.py b/superset/sql_lab.py index 4d72032ea77d2..cc42ed96fb389 100644 --- a/superset/sql_lab.py +++ b/superset/sql_lab.py @@ -13,7 +13,7 @@ from superset import ( app, db, utils, dataframe, results_backend) -from superset.models import core as models +from superset.models.sql_lab import Query from superset.sql_parse import SupersetQuery from superset.db_engine_specs import LimitMethod from superset.jinja_context import get_template_processor @@ -56,14 +56,15 @@ def get_sql_results(self, query_id, return_results=True, store_results=False): session = db.session() session.commit() # HACK try: - query = session.query(models.Query).filter_by(id=query_id).one() + query = session.query(Query).filter_by(id=query_id).one() except Exception as e: - logging.error("Query with id `{}` could not be retrieved".format(query_id)) + logging.error( + "Query with id `{}` could not be retrieved".format(query_id)) logging.error("Sleeping for a sec and retrying...") # Nasty hack to get around a race condition where the worker # cannot find the query it's supposed to run sleep(1) - query = session.query(models.Query).filter_by(id=query_id).one() + query = session.query(Query).filter_by(id=query_id).one() database = query.database db_engine_spec = database.db_engine_spec diff --git a/superset/templates/superset/basic.html b/superset/templates/superset/basic.html index 73ba2879dbcab..d07fa665feaa3 100644 --- a/superset/templates/superset/basic.html +++ b/superset/templates/superset/basic.html @@ -1,3 +1,4 @@ +{% import 'appbuilder/general/lib.html' as lib %} @@ -37,6 +38,7 @@ <div id="app" data-bootstrap="{{ bootstrap_data }}" > <img src="/static/assets/images/loading.gif" style="width: 50px; margin: 10px;"> </div> + {{ csrf_token() if csrf_token else None }} {% endblock %} <!-- Modal for misc messages / alerts --> diff --git a/superset/views/__init__.py b/superset/views/__init__.py index 6a410e5e906f7..b964e8b14e316 100644 --- a/superset/views/__init__.py +++ b/superset/views/__init__.py @@ -1,2 +1,3 @@ from . import base # noqa from . import core # noqa +from . import sql_lab # noqa diff --git a/superset/views/base.py b/superset/views/base.py index 7c15d69493e9a..132ba82b8cb4f 100644 --- a/superset/views/base.py +++ b/superset/views/base.py @@ -1,7 +1,9 @@ -import logging +import functools import json +import logging +import traceback -from flask import g, redirect +from flask import g, redirect, Response from flask_babel import gettext as __ from flask_appbuilder import BaseView @@ -15,6 +17,42 @@ from superset.connectors.connector_registry import ConnectorRegistry +def get_error_msg(): + if conf.get("SHOW_STACKTRACE"): + error_msg = traceback.format_exc() + else: + error_msg = "FATAL ERROR \n" + error_msg += ( + "Stacktrace is hidden. Change the SHOW_STACKTRACE " + "configuration setting to enable it") + return error_msg + + +def json_error_response(msg, status=None, stacktrace=None): + data = {'error': msg} + if stacktrace: + data['stacktrace'] = stacktrace + status = status if status else 500 + return Response( + json.dumps(data), + status=status, mimetype="application/json") + + +def api(f): + """ + A decorator to label an endpoint as an API. Catches uncaught exceptions and + return the response in the JSON format + """ + def wraps(self, *args, **kwargs): + try: + return f(self, *args, **kwargs) + except Exception as e: + logging.exception(e) + return json_error_response(get_error_msg()) + + return functools.update_wrapper(wraps, f) + + def get_datasource_exist_error_mgs(full_name): return __("Datasource %(name)s already exists", name=full_name) diff --git a/superset/views/core.py b/superset/views/core.py index 25627c927d6f2..5f917c82c3f57 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -13,7 +13,6 @@ import traceback import zlib -import functools import sqlalchemy as sqla from flask import ( @@ -38,11 +37,12 @@ from superset.utils import has_access from superset.connectors.connector_registry import ConnectorRegistry import superset.models.core as models +from superset.models.sql_lab import Query from superset.sql_parse import SupersetQuery from .base import ( - SupersetModelView, BaseSupersetView, DeleteMixin, - SupersetFilter, get_user_roles + api, SupersetModelView, BaseSupersetView, DeleteMixin, + SupersetFilter, get_user_roles, json_error_response, get_error_msg ) config = app.config @@ -71,46 +71,10 @@ def get_datasource_access_error_msg(datasource_name): "`all_datasource_access` permission", name=datasource_name) -def get_error_msg(): - if config.get("SHOW_STACKTRACE"): - error_msg = traceback.format_exc() - else: - error_msg = "FATAL ERROR \n" - error_msg += ( - "Stacktrace is hidden. Change the SHOW_STACKTRACE " - "configuration setting to enable it") - return error_msg - - -def json_error_response(msg, status=None, stacktrace=None): - data = {'error': msg} - if stacktrace: - data['stacktrace'] = stacktrace - status = status if status else 500 - return Response( - json.dumps(data), - status=status, mimetype="application/json") - - def json_success(json_msg, status=200): return Response(json_msg, status=status, mimetype="application/json") -def api(f): - """ - A decorator to label an endpoint as an API. Catches uncaught exceptions and - return the response in the JSON format - """ - def wraps(self, *args, **kwargs): - try: - return f(self, *args, **kwargs) - except Exception as e: - logging.exception(e) - return json_error_response(get_error_msg()) - - return functools.update_wrapper(wraps, f) - - def is_owner(obj, user): """ Check if user is owner of the slice """ return obj and obj.owners and user in obj.owners @@ -566,19 +530,6 @@ class LogModelView(SupersetModelView): icon="fa-list-ol") -class QueryView(SupersetModelView): - datamodel = SQLAInterface(models.Query) - list_columns = ['user', 'database', 'status', 'start_time', 'end_time'] - -appbuilder.add_view( - QueryView, - "Queries", - label=__("Queries"), - category="Manage", - category_label=__("Manage"), - icon="fa-search") - - @app.route('/health') def health(): return "OK" @@ -1928,7 +1879,7 @@ def results(self, key): status=410 ) - query = db.session.query(models.Query).filter_by(results_key=key).one() + query = db.session.query(Query).filter_by(results_key=key).one() rejected_tables = self.rejected_datasources( query.sql, query.database, query.schema) if rejected_tables: @@ -1950,7 +1901,7 @@ def stop_query(self): client_id = request.form.get('client_id') try: query = ( - db.session.query(models.Query) + db.session.query(Query) .filter_by(client_id=client_id).one() ) query.status = utils.QueryStatus.STOPPED @@ -1990,7 +1941,7 @@ def sql_json(self): tmp_table_name ) - query = models.Query( + query = Query( database_id=int(database_id), limit=int(app.config.get('SQL_MAX_ROW', None)), sql=sql, @@ -2061,7 +2012,7 @@ def sql_json(self): def csv(self, client_id): """Download the query results as csv.""" query = ( - db.session.query(models.Query) + db.session.query(Query) .filter_by(client_id=client_id) .one() ) @@ -2126,10 +2077,10 @@ def queries(self, last_updated_ms): last_updated_dt = utils.EPOCH + timedelta(seconds=last_updated_ms_int / 1000) sql_queries = ( - db.session.query(models.Query) + db.session.query(Query) .filter( - models.Query.user_id == g.user.get_id(), - models.Query.changed_on >= last_updated_dt, + Query.user_id == g.user.get_id(), + Query.changed_on >= last_updated_dt, ) .all() ) @@ -2142,7 +2093,7 @@ def queries(self, last_updated_ms): @log_this def search_queries(self): """Search for queries.""" - query = db.session.query(models.Query) + query = db.session.query(Query) search_user_id = request.args.get('user_id') database_id = request.args.get('database_id') search_text = request.args.get('search_text') @@ -2153,30 +2104,30 @@ def search_queries(self): if search_user_id: # Filter on db Id - query = query.filter(models.Query.user_id == search_user_id) + query = query.filter(Query.user_id == search_user_id) if database_id: # Filter on db Id - query = query.filter(models.Query.database_id == database_id) + query = query.filter(Query.database_id == database_id) if status: # Filter on status - query = query.filter(models.Query.status == status) + query = query.filter(Query.status == status) if search_text: # Filter on search text query = query \ - .filter(models.Query.sql.like('%{}%'.format(search_text))) + .filter(Query.sql.like('%{}%'.format(search_text))) if from_time: - query = query.filter(models.Query.start_time > int(from_time)) + query = query.filter(Query.start_time > int(from_time)) if to_time: - query = query.filter(models.Query.start_time < int(to_time)) + query = query.filter(Query.start_time < int(to_time)) query_limit = config.get('QUERY_SEARCH_LIMIT', 1000) sql_queries = ( - query.order_by(models.Query.start_time.asc()) + query.order_by(Query.start_time.asc()) .limit(query_limit) .all() ) @@ -2255,8 +2206,11 @@ def sqllab(self): d = { 'defaultDbId': config.get('SQLLAB_DEFAULT_DBID'), } + from flask_wtf import FlaskForm + ff = FlaskForm() return self.render_template( 'superset/sqllab.html', + csrf_token=ff.csrf_token, bootstrap_data=json.dumps(d, default=utils.json_iso_dttm_ser) ) appbuilder.add_view_no_menu(Superset) diff --git a/superset/views/sql_lab.py b/superset/views/sql_lab.py new file mode 100644 index 0000000000000..3c868cf6e704a --- /dev/null +++ b/superset/views/sql_lab.py @@ -0,0 +1,69 @@ +from flask import redirect, g + +from flask_appbuilder import expose +from flask_appbuilder.models.sqla.interface import SQLAInterface + +from flask_babel import gettext as __ + +from superset import appbuilder +from superset.models.sql_lab import Query, SavedQuery +from .base import SupersetModelView, BaseSupersetView, DeleteMixin + + +class QueryView(SupersetModelView): + datamodel = SQLAInterface(Query) + list_columns = ['user', 'database', 'status', 'start_time', 'end_time'] + +appbuilder.add_view( + QueryView, + "Queries", + label=__("Queries"), + category="Manage", + category_label=__("Manage"), + icon="fa-search") + + +class SavedQueryView(SupersetModelView, DeleteMixin): + datamodel = SQLAInterface(SavedQuery) + list_columns = [ + 'label', 'user', 'database', 'schema', 'description', + 'modified', 'pop_tab_link'] + show_columns = [ + 'id', 'label', 'user', 'database', + 'description', 'sql', 'pop_tab_link'] + add_columns = ['label', 'database', 'description', 'sql'] + edit_columns = add_columns + base_order = ('changed_on', 'desc') + + def pre_add(self, obj): + obj.user = g.user + + def pre_update(self, obj): + self.pre_add(obj) + + +class SavedQueryViewApi(SavedQueryView): + show_columns = ['label', 'db_id', 'schema', 'description', 'sql'] + add_columns = show_columns + edit_columns = add_columns + +appbuilder.add_view_no_menu(SavedQueryViewApi) +appbuilder.add_view_no_menu(SavedQueryView) + +appbuilder.add_link( + __('Saved Queries'), + href='/sqllab/my_queries/', + icon="fa-save", + category='SQL Lab') + + +class SqlLab(BaseSupersetView): + """The base views for Superset!""" + @expose("/my_queries/") + def my_queries(self): + """Assigns a list of found users to the given role.""" + return redirect( + '/savedqueryview/list/?_flt_0_user={}'.format(g.user.id)) + + +appbuilder.add_view_no_menu(SqlLab) diff --git a/tests/celery_tests.py b/tests/celery_tests.py index 43e1b6f29ef82..a47984172787d 100644 --- a/tests/celery_tests.py +++ b/tests/celery_tests.py @@ -16,6 +16,7 @@ from superset import app, appbuilder, cli, db, dataframe from superset.models import core as models from superset.models.helpers import QueryStatus +from superset.models.sql_lab import Query from superset.security import sync_role_definitions from superset.sql_parse import SupersetQuery @@ -73,13 +74,13 @@ def __init__(self, *args, **kwargs): def get_query_by_name(self, sql): session = db.session - query = session.query(models.Query).filter_by(sql=sql).first() + query = session.query(Query).filter_by(sql=sql).first() session.close() return query def get_query_by_id(self, id): session = db.session - query = session.query(models.Query).filter_by(id=id).first() + query = session.query(Query).filter_by(id=id).first() session.close() return query diff --git a/tests/core_tests.py b/tests/core_tests.py index a6d35becc2fd4..d18f2d93900c5 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -16,6 +16,7 @@ from superset import db, utils, appbuilder, sm, jinja_context, sql_lab from superset.models import core as models +from superset.models.sql_lab import Query from superset.views.core import DatabaseView from superset.connectors.sqla.models import SqlaTable @@ -38,11 +39,11 @@ def setUpClass(cls): )} def setUp(self): - db.session.query(models.Query).delete() + db.session.query(Query).delete() db.session.query(models.DatasourceAccessRequest).delete() def tearDown(self): - db.session.query(models.Query).delete() + db.session.query(Query).delete() def test_login(self): resp = self.get_resp( diff --git a/tests/sqllab_tests.py b/tests/sqllab_tests.py index e1dac77103dbc..0cc00bafe4e35 100644 --- a/tests/sqllab_tests.py +++ b/tests/sqllab_tests.py @@ -10,7 +10,7 @@ from flask_appbuilder.security.sqla import models as ab_models from superset import db, utils, appbuilder, sm -from superset.models import core as models +from superset.models.sql_lab import Query from .base_tests import SupersetTestCase @@ -22,7 +22,7 @@ def __init__(self, *args, **kwargs): super(SqlLabTests, self).__init__(*args, **kwargs) def run_some_queries(self): - db.session.query(models.Query).delete() + db.session.query(Query).delete() db.session.commit() self.run_sql( "SELECT * FROM ab_user", @@ -39,7 +39,7 @@ def run_some_queries(self): self.logout() def tearDown(self): - db.session.query(models.Query).delete() + db.session.query(Query).delete() db.session.commit() self.logout() @@ -77,7 +77,7 @@ def test_sql_json_has_access(self): astronaut, password='general') data = self.run_sql('SELECT * FROM ab_user', "3", user_name='gagarin') - db.session.query(models.Query).delete() + db.session.query(Query).delete() db.session.commit() self.assertLess(0, len(data['data'])) @@ -102,7 +102,7 @@ def test_queries_endpoint(self): self.assertEquals(4, len(data)) now = datetime.now() + timedelta(days=1) - query = db.session.query(models.Query).filter_by( + query = db.session.query(Query).filter_by( sql='SELECT * FROM ab_user LIMIT 1').first() query.changed_on = now db.session.commit() @@ -176,11 +176,11 @@ def test_search_query_on_time(self): self.run_some_queries() self.login('admin') first_query_time = ( - db.session.query(models.Query) + db.session.query(Query) .filter_by(sql='SELECT * FROM ab_user').one() ).start_time second_query_time = ( - db.session.query(models.Query) + db.session.query(Query) .filter_by(sql='SELECT * FROM ab_permission').one() ).start_time # Test search queries on time filter