From cb46a61728e92a76d1f3f16fea230a5adc567283 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Mon, 29 Feb 2016 00:04:37 -0800 Subject: [PATCH] Linting --- README.md | 1 + panoramix/__init__.py | 3 +- panoramix/data/countries.py | 34 +++++------ panoramix/migrations/env.py | 1 - panoramix/models.py | 116 +++--------------------------------- panoramix/utils.py | 53 ++++++++-------- panoramix/viz.py | 33 +++++----- 7 files changed, 67 insertions(+), 174 deletions(-) diff --git a/README.md b/README.md index 0295d537d9de2..749c75b1a6109 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ Panoramix [![Join the chat at https://gitter.im/mistercrunch/panoramix](https://badges.gitter.im/mistercrunch/panoramix.svg)](https://gitter.im/mistercrunch/panoramix?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) ![img](https://travis-ci.org/mistercrunch/panoramix.svg?branch=master) [![Coverage Status](https://coveralls.io/repos/mistercrunch/panoramix/badge.svg?branch=master&service=github)](https://coveralls.io/github/mistercrunch/panoramix?branch=master) +[![Code Health](https://landscape.io/github/mistercrunch/panoramix/immune_to_filter/landscape.svg?style=flat)](https://landscape.io/github/mistercrunch/panoramix/immune_to_filter) Panoramix is a data exploration platform designed to be visual, intuitive and interactive. diff --git a/panoramix/__init__.py b/panoramix/__init__.py index 02b4820ec5b07..d90c5507711a9 100644 --- a/panoramix/__init__.py +++ b/panoramix/__init__.py @@ -4,7 +4,7 @@ from flask.ext.appbuilder import SQLA, AppBuilder, IndexView from flask.ext.appbuilder.baseviews import expose from flask.ext.migrate import Migrate -from panoramix import config +from panoramix import config, views APP_DIR = os.path.dirname(__file__) @@ -34,4 +34,3 @@ def index(self): sm = appbuilder.sm get_session = appbuilder.get_session -from panoramix import views diff --git a/panoramix/data/countries.py b/panoramix/data/countries.py index 4ef6d56226cd0..0a39d316b85ef 100644 --- a/panoramix/data/countries.py +++ b/panoramix/data/countries.py @@ -684,7 +684,7 @@ "area": 4167, "cioc": "", "cca2": "PF", - "capital": "Papeet\u0113", + "capital": u"Papeet\u0113", "lat": -15, "lng": -140, "cca3": "PYF" @@ -754,7 +754,7 @@ "area": 56785, "cioc": "TOG", "cca2": "TG", - "capital": "Lom\u00e9", + "capital": u"Lom\u00e9", "lat": 8, "lng": 1.16666666, "cca3": "TGO" @@ -774,7 +774,7 @@ "area": 549, "cioc": "GUM", "cca2": "GU", - "capital": "Hag\u00e5t\u00f1a", + "capital": u"Hag\u00e5t\u00f1a", "lat": 13.46666666, "lng": 144.78333333, "cca3": "GUM" @@ -834,7 +834,7 @@ "area": 51100, "cioc": "CRC", "cca2": "CR", - "capital": "San Jos\u00e9", + "capital": u"San Jos\u00e9", "lat": 10, "lng": -84, "cca3": "CRI" @@ -844,7 +844,7 @@ "area": 475442, "cioc": "CMR", "cca2": "CM", - "capital": "Yaound\u00e9", + "capital": u"Yaound\u00e9", "lat": 6, "lng": 12, "cca3": "CMR" @@ -1070,7 +1070,7 @@ "cca3": "BLR" }, { - "name": "Saint Barth\u00e9lemy", + "name": u"Saint Barth\u00e9lemy", "area": 21, "cioc": "", "cca2": "BL", @@ -1274,7 +1274,7 @@ "area": 7747, "cioc": "", "cca2": "TF", - "capital": "Port-aux-Fran\u00e7ais", + "capital": u"Port-aux-Fran\u00e7ais", "lat": -49.25, "lng": 69.167, "cca3": "ATF" @@ -1380,7 +1380,7 @@ "cca3": "PER" }, { - "name": "R\u00e9union", + "name": u"R\u00e9union", "area": 2511, "cioc": "", "cca2": "RE", @@ -1484,7 +1484,7 @@ "area": 1141748, "cioc": "COL", "cca2": "CO", - "capital": "Bogot\u00e1", + "capital": u"Bogot\u00e1", "lat": 4, "lng": -72, "cca3": "COL" @@ -1534,7 +1534,7 @@ "area": 33846, "cioc": "MDA", "cca2": "MD", - "capital": "Chi\u0219in\u0103u", + "capital": u"Chi\u0219in\u0103u", "lat": 47, "lng": 29, "cca3": "MDA" @@ -1594,7 +1594,7 @@ "area": 300, "cioc": "MDV", "cca2": "MV", - "capital": "Mal\u00e9", + "capital": u"Mal\u00e9", "lat": 3.25, "lng": 73, "cca3": "MDV" @@ -1620,7 +1620,7 @@ "cca3": "SPM" }, { - "name": "Cura\u00e7ao", + "name": u"Cura\u00e7ao", "area": 444, "cioc": "", "cca2": "CW", @@ -1704,7 +1704,7 @@ "area": 1393, "cioc": "", "cca2": "FO", - "capital": "T\u00f3rshavn", + "capital": u"T\u00f3rshavn", "lat": 62, "lng": -7, "cca3": "FRO" @@ -1860,7 +1860,7 @@ "cca3": "TUV" }, { - "name": "\u00c5land Islands", + "name": u"\u00c5land Islands", "area": 1580, "cioc": "", "cca2": "AX", @@ -1914,7 +1914,7 @@ "area": 8515767, "cioc": "BRA", "cca2": "BR", - "capital": "Bras\u00edlia", + "capital": u"Bras\u00edlia", "lat": -10, "lng": -55, "cca3": "BRA" @@ -2334,7 +2334,7 @@ "area": 266000, "cioc": "", "cca2": "EH", - "capital": "El Aai\u00fan", + "capital": u"El Aai\u00fan", "lat": 24.5, "lng": -13, "cca3": "ESH" @@ -2394,7 +2394,7 @@ "area": 18575, "cioc": "", "cca2": "NC", - "capital": "Noum\u00e9a", + "capital": u"Noum\u00e9a", "lat": -21.5, "lng": 165.5, "cca3": "NCL" diff --git a/panoramix/migrations/env.py b/panoramix/migrations/env.py index 83114cab076cc..e3713a3e49c36 100755 --- a/panoramix/migrations/env.py +++ b/panoramix/migrations/env.py @@ -3,7 +3,6 @@ from sqlalchemy import engine_from_config, pool from logging.config import fileConfig import logging -from panoramix import db, models from flask.ext.appbuilder import Base # this is the Alembic Config object, which provides diff --git a/panoramix/models.py b/panoramix/models.py index a09eed63d7eda..e5785f7a683ae 100644 --- a/panoramix/models.py +++ b/panoramix/models.py @@ -11,20 +11,20 @@ from flask import flash from flask.ext.appbuilder import Model from flask.ext.appbuilder.models.mixins import AuditMixin -from pandas import read_sql_query +import pandas as pd from pydruid import client from pydruid.utils.filters import Dimension, Filter + import sqlalchemy as sqla from sqlalchemy import ( Column, Integer, String, ForeignKey, Text, Boolean, DateTime, Table, create_engine, MetaData, desc, select, and_, func) +from sqlalchemy.engine import reflection from sqlalchemy.orm import relationship from sqlalchemy.sql import table, literal_column, text, column from sqlalchemy.sql.elements import ColumnClause from sqlalchemy_utils import EncryptedType - - from panoramix import app, db, get_session, utils from panoramix.viz import viz_types from sqlalchemy.ext.declarative import declared_attr @@ -154,12 +154,10 @@ def slice_link(self): @property def js_files(self): - from panoramix.viz import viz_types return viz_types[self.viz_type].js_files @property def css_files(self): - from panoramix.viz import viz_types return viz_types[self.viz_type].css_files def get_viz(self): @@ -264,10 +262,6 @@ def get_table(self, table_name): autoload_with=self.get_sqla_engine()) def get_columns(self, table_name): - - from sqlalchemy import create_engine - from sqlalchemy.engine import reflection - engine = self.get_sqla_engine() insp = reflection.Inspector.from_engine(engine) return insp.get_columns(table_name) @@ -340,7 +334,6 @@ def dttm_cols(self): @property def html(self): - import pandas as pd t = ((c.column_name, c.type) for c in self.columns) df = pd.DataFrame(t) df.columns = ['field', 'type'] @@ -367,101 +360,6 @@ def metrics_combo(self): for m in self.metrics], key=lambda x: x[1]) - def query_bkp( - self, groupby, metrics, - granularity, - from_dttm, to_dttm, - limit_spec=None, - filter=None, - is_timeseries=True, - timeseries_limit=15, - row_limit=None, - extras=None): # pragma: no cover - """ - Unused, legacy way of querying by building a SQL string without - using the sqlalchemy expression API (new approach which supports - all dialects) - """ - from pandas import read_sql_query - qry_start_dttm = datetime.now() - metrics_exprs = [ - "{} AS {}".format(m.expression, m.metric_name) - for m in self.metrics if m.metric_name in metrics] - from_dttm_iso = from_dttm.isoformat() - to_dttm_iso = to_dttm.isoformat() - - if metrics: - main_metric_expr = [ - m.expression for m in self.metrics - if m.metric_name == metrics[0]][0] - else: - main_metric_expr = "COUNT(*)" - - select_exprs = [] - groupby_exprs = [] - - if groupby: - select_exprs = copy(groupby) - groupby_exprs = [s for s in groupby] - inner_groupby_exprs = [s for s in groupby] - select_exprs += metrics_exprs - if granularity != "all": - select_exprs += ['ds as timestamp'] - groupby_exprs += ['ds'] - - select_exprs = ",\n".join(select_exprs) - groupby_exprs = ",\n".join(groupby_exprs) - - where_clause = [ - "ds >= '{from_dttm_iso}'", - "ds < '{to_dttm_iso}'" - ] - for col, op, eq in filter: - if op in ('in', 'not in'): - l = ["'{}'".format(s) for s in eq.split(",")] - l = ", ".join(l) - op = op.upper() - where_clause.append( - "{col} {op} ({l})".format(**locals()) - ) - where_clause = " AND\n".join(where_clause).format(**locals()) - on_clause = " AND ".join(["{g} = __{g}".format(g=g) for g in groupby]) - limiting_join = "" - if timeseries_limit and groupby: - inner_select = ", ".join([ - "{g} as __{g}".format(g=g) for g in inner_groupby_exprs]) - inner_groupby_exprs = ", ".join(inner_groupby_exprs) - limiting_join = ( - "JOIN ( \n" - " SELECT {inner_select} \n" - " FROM {self.table_name} \n" - " WHERE \n" - " {where_clause}\n" - " GROUP BY {inner_groupby_exprs}\n" - " ORDER BY {main_metric_expr} DESC\n" - " LIMIT {timeseries_limit}\n" - ") z ON {on_clause}\n" - ).format(**locals()) - - sql = ( - "SELECT\n" - " {select_exprs}\n" - "FROM {self.table_name}\n" - "{limiting_join}" - "WHERE\n" - " {where_clause}\n" - "GROUP BY\n" - " {groupby_exprs}\n" - ).format(**locals()) - df = read_sql_query( - sql=sql, - con=self.database.get_sqla_engine() - ) - textwrap.dedent(sql) - - return QueryResult( - df=df, duration=datetime.now() - qry_start_dttm, query=sql) - @property def sql_url(self): return self.database.sql_url + "?table_name=" + str(self.table_name) @@ -596,7 +494,7 @@ def query( engine = self.database.get_sqla_engine() sql = "{}".format( qry.compile(engine, compile_kwargs={"literal_binds": True})) - df = read_sql_query( + df = pd.read_sql_query( sql=sql, con=engine ) @@ -1026,8 +924,8 @@ def query( if ( not is_timeseries and - granularity == "all" - and 'timestamp' in df.columns): + granularity == "all" and + 'timestamp' in df.columns): del df['timestamp'] # Reordering columns @@ -1075,7 +973,7 @@ class DruidMetric(Model): def json_obj(self): try: obj = json.loads(self.json) - except: + except Exception: obj = {} return obj diff --git a/panoramix/utils.py b/panoramix/utils.py index 96a6d658f66d9..feac05044185c 100644 --- a/panoramix/utils.py +++ b/panoramix/utils.py @@ -1,4 +1,4 @@ -from datetime import datetime, date, timedelta +from datetime import datetime import functools import hashlib import json @@ -12,32 +12,31 @@ from panoramix import db - class memoized(object): - """Decorator that caches a function's return value each time it is called. - If called later with the same arguments, the cached value is returned, and - not re-evaluated. - """ - def __init__(self, func): - self.func = func - self.cache = {} - def __call__(self, *args): - try: - return self.cache[args] - except KeyError: - value = self.func(*args) - self.cache[args] = value - return value - except TypeError: - # uncachable -- for instance, passing a list as an argument. - # Better to not cache than to blow up entirely. - return self.func(*args) - def __repr__(self): - """Return the function's docstring.""" - return self.func.__doc__ - def __get__(self, obj, objtype): - """Support instance methods.""" - return functools.partial(self.__call__, obj) + """Decorator that caches a function's return value each time it is called. + If called later with the same arguments, the cached value is returned, and + not re-evaluated. + """ + def __init__(self, func): + self.func = func + self.cache = {} + def __call__(self, *args): + try: + return self.cache[args] + except KeyError: + value = self.func(*args) + self.cache[args] = value + return value + except TypeError: + # uncachable -- for instance, passing a list as an argument. + # Better to not cache than to blow up entirely. + return self.func(*args) + def __repr__(self): + """Return the function's docstring.""" + return self.func.__doc__ + def __get__(self, obj, objtype): + """Support instance methods.""" + return functools.partial(self.__call__, obj) def parse_human_datetime(s): @@ -187,7 +186,7 @@ def init(): table_perms += [ table.perm for table in session.query(models.DruidDatasource).all()] for table_perm in table_perms: - merge_perm(sm, 'datasource_access', table.perm) + merge_perm(sm, 'datasource_access', table_perm) def log_this(f): diff --git a/panoramix/viz.py b/panoramix/viz.py index 14b4c46188a1d..e719704e8ac46 100644 --- a/panoramix/viz.py +++ b/panoramix/viz.py @@ -8,7 +8,6 @@ from pandas.io.json import dumps from werkzeug.datastructures import ImmutableMultiDict from werkzeug.urls import Href -import numpy as np import pandas as pd from panoramix import app, utils @@ -160,7 +159,7 @@ def query_filters(self): extra_filters = form_data.get('extra_filters', []) if extra_filters: extra_filters = json.loads(extra_filters) - for slice_id, slice_filters in extra_filters.items(): + for slice_filters in extra_filters.values(): if slice_filters: for col, vals in slice_filters: if col and vals: @@ -296,8 +295,8 @@ def query_obj(self): d['groupby'] = [] return d - def get_df(self): - df = super(TableViz, self).get_df() + def get_df(self, query_obj=None): + df = super(TableViz, self).get_df(query_obj) if ( self.form_data.get("granularity") == "all" and 'timestamp' in df): @@ -353,8 +352,8 @@ def query_obj(self): d['groupby'] = list(set(groupby) | set(columns)) return d - def get_df(self): - df = super(PivotTableViz, self).get_df() + def get_df(self, query_obj=None): + df = super(PivotTableViz, self).get_df(query_obj) if ( self.form_data.get("granularity") == "all" and 'timestamp' in df): @@ -486,8 +485,8 @@ def query_obj(self): raise Exception("Pick a metric for x, y and size") return d - def get_df(self): - df = super(BubbleViz, self).get_df() + def get_df(self, query_obj=None): + df = super(BubbleViz, self).get_df(query_obj) df = df.fillna(0) df['x'] = df[[self.x_metric]] df['y'] = df[[self.y_metric]] @@ -673,9 +672,7 @@ def to_series(self, df, classed='', title_suffix=''): d = { "key": series_title, "classed": classed, - "values": [ - {'x': ds, 'y': ys[ds]} - for i, ds in enumerate(df.timestamp)] + "values": [{'x': ds, 'y': ys[ds]} for ds in df.timestamp], } chart_data.append(d) return chart_data @@ -759,8 +756,8 @@ def query_obj(self): d['is_timeseries'] = False return d - def get_df(self): - df = super(DistributionPieViz, self).get_df() + def get_df(self, query_obj=None): + df = super(DistributionPieViz, self).get_df(query_obj) df = df.pivot_table( index=self.groupby, values=[self.metrics[0]]) @@ -820,8 +817,8 @@ def query_obj(self): raise Exception("Pick at least one field for [Series]") return d - def get_df(self): - df = super(DistributionPieViz, self).get_df() + def get_df(self, query_obj=None): + df = super(DistributionPieViz, self).get_df(query_obj) fd = self.form_data row = df.groupby(self.groupby).sum()[self.metrics[0]].copy() @@ -892,8 +889,8 @@ class SunburstViz(BaseViz): }, } - def get_df(self): - df = super(SunburstViz, self).get_df() + def get_df(self, query_obj=None): + df = super(SunburstViz, self).get_df(query_obj) return df def get_json_data(self): @@ -1101,7 +1098,7 @@ def query_obj(self): self.form_data['metric']] return qry - def get_df(self): + def get_df(self, query_obj=None): qry = self.query_obj() filters = [g for g in qry['groupby']]