diff --git a/superset/__init__.py b/superset/__init__.py
index 3b503a48c84e2..f0450195a1cd4 100644
--- a/superset/__init__.py
+++ b/superset/__init__.py
@@ -108,6 +108,7 @@ def get_manifest():
# Setup the cache prior to registering the blueprints.
cache = setup_cache(app, conf.get("CACHE_CONFIG"))
tables_cache = setup_cache(app, conf.get("TABLE_NAMES_CACHE_CONFIG"))
+thumbnail_cache = setup_cache(app, conf.get("THUMBNAIL_CACHE_CONFIG"))
for bp in conf.get("BLUEPRINTS"):
try:
@@ -120,6 +121,10 @@ def get_manifest():
if conf.get("SILENCE_FAB"):
logging.getLogger("flask_appbuilder").setLevel(logging.ERROR)
+logging.getLogger("urllib3").setLevel(logging.ERROR)
+logging.getLogger("selenium").setLevel(logging.ERROR)
+logging.getLogger("PIL").setLevel(logging.ERROR)
+
if app.debug:
app.logger.setLevel(logging.DEBUG) # pylint: disable=no-member
else:
diff --git a/superset/assets/stylesheets/less/cosmo/variables.less b/superset/assets/stylesheets/less/cosmo/variables.less
index 7b979c2245251..dc61b52cbb13e 100644
--- a/superset/assets/stylesheets/less/cosmo/variables.less
+++ b/superset/assets/stylesheets/less/cosmo/variables.less
@@ -29,8 +29,8 @@
@gray-darker: lighten(@gray-base, 13.5%);
@gray-dark: lighten(@gray-base, 20%);
@gray: lighten(@gray-base, 33.5%);
-@gray-light: lighten(@gray-base, 70%);
-@gray-lighter: lighten(@gray-base, 95%);
+@gray-light: lighten(@gray-base, 80%);
+@gray-lighter: lighten(@gray-base, 90%);
@brand-primary: #00A699;
@brand-success: #4AC15F;
diff --git a/superset/cli.py b/superset/cli.py
index 572122086b7a9..1d2db3f83742e 100755
--- a/superset/cli.py
+++ b/superset/cli.py
@@ -423,6 +423,75 @@ def load_test_users():
load_test_users_run()
+@app.cli.command()
+@click.option(
+ "--asynchronous",
+ "-a",
+ is_flag=True,
+ default=False,
+ help="Trigger commands to run remotely on a worker",
+)
+@click.option(
+ "--dashboards_only",
+ "-d",
+ is_flag=True,
+ default=False,
+ help="Only process dashboards",
+)
+@click.option(
+ "--charts_only", "-c", is_flag=True, default=False, help="Only process charts"
+)
+@click.option(
+ "--force",
+ "-f",
+ is_flag=True,
+ default=False,
+ help="Force refresh, even if previously cached",
+)
+@click.option("--id", "-i", multiple=True)
+def compute_thumbnails(asynchronous, dashboards_only, charts_only, force, id):
+ """Compute thumbnails"""
+ from superset.models import core as models
+ from superset.tasks.thumbnails import (
+ cache_chart_thumbnail,
+ cache_dashboard_thumbnail,
+ )
+
+ if not charts_only:
+ query = db.session.query(models.Dashboard)
+ if id:
+ query = query.filter(models.Dashboard.id.in_(id))
+ dashboards = query.all()
+ count = len(dashboards)
+ for i, dash in enumerate(dashboards):
+ if asynchronous:
+ func = cache_dashboard_thumbnail.delay
+ action = "Triggering"
+ else:
+ func = cache_dashboard_thumbnail
+ action = "Processing"
+ msg = f'{action} dashboard "{dash.dashboard_title}" ({i+1}/{count})'
+ click.secho(msg, fg="green")
+ func(dash.id, force=force)
+
+ if not dashboards_only:
+ query = db.session.query(models.Slice)
+ if id:
+ query = query.filter(models.Slice.id.in_(id))
+ slices = query.all()
+ count = len(slices)
+ for i, slc in enumerate(slices):
+ if asynchronous:
+ func = cache_chart_thumbnail.delay
+ action = "Triggering"
+ else:
+ func = cache_chart_thumbnail
+ action = "Processing"
+ msg = f'{action} chart "{slc.slice_name}" ({i+1}/{count})'
+ click.secho(msg, fg="green")
+ func(slc.id, force=force)
+
+
def load_test_users_run():
"""
Loads admin, alpha, and gamma user for testing purposes
diff --git a/superset/config.py b/superset/config.py
index bcdd0f63c6805..d3f72467de992 100644
--- a/superset/config.py
+++ b/superset/config.py
@@ -380,6 +380,8 @@
# you'll want to use a proper broker as specified here:
# http://docs.celeryproject.org/en/latest/getting-started/brokers/index.html
+CELERYD_LOG_LEVEL = "DEBUG"
+
class CeleryConfig(object):
BROKER_URL = "sqla+sqlite:///celerydb.sqlite"
diff --git a/superset/connectors/base/models.py b/superset/connectors/base/models.py
index da7fec5d6a78f..5050ac18e6937 100644
--- a/superset/connectors/base/models.py
+++ b/superset/connectors/base/models.py
@@ -157,6 +157,16 @@ def short_data(self):
def select_star(self):
pass
+ @property
+ def data_summary(self):
+ return {
+ "datasource_name": self.datasource_name,
+ "type": self.type,
+ "schema": self.schema,
+ "id": self.id,
+ "explore_url": self.explore_url,
+ }
+
@property
def data(self):
"""Data representation of the datasource sent to the frontend"""
diff --git a/superset/models/core.py b/superset/models/core.py
index e3dcafa0a701b..c2972c114cf02 100755
--- a/superset/models/core.py
+++ b/superset/models/core.py
@@ -58,6 +58,7 @@
from superset.models.helpers import AuditMixinNullable, ImportMixin
from superset.models.tags import ChartUpdater, DashboardUpdater, FavStarUpdater
from superset.models.user_attributes import UserAttribute
+from superset.tasks.thumbnails import cache_dashboard_thumbnail
from superset.utils import cache as cache_util, core as utils
from superset.viz import viz_types
from urllib import parse # noqa
@@ -187,6 +188,26 @@ def cls_model(self):
def datasource(self):
return self.get_datasource
+ @property
+ def thumbnail_url(self):
+ # SHA here is to force bypassing the browser cache when chart has changed
+ sha = utils.md5_hex(self.params, 6)
+ return f"/thumb/chart/{self.id}/{sha}/"
+
+ @property
+ def thumbnail_img(self):
+ return Markup(f'
')
+
+ @property
+ def thumbnail_link(self):
+ return Markup(
+ f"""
+
+ {self.thumbnail_img}
+
+ """
+ )
+
def clone(self):
return Slice(
slice_name=self.slice_name,
@@ -711,6 +732,34 @@ def export_dashboards(cls, dashboard_ids):
indent=4,
)
+ @property
+ def thumbnail_url(self):
+ # SHA here is to force bypassing the browser cache when chart has changed
+ sha = utils.md5_hex(self.position_json, 6)
+ return f"/thumb/dashboard/{self.id}/{sha}/"
+
+ @property
+ def thumbnail_img(self):
+ return Markup(f'')
+
+ @property
+ def thumbnail_link(self):
+ return Markup(
+ f"""
+
+ {self.thumbnail_img}
+
+ """
+ )
+
+
+def event_after_dashboard_changed(mapper, connection, target):
+ cache_dashboard_thumbnail.delay(target.id, force=True)
+
+
+sqla.event.listen(Dashboard, "before_insert", event_after_dashboard_changed)
+sqla.event.listen(Dashboard, "before_update", event_after_dashboard_changed)
+
class Database(Model, AuditMixinNullable, ImportMixin):
diff --git a/superset/tasks/cache.py b/superset/tasks/cache.py
index 136de81e27190..a2da1483b740c 100644
--- a/superset/tasks/cache.py
+++ b/superset/tasks/cache.py
@@ -25,7 +25,6 @@
from sqlalchemy import and_, func
from superset import app, db
-from superset.models.core import Dashboard, Log, Slice
from superset.models.tags import Tag, TaggedObject
from superset.tasks.celery_app import app as celery_app
from superset.utils.core import parse_human_datetime
@@ -132,6 +131,8 @@ class DummyStrategy(Strategy):
def get_urls(self):
session = db.create_scoped_session()
+ from superset.models.core import Slice
+
charts = session.query(Slice).all()
return [get_url(chart) for chart in charts]
@@ -166,6 +167,8 @@ def get_urls(self):
urls = []
session = db.create_scoped_session()
+ from superset.models.core import Dashboard, Log
+
records = (
session.query(Log.dashboard_id, func.count(Log.dashboard_id))
.filter(and_(Log.dashboard_id.isnot(None), Log.dttm >= self.since))
@@ -223,6 +226,8 @@ def get_urls(self):
)
.all()
)
+ from superset.models.core import Dashboard, Slice
+
dash_ids = [tagged_object.object_id for tagged_object in tagged_objects]
tagged_dashboards = session.query(Dashboard).filter(Dashboard.id.in_(dash_ids))
for dashboard in tagged_dashboards:
diff --git a/superset/tasks/schedules.py b/superset/tasks/schedules.py
index e8c7d6746af8c..36b8fbe99303f 100644
--- a/superset/tasks/schedules.py
+++ b/superset/tasks/schedules.py
@@ -37,7 +37,6 @@
import simplejson as json
from werkzeug.utils import parse_cookie
-# Superset framework imports
from superset import app, db, security_manager
from superset.models.schedules import (
EmailDeliveryType,
@@ -46,7 +45,7 @@
SliceEmailReportFormat,
)
from superset.tasks.celery_app import app as celery_app
-from superset.utils.core import get_email_address_list, send_email_smtp
+from superset.utils import core as utils
# Globals
config = app.config
@@ -66,13 +65,13 @@ def _get_recipients(schedule):
to = schedule.recipients
yield (to, bcc)
else:
- for to in get_email_address_list(schedule.recipients):
+ for to in utils.get_email_address_list(schedule.recipients):
yield (to, bcc)
def _deliver_email(schedule, subject, email):
for (to, bcc) in _get_recipients(schedule):
- send_email_smtp(
+ utils.send_email_smtp(
to,
subject,
email.body,
@@ -85,8 +84,9 @@ def _deliver_email(schedule, subject, email):
)
-def _generate_mail_content(schedule, screenshot, name, url):
- if schedule.delivery_type == EmailDeliveryType.attachment:
+def _generate_mail_content(delivery_type, screenshot, name, url):
+ config = app.config
+ if delivery_type == EmailDeliveryType.attachment:
images = None
data = {"screenshot.png": screenshot}
body = __(
@@ -94,7 +94,9 @@ def _generate_mail_content(schedule, screenshot, name, url):
name=name,
url=url,
)
- elif schedule.delivery_type == EmailDeliveryType.inline:
+ else:
+ # Implicit: delivery_type == EmailDeliveryType.inline:
+
# Get the domain from the 'From' address ..
# and make a message id without the < > in the ends
domain = parseaddr(config.get("SMTP_MAIL_FROM"))[1].split("@")[1]
@@ -239,13 +241,10 @@ def deliver_dashboard(schedule):
prefix=config.get("EMAIL_REPORTS_SUBJECT_PREFIX"),
title=dashboard.dashboard_title,
)
+ _deliver_email(_get_recipients(schedule), subject, email)
- _deliver_email(schedule, subject, email)
-
-
-def _get_slice_data(schedule):
- slc = schedule.slice
+def _get_slice_data(slc, delivery_type):
slice_url = _get_url_path(
"Superset.explore_json", csv="true", form_data=json.dumps({"slice_id": slc.id})
)
@@ -266,7 +265,7 @@ def _get_slice_data(schedule):
# TODO: Move to the csv module
rows = [r.split(b",") for r in response.content.splitlines()]
- if schedule.delivery_type == EmailDeliveryType.inline:
+ if delivery_type == EmailDeliveryType.inline:
data = None
# Parse the csv file and generate HTML
@@ -280,7 +279,7 @@ def _get_slice_data(schedule):
link=url,
)
- elif schedule.delivery_type == EmailDeliveryType.attachment:
+ elif delivery_type == EmailDeliveryType.attachment:
data = {__("%(name)s.csv", name=slc.slice_name): response.content}
body = __(
'Explore in Superset',
@@ -326,24 +325,25 @@ def _get_slice_visualization(schedule):
return _generate_mail_content(schedule, screenshot, slc.slice_name, slice_url)
-def deliver_slice(schedule):
+def deliver_slice(slc, recipients, email_format, delivery_type):
"""
Given a schedule, delivery the slice as an email report
"""
- if schedule.email_format == SliceEmailReportFormat.data:
- email = _get_slice_data(schedule)
- elif schedule.email_format == SliceEmailReportFormat.visualization:
- email = _get_slice_visualization(schedule)
+ config = app.config
+ if email_format == SliceEmailReportFormat.data:
+ email = _get_slice_data(slc, delivery_type)
+ elif email_format == SliceEmailReportFormat.visualization:
+ email = _get_slice_visualization(slc, delivery_type)
else:
raise RuntimeError("Unknown email report format")
subject = __(
"%(prefix)s %(title)s",
prefix=config.get("EMAIL_REPORTS_SUBJECT_PREFIX"),
- title=schedule.slice.slice_name,
+ title=slc.slice_name,
)
- _deliver_email(schedule, subject, email)
+ _deliver_email(recipients, subject, email)
@celery_app.task(name="email_reports.send", bind=True, soft_time_limit=300)
@@ -362,9 +362,16 @@ def schedule_email_report(task, report_type, schedule_id, recipients=None):
schedule.recipients = recipients
if report_type == ScheduleType.dashboard.value:
- deliver_dashboard(schedule)
+ deliver_dashboard(
+ schedule.dashboard, _get_recipients(schedule), schedule.delivery_type
+ )
elif report_type == ScheduleType.slice.value:
- deliver_slice(schedule)
+ deliver_slice(
+ schedule.slice,
+ _get_recipients(schedule),
+ schedule.email_format,
+ schedule.delivery_type,
+ )
else:
raise RuntimeError("Unknown report type")
@@ -412,6 +419,7 @@ def schedule_window(report_type, start_at, stop_at, resolution):
@celery_app.task(name="email_reports.schedule_hourly")
def schedule_hourly():
""" Celery beat job meant to be invoked hourly """
+ config = app.config
if not config.get("ENABLE_SCHEDULED_EMAIL_REPORTS"):
logging.info("Scheduled email reports not enabled in config")
diff --git a/superset/tasks/thumbnails.py b/superset/tasks/thumbnails.py
new file mode 100644
index 0000000000000..48ac2b59df8ca
--- /dev/null
+++ b/superset/tasks/thumbnails.py
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=C,R,W
+
+"""Utility functions used across Superset"""
+
+import logging
+
+from superset import app, security_manager, thumbnail_cache
+from superset.tasks.celery_app import app as celery_app
+from superset.utils.selenium import DashboardScreenshot, SliceScreenshot
+
+
+@celery_app.task(name="cache_chart_thumbnail", soft_time_limit=300)
+def cache_chart_thumbnail(chart_id, force=False):
+ with app.app_context():
+ logging.info(f"Caching chart {chart_id}")
+ screenshot = SliceScreenshot(id=chart_id)
+ user = security_manager.find_user("Admin")
+ screenshot.compute_and_cache(user=user, cache=thumbnail_cache, force=force)
+
+
+@celery_app.task(name="cache_dashboard_thumbnail", soft_time_limit=300)
+def cache_dashboard_thumbnail(dashboard_id, force=False):
+ with app.app_context():
+ logging.info(f"Caching dashboard {dashboard_id}")
+ screenshot = DashboardScreenshot(id=dashboard_id)
+ user = security_manager.find_user("Admin")
+ screenshot.compute_and_cache(user=user, cache=thumbnail_cache, force=force)
diff --git a/superset/utils/core.py b/superset/utils/core.py
index 69c70f575c381..4ed35439baf11 100644
--- a/superset/utils/core.py
+++ b/superset/utils/core.py
@@ -25,6 +25,7 @@
from email.utils import formatdate
import errno
import functools
+import hashlib
import json
import logging
import os
diff --git a/superset/utils/selenium.py b/superset/utils/selenium.py
new file mode 100644
index 0000000000000..cd192bc267822
--- /dev/null
+++ b/superset/utils/selenium.py
@@ -0,0 +1,275 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=C,R,W
+from io import BytesIO
+import logging
+import time
+import urllib
+
+from flask import current_app, request, Response, session, url_for
+from flask_login import login_user
+from PIL import Image
+from retry.api import retry_call
+from selenium.common.exceptions import TimeoutException, WebDriverException
+from selenium.webdriver import chrome, firefox
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+from werkzeug.utils import parse_cookie
+
+# Time in seconds, we will wait for the page to load and render
+SELENIUM_CHECK_INTERVAL = 2
+SELENIUM_RETRIES = 5
+SELENIUM_HEADSTART = 3
+
+
+def headless_url(path):
+ return urllib.parse.urljoin(current_app.config.get("WEBDRIVER_BASEURL"), path)
+
+
+def get_url_path(view, **kwargs):
+ with current_app.test_request_context():
+ return headless_url(url_for(view, **kwargs))
+
+
+class BaseScreenshot:
+ thumbnail_type = None
+ orm_class = None
+ element = None
+ window_size = (800, 600)
+ thumb_size = (400, 300)
+
+ def __init__(self, id):
+ self.id = id
+ self.screenshot = None
+
+ @property
+ def cache_key(self):
+ return f"thumb__{self.thumbnail_type}__{self.id}"
+
+ @property
+ def url(self):
+ raise NotImplementedError()
+
+ def fetch_screenshot(self, user, window_size=None):
+ window_size = window_size or self.window_size
+ self.screenshot = get_png_from_url(
+ self.url, window_size, self.element, user=user
+ )
+ return self.screenshot
+
+ def get_thumb_as_bytes(self, *args, **kwargs):
+ payload = self.get_thumb(*args, **kwargs)
+ return BytesIO(payload)
+
+ def get_from_cache(self, cache):
+ payload = cache.get(self.cache_key)
+ if payload:
+ return BytesIO(payload)
+
+ def compute_and_cache(
+ self, user, cache, window_size=None, thumb_size=None, force=True
+ ):
+ cache_key = self.cache_key
+ if not force and cache.get(cache_key):
+ logging.info("Thumb already cached, skipping...")
+ return
+ window_size = window_size or self.window_size
+ thumb_size = thumb_size or self.thumb_size
+ logging.info(f"Processing url for thumbnail: {cache_key}")
+
+ payload = None
+
+ # Assuming all sorts of things can go wrong with Selenium
+ try:
+ payload = self.fetch_screenshot(window_size=window_size, user=user)
+ except Exception as e:
+ logging.error("Failed at generating thumbnail")
+ logging.exception(e)
+
+ if payload and window_size != thumb_size:
+ try:
+ payload = self.resize_image(payload, size=thumb_size)
+ except Exception as e:
+ logging.error("Failed at resizing thumbnail")
+ logging.exception(e)
+ payload = None
+
+ if payload and cache:
+ logging.info(f"Caching thumbnail: {cache_key}")
+ cache.set(cache_key, payload)
+
+ return payload
+
+ def get_thumb(self, user, window_size=None, thumb_size=None, cache=None):
+ payload = None
+ cache_key = self.cache_key
+ window_size = window_size or self.window_size
+ thumb_size = thumb_size or self.thumb_size
+ if cache:
+ payload = cache.get(cache_key)
+ if not payload:
+ payload = self.compute_and_cache(user, cache, window_size, thumb_size)
+ else:
+ logging.info(f"Loaded thumbnail from cache: {cache_key}")
+ return payload
+
+ @classmethod
+ def resize_image(cls, img_bytes, output="png", size=None, crop=True):
+ size = size or cls.thumb_size
+ img = Image.open(BytesIO(img_bytes))
+ logging.debug(f"Selenium image size: {img.size}")
+ if crop and img.size[1] != cls.window_size[1]:
+ desired_ratio = float(cls.window_size[1]) / cls.window_size[0]
+ desired_width = int(img.size[0] * desired_ratio)
+ logging.debug(f"Cropping to: {img.size[0]}*{desired_width}")
+ img = img.crop((0, 0, img.size[0], desired_width))
+ logging.debug(f"Resizing to {size}")
+ img = img.resize(size, Image.ANTIALIAS)
+ new_img = BytesIO()
+ if output != "png":
+ img = img.convert("RGB")
+ img.save(new_img, output)
+ new_img.seek(0)
+ return new_img.read()
+
+
+class SliceScreenshot(BaseScreenshot):
+ thumbnail_type = "slice"
+ element = "chart-container"
+ window_size = (600, int(600 * 0.75))
+ thumb_size = (300, int(300 * 0.75))
+
+ @property
+ def url(self):
+ return get_url_path("Superset.slice", slice_id=self.id, standalone="true")
+
+
+class DashboardScreenshot(BaseScreenshot):
+ thumbnail_type = "dashboard"
+ element = "grid-container"
+ window_size = (1600, int(1600 * 0.75))
+ thumb_size = (400, int(400 * 0.75))
+
+ @property
+ def url(self):
+ return get_url_path("Superset.dashboard", dashboard_id=self.id)
+
+
+def _destroy_webdriver(driver):
+ """Destroy a driver"""
+ # This is some very flaky code in selenium. Hence the retries
+ # and catch-all exceptions
+ try:
+ retry_call(driver.close, tries=2)
+ except Exception:
+ pass
+ try:
+ driver.quit()
+ except Exception:
+ pass
+
+
+def get_auth_cookies(user):
+ # Login with the user specified to get the reports
+ with current_app.test_request_context():
+ login_user(user)
+
+ # A mock response object to get the cookie information from
+ response = Response()
+ current_app.session_interface.save_session(current_app, session, response)
+
+ cookies = []
+
+ # Set the cookies in the driver
+ for name, value in response.headers:
+ if name.lower() == "set-cookie":
+ cookie = parse_cookie(value)
+ cookies.append(cookie["session"])
+ return cookies
+
+
+def create_webdriver(user=None, webdriver="chrome", window=None):
+ """Creates a selenium webdriver
+
+ If no user is specified, we use the current request's context"""
+ # Create a webdriver for use in fetching reports
+ if webdriver == "firefox":
+ driver_class = firefox.webdriver.WebDriver
+ options = firefox.options.Options()
+ else:
+ # webdriver == 'chrome':
+ driver_class = chrome.webdriver.WebDriver
+ options = chrome.options.Options()
+ arg = f"--window-size={window[0]},{window[1]}"
+ options.add_argument(arg)
+
+ options.add_argument("--headless")
+
+ # Prepare args for the webdriver init
+ kwargs = dict(options=options)
+ kwargs.update(current_app.config.get("WEBDRIVER_CONFIGURATION"))
+
+ logging.info("Init selenium driver")
+ driver = driver_class(**kwargs)
+
+ # Setting cookies requires doing a request first
+ driver.get(headless_url("/login/"))
+
+ if user:
+ # Set the cookies in the driver
+ for cookie in get_auth_cookies(user):
+ info = dict(name="session", value=cookie)
+ driver.add_cookie(info)
+ elif request.cookies:
+ cookies = request.cookies
+ for k, v in cookies.items():
+ cookie = dict(name=k, value=v)
+ driver.add_cookie(cookie)
+ return driver
+
+
+def get_png_from_url(
+ url, window, element, user, webdriver="chrome", retries=SELENIUM_RETRIES
+):
+ driver = create_webdriver(user, webdriver, window)
+ driver.set_window_size(*window)
+ driver.get(url)
+ img = None
+ logging.debug(f"Sleeping for {SELENIUM_HEADSTART} seconds")
+ time.sleep(SELENIUM_HEADSTART)
+ try:
+ logging.debug(f"Wait for the presence of {element}")
+ element = WebDriverWait(driver, 10).until(
+ EC.presence_of_element_located((By.CLASS_NAME, element))
+ )
+ logging.debug(f"Wait for .loading to be done")
+ WebDriverWait(driver, 60).until_not(
+ EC.presence_of_all_elements_located((By.CLASS_NAME, "loading"))
+ )
+ logging.info("Taking a PNG screenshot")
+ img = element.screenshot_as_png
+ except TimeoutException:
+ logging.error("Selenium timed out")
+ except WebDriverException as e:
+ logging.exception(e)
+ # Some webdrivers do not support screenshots for elements.
+ # In such cases, take a screenshot of the entire page.
+ img = driver.screenshot() # pylint: disable=no-member
+ finally:
+ _destroy_webdriver(driver)
+ return img
diff --git a/superset/views/__init__.py b/superset/views/__init__.py
index 0b69633470cda..01521e63110df 100644
--- a/superset/views/__init__.py
+++ b/superset/views/__init__.py
@@ -23,5 +23,6 @@
from . import datasource # noqa
from . import schedules # noqa
from . import tags # noqa
+from . import thumbnails # noqa
from .log import views # noqa
from .log import api as log_api # noqa
diff --git a/superset/views/thumbnails.py b/superset/views/thumbnails.py
new file mode 100644
index 0000000000000..2f313780b3e28
--- /dev/null
+++ b/superset/views/thumbnails.py
@@ -0,0 +1,38 @@
+# pylint: disable=C,R,W
+from flask import send_file
+from flask_appbuilder import expose
+from flask_appbuilder.security.decorators import has_access
+
+from superset import app, appbuilder, thumbnail_cache
+from superset.utils.selenium import DashboardScreenshot, SliceScreenshot
+from .base import BaseSupersetView
+
+config = app.config
+NO_IMAGE_SRC = "/static/assets/images/no-image.png"
+
+
+class Thumb(BaseSupersetView):
+ """Base views for thumbnails"""
+
+ @expose("/chart///")
+ @has_access
+ def chart(self, slice_id, sha=None):
+ """Returns an thumbnail for a given chart, uses cache if possible"""
+ # TODO SECURITY
+ screenshot = SliceScreenshot(id=slice_id)
+ # TODO handle no image
+ img = screenshot.get_from_cache(thumbnail_cache)
+ return send_file(img, mimetype="image/png")
+
+ @expose("/dashboard///")
+ @has_access
+ def dashboard(self, dashboard_id, sha=None):
+ """Returns an thumbnail for a given dash, uses cache if possible"""
+ # TODO SECURITY
+ screenshot = DashboardScreenshot(id=dashboard_id)
+ img = screenshot.get_from_cache(thumbnail_cache)
+ # TODO handle no image
+ return send_file(img, mimetype="image/png")
+
+
+appbuilder.add_view_no_menu(Thumb)
diff --git a/tests/email_tests.py b/tests/email_tests.py
index 8ae8601242813..da68aad845d5c 100644
--- a/tests/email_tests.py
+++ b/tests/email_tests.py
@@ -25,7 +25,7 @@
from unittest import mock
from superset import app
-from superset.utils import core as utils
+from superset.utils.email import send_email_smtp, send_MIME_email
from .utils import read_fixture
send_email_test = mock.Mock()
diff --git a/tests/schedules_test.py b/tests/schedules_test.py
index 9b583946e937a..5c60ee0414e27 100644
--- a/tests/schedules_test.py
+++ b/tests/schedules_test.py
@@ -21,7 +21,7 @@
from flask_babel import gettext as __
from selenium.common.exceptions import WebDriverException
-from superset import app, db
+from superset import app, db, security_manager
from superset.models.core import Dashboard, Slice
from superset.models.schedules import (
DashboardEmailSchedule,
@@ -30,11 +30,11 @@
SliceEmailSchedule,
)
from superset.tasks.schedules import (
- create_webdriver,
deliver_dashboard,
deliver_slice,
next_schedules,
)
+from superset.utils.selenium import create_webdriver
from .utils import read_fixture
@@ -157,8 +157,9 @@ def test_create_driver(self, mock_driver_class):
mock_driver_class.return_value = mock_driver
mock_driver.find_elements_by_id.side_effect = [True, False]
- create_webdriver()
- create_webdriver()
+ alpha_user = security_manager.find_user(username='alpha')
+ with app.app_context():
+ create_webdriver(alpha_user, webdriver='firefox')
mock_driver.add_cookie.assert_called_once()
@patch("superset.tasks.schedules.firefox.webdriver.WebDriver")
@@ -249,6 +250,7 @@ def test_dashboard_chrome_like(self, mtime, send_email_smtp, driver_class):
)
deliver_dashboard(schedule)
+
mtime.sleep.assert_called_once()
driver.screenshot.assert_called_once()
send_email_smtp.assert_called_once()
diff --git a/tests/sqllab_tests.py b/tests/sqllab_tests.py
index 1774c265e17e0..224adcb3f21c2 100644
--- a/tests/sqllab_tests.py
+++ b/tests/sqllab_tests.py
@@ -26,7 +26,8 @@
from superset.dataframe import SupersetDataFrame
from superset.db_engine_specs import BaseEngineSpec
from superset.models.sql_lab import Query
-from superset.utils.core import datetime_to_epoch, get_main_database
+from superset.utils.core import get_main_database
+from superset.utils.dates import datetime_to_epoch
from .base_tests import SupersetTestCase
diff --git a/tests/utils_tests.py b/tests/utils_tests.py
index 1df71a7788c19..033d87430a886 100644
--- a/tests/utils_tests.py
+++ b/tests/utils_tests.py
@@ -29,7 +29,6 @@
from superset.exceptions import SupersetException
from superset.models.core import Database
from superset.utils.core import (
- base_json_conv,
convert_legacy_filters_into_adhoc,
datetime_f,
get_or_create_db,
@@ -49,6 +48,7 @@
zlib_compress,
zlib_decompress_to_string,
)
+from superset.utils.json import base_json_conv, json_int_dttm_ser, json_iso_dttm_ser
def mock_parse_human_datetime(s):