Migrate the application to Python 3

.circleci/config.yml

@@ -44,6 +44,7 @@ jobs:
             mkdir -p /tmp/test-results/unit-tests
             docker cp tests:/app/coverage.xml ./coverage.xml
             docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml
+          when: always
       - store_test_results:
           path: /tmp/test-results
       - store_artifacts:
@@ -63,8 +64,8 @@ jobs:
       - image: circleci/node:8
     steps:
       - checkout
-      - run: sudo apt install python-pip
-      - run: sudo pip install -r requirements_bundles.txt
+      - run: sudo apt install python3-pip
+      - run: sudo pip3 install -r requirements_bundles.txt
       - run: npm install
       - run: npm run bundle
       - run: npm test
@@ -99,8 +100,8 @@ jobs:
     steps:
       - setup_remote_docker
       - checkout
-      - run: sudo apt install python-pip
-      - run: sudo pip install -r requirements_bundles.txt
+      - run: sudo apt install python3-pip
+      - run: sudo pip3 install -r requirements_bundles.txt
       - run: .circleci/update_version
       - run: npm run bundle
       - run: .circleci/docker_build

Dockerfile

@@ -8,11 +8,40 @@ COPY client /frontend/client
 COPY webpack.config.js /frontend/
 RUN npm run build
 
-FROM redash/base:debian
+FROM python:3.7-slim
+
+EXPOSE 5000
 
 # Controls whether to install extra dependencies needed for all data sources.
 ARG skip_ds_deps
 
+RUN useradd --create-home redash
+
+# Ubuntu packages
+RUN apt-get update && \
+  apt-get install -y \
+    curl \
+    gnupg \
+    build-essential \
+    pwgen \
+    libffi-dev \
+    sudo \
+    git-core \
+    wget \
+    # Postgres client
+    libpq-dev \
+    # for SAML
+    xmlsec1 \
+    # Additional packages required for data sources:
+    libssl-dev \
+    default-libmysqlclient-dev \
+    freetds-dev \
+    libsasl2-dev && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
 # We first copy only the requirements file, to avoid rebuilding on every file
 # change.
 COPY requirements.txt requirements_bundles.txt requirements_dev.txt requirements_all_ds.txt ./

bin/bundle-extensions

@@ -1,9 +1,8 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 """Copy bundle extension files to the client/app/extension directory"""
 import logging
 import os
-from pathlib2 import Path
+from pathlib import Path
 from shutil import copy
 from collections import OrderedDict as odict
 

bin/get_changes.py

@@ -1,9 +1,10 @@
-#!/bin/env python
-from __future__ import print_function
+#!/bin/env python3
+
 import sys
 import re
 import subprocess
 
+
 def get_change_log(previous_sha):
     args = ['git', '--no-pager', 'log', '--merges', '--grep', 'Merge pull request', '--pretty=format:"%h|%s|%b|%p"', 'master...{}'.format(previous_sha)]
     log = subprocess.check_output(args)
@@ -33,4 +34,4 @@ def get_change_log(previous_sha):
     changes = get_change_log(previous_sha)
 
     for change in changes:
-        print(change)
+        print(change)

bin/release_manager.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+#!/usr/bin/env python3
 import os
 import sys
 import re

bin/upgrade

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import urllib
 import argparse
 import os
@@ -27,7 +27,7 @@ def run(cmd, cwd=None):
 
 
 def confirm(question):
-    reply = str(raw_input(question + ' (y/n): ')).lower().strip()
+    reply = str(input(question + ' (y/n): ')).lower().strip()
 
     if reply[0] == 'y':
         return True

migrations/versions/65fc9ede4746_add_is_draft_status_to_queries_and_.py

@@ -1,11 +1,11 @@
 """Add is_draft status to queries and dashboards
 
 Revision ID: 65fc9ede4746
-Revises: 
+Revises:
 Create Date: 2016-12-07 18:08:13.395586
 
 """
-from __future__ import print_function
+
 from alembic import op
 import sqlalchemy as sa
 
@@ -26,7 +26,7 @@ def upgrade():
         op.execute("UPDATE dashboards SET is_draft = false")
     except ProgrammingError as e:
         # The columns might exist if you ran the old migrations.
-        if 'column "is_draft" of relation "queries" already exists' in e.message:
+        if 'column "is_draft" of relation "queries" already exists' in str(e):
             print("Can't run this migration as you already have is_draft columns, please run:")
             print("./manage.py db stamp {} # you might need to alter the command to match your environment.".format(revision))
             exit()

migrations/versions/969126bd800f_.py

@@ -5,7 +5,7 @@
 Create Date: 2018-01-31 15:20:30.396533
 
 """
-from __future__ import print_function
+
 import simplejson
 from alembic import op
 import sqlalchemy as sa

redash/__init__.py

@@ -2,8 +2,6 @@
 import logging
 import os
 import sys
-import urllib
-import urlparse
 
 import redis
 from flask_mail import Mail
@@ -41,6 +39,7 @@ def setup_logging():
 setup_logging()
 
 redis_connection = redis.from_url(settings.REDIS_URL)
+rq_redis_connection = redis.from_url(settings.RQ_REDIS_URL)
 mail = Mail()
 migrate = Migrate()
 statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX)

redash/app.py

@@ -10,7 +10,7 @@ def __init__(self, *args, **kwargs):
         kwargs.update({
             'template_folder': settings.STATIC_ASSETS_PATH,
             'static_folder': settings.STATIC_ASSETS_PATH,
-            'static_path': '/static',
+            'static_url_path': '/static',
         })
         super(Redash, self).__init__(__name__, *args, **kwargs)
         # Make sure we get the right referral address even behind proxies like nginx.

redash/authentication/__init__.py

@@ -2,7 +2,7 @@
 import hmac
 import logging
 import time
-from urlparse import urlsplit, urlunsplit
+from urllib.parse import urlsplit, urlunsplit
 
 from flask import jsonify, redirect, request, url_for
 from flask_login import LoginManager, login_user, logout_user, user_logged_in
@@ -33,8 +33,8 @@ def sign(key, path, expires):
     if not key:
         return None
 
-    h = hmac.new(str(key), msg=path, digestmod=hashlib.sha1)
-    h.update(str(expires))
+    h = hmac.new(key.encode(), msg=path.encode(), digestmod=hashlib.sha1)
+    h.update(str(expires).encode())
 
     return h.hexdigest()
 
@@ -93,7 +93,7 @@ def hmac_load_user_from_request(request):
             calculated_signature = sign(query.api_key, request.path, expires)
 
             if query.api_key and signature == calculated_signature:
-                return models.ApiUser(query.api_key, query.org, query.groups.keys(), name="ApiKey: Query {}".format(query.id))
+                return models.ApiUser(query.api_key, query.org, list(query.groups.keys()), name="ApiKey: Query {}".format(query.id))
 
     return None
 
@@ -118,7 +118,7 @@ def get_user_from_api_key(api_key, query_id):
             if query_id:
                 query = models.Query.get_by_id_and_org(query_id, org)
                 if query and query.api_key == api_key:
-                    user = models.ApiUser(api_key, query.org, query.groups.keys(), name="ApiKey: Query {}".format(query.id))
+                    user = models.ApiUser(api_key, query.org, list(query.groups.keys()), name="ApiKey: Query {}".format(query.id))
 
     return user
 

redash/authentication/account.py

@@ -48,7 +48,7 @@ def send_verify_email(user, org):
     }
     html_content = render_template('emails/verify.html', **context)
     text_content = render_template('emails/verify.txt', **context)
-    subject = u"{}, please verify your email address".format(user.name)
+    subject = "{}, please verify your email address".format(user.name)
 
     send_mail.delay([user.email], subject, html_content, text_content)
 
@@ -57,7 +57,7 @@ def send_invite_email(inviter, invited, invite_url, org):
     context = dict(inviter=inviter, invited=invited, org=org, invite_url=invite_url)
     html_content = render_template('emails/invite.html', **context)
     text_content = render_template('emails/invite.txt', **context)
-    subject = u"{} invited you to join Redash".format(inviter.name)
+    subject = "{} invited you to join Redash".format(inviter.name)
 
     send_mail.delay([invited.email], subject, html_content, text_content)
 
@@ -67,7 +67,7 @@ def send_password_reset_email(user):
     context = dict(user=user, reset_link=reset_link)
     html_content = render_template('emails/reset.html', **context)
     text_content = render_template('emails/reset.txt', **context)
-    subject = u"Reset your password"
+    subject = "Reset your password"
 
     send_mail.delay([user.email], subject, html_content, text_content)
     return reset_link
@@ -76,6 +76,6 @@ def send_password_reset_email(user):
 def send_user_disabled_email(user):
     html_content = render_template('emails/reset_disabled.html', user=user)
     text_content = render_template('emails/reset_disabled.txt', user=user)
-    subject = u"Your Redash account is disabled"
+    subject = "Your Redash account is disabled"
 
     send_mail.delay([user.email], subject, html_content, text_content)

redash/cli/__init__.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+
 
 import click
 import simplejson
@@ -53,7 +53,7 @@ def status():
 @manager.command()
 def check_settings():
     """Show the settings as Redash sees them (useful for debugging)."""
-    for name, item in current_app.config.iteritems():
+    for name, item in current_app.config.items():
         print("{} = {}".format(name, item))
 
 

redash/cli/data_sources.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+
 from sys import exit
 
 import click
@@ -15,11 +15,11 @@
 manager = AppGroup(help="Data sources management commands.")
 
 
-@manager.command()
+@manager.command(name='list')
 @click.option('--org', 'organization', default=None,
               help="The organization the user belongs to (leave blank for "
               "all organizations).")
-def list(organization=None):
+def list_command(organization=None):
     """List currently configured data sources."""
     if organization:
         org = models.Organization.get_by_slug(organization)
@@ -99,11 +99,11 @@ def new(name=None, type=None, options=None, organization='default'):
             print("{}. {}".format(i + 1, query_runner_name))
 
         idx = 0
-        while idx < 1 or idx > len(query_runners.keys()):
+        while idx < 1 or idx > len(list(query_runners.keys())):
             idx = click.prompt("[{}-{}]".format(1, len(query_runners.keys())),
                                type=int)
 
-        type = query_runners.keys()[idx - 1]
+        type = list(query_runners.keys())[idx - 1]
     else:
         validate_data_source_type(type)
 
@@ -119,7 +119,7 @@ def new(name=None, type=None, options=None, organization='default'):
 
         options_obj = {}
 
-        for k, prop in schema['properties'].iteritems():
+        for k, prop in schema['properties'].items():
             required = k in schema.get('required', [])
             default_value = "<<DEFAULT_VALUE>>"
             if required:

redash/cli/groups.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+
 from sys import exit
 
 from sqlalchemy.orm.exc import NoResultFound
@@ -36,7 +36,7 @@ def create(name, permissions=None, organization='default'):
             permissions=permissions))
         models.db.session.commit()
     except Exception as e:
-        print("Failed create group: %s" % e.message)
+        print("Failed create group: %s" % e)
         exit(1)
 
 
@@ -67,7 +67,7 @@ def change_permissions(group_id, permissions=None):
         models.db.session.add(group)
         models.db.session.commit()
     except Exception as e:
-        print("Failed change permission: %s" % e.message)
+        print("Failed change permission: %s" % e)
         exit(1)
 
 
@@ -80,10 +80,10 @@ def extract_permissions_string(permissions):
     return permissions
 
 
-@manager.command()
+@manager.command(name='list')
 @option('--org', 'organization', default=None,
         help="The organization to limit to (leave blank for all).")
-def list(organization=None):
+def list_command(organization=None):
     """List all groups"""
     if organization:
         org = models.Organization.get_by_slug(organization)

redash/cli/organization.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+
 from click import argument
 from flask.cli import AppGroup
 
@@ -29,8 +29,8 @@ def show_google_apps_domains():
         ', '.join(organization.google_apps_domains)))
 
 
-@manager.command()
-def list():
+@manager.command(name='list')
+def list_command():
     """List all organizations"""
     orgs = models.Organization.query
     for i, org in enumerate(orgs.order_by(models.Organization.name)):

redash/cli/rq.py

@@ -7,7 +7,7 @@
 from flask.cli import AppGroup
 from rq import Connection, Worker
 
-from redash import redis_connection
+from redash import rq_redis_connection
 from redash.schedule import rq_scheduler, schedule_periodic_jobs
 
 manager = AppGroup(help="RQ management commands.")
@@ -22,15 +22,17 @@ def scheduler():
 @manager.command()
 @argument('queues', nargs=-1)
 def worker(queues='default'):
-    with Connection(redis_connection):
+    if not queues:
+        queues = ('default',)
+    with Connection(rq_redis_connection):
         w = Worker(queues)
         w.work()
 
 
 @manager.command()
 def healthcheck():
     hostname = socket.gethostname()
-    with Connection(redis_connection):
+    with Connection(rq_redis_connection):
         all_workers = Worker.all()
 
         local_workers = [w for w in all_workers if w.hostname == hostname]