Updates from Halloween (#20)

Merging halloween dev branch to master since code freeze is over

@lyft/command-center @amalakar @amalakar

README.md

@@ -1,22 +1,22 @@
-# superset
+# Superset
 
-internal visualization tool for interactively exploring lyft data (apache/incubator-superset)
+Internal visualization tool for interactively exploring lyft data using [apache/incubator-superset](https://github.com/apache/incubator-superset)
 
-================== REMOVE THIS BLOCK ==================
-# INITIAL COMMIT INSTRUCTIONS
-These instructions are **only** meant for the person doing the initial commit to the repo.
-This section of the README should **not** be checked in.
-If you see this in GitHub create a PR to remove it #MakeItHappen
+## Running
 
-Congratulations on creating your Lyft service! Thank you for contributing to Lyft's codebase. Now that the infrastructure for your service has been set up there are a few more steps required in order to get your awesome new service deployed.
+To start the service, run `control start -g superset.dev` While in development, the service loads examples and creates an admin with credentials username and password equal to `admin`.
 
-1. As part of the script you ran to create this library it modified the [metaservice](https://github.com/lyft/metaservice/) repository to have a new definition for your new repository. Get someone in #provisioning to +1 your new repo using pull request
-https://github.com/lyft/metaservice/compare/add_superset_definition?expand=1
-1. After you get a +1 on that pull request, merge it, and deploy your repository should be available at https://github.com/lyft/superset.
-1. If you chose to add your service to the ops repo you can find a pull request to add
-your service here: https://github.com/lyft/ops/compare/add_superset_service?expand=1
-* after merging, wait for a base deploy (monitor #base-train), after which the Jenkins pipeline for the service should be available
-================== END REMOVE BLOCK ==================
+## Intialization
+
+Create a new admin user
+```bash
+make create_admin 
+```
+
+Load examples
+```bash
+make init_examples
+```
 
 ## Requirements
 * `requirements.txt` for Python dependencies: remember to update `lyft-stdlib` to the [latest version](https://github.com/lyft/python-lyft-stdlib/releases)
@@ -37,7 +37,6 @@ To include UI in your project, merge in the
 [feature-include-ui](https://github.com/lyft/python-service-template/tree/feature-include-ui) branch
 
 
-
 ### Develop and test superset in Devbox
 
 See [devbox](https://github.com/lyft/devbox).

manifest.yaml

@@ -8,7 +8,7 @@ containers:
     provision: /code/ops/base/starters/post_starters.sh
     exports:
       - name: api
-        port: 8088
+        port: 80
 groups:
   - name: dev
     members:

ops/config/pillar/env.sls

@@ -1,4 +1,7 @@
 environment:
+  common:
+    LC_ALL: C.UTF-8
+    LANG: C.UTF-8
   development:
     PORT: 80
     APPLICATION_MONGODB_DBNAME: {{ grains.service_group }}
@@ -9,8 +12,6 @@ environment:
     DEBUG: true
     REDIS_URL: redis://local-development-iad-onebox.lyft.net:6379
     REDIS_KEY_PREFIX: '{{ grains.service_name }}-'
-    LC_ALL: C.UTF-8
-    LANG: C.UTF-8
   staging:
     PORT: 80
     APPLICATION_ENV: {{ grains.service_instance }}

ops/config/pillar/superset.sls

@@ -9,7 +9,7 @@
 python_executable: python3.6
 
 workers:
-  web: gunicorn -w 2 --timeout 120 -b  0.0.0.0:8088 --limit-request-line 0 --limit-request-field_size 0 superset:app
+  web: gunicorn superset:app --workers={{ workers }} --forwarded-allow-ips="*" -k gevent --timeout 120 --worker-connections=1000 -c /etc/gunicorn/gunicorn.conf
 
 envoy_with_gunicorn: True
 

ops/config/states/superset/dependencies.sls

@@ -14,7 +14,6 @@ Ensure pip dependencies for superset are installed:
   pkg.installed:
     - pkgs:
       - build-essential
-      - libssl-dev
       - libffi-dev
       - libsasl2-dev
       - libldap2-dev

ops/config/states/superset/init.sls

@@ -23,3 +23,21 @@ Ensure basic roles and permissions are defined:
  cmd.run:
    - name: /usr/local/bin/service_venv superset init
    - cwd: /srv/service/current
+
+{% if grains.service_instance == 'development' %}
+
+Ensure load mock admin user:
+  file.managed:
+    - name: /etc/starter/post/9-ensure-repos
+    - makedirs: True
+    - mode: 755
+    - contents: |
+        #!/bin/bash
+        /usr/local/bin/service_venv service_venv fabmanager create-admin --app superset --username admin --firstname Superset --lastname Lyft --password password --email superset@lyft.com
+
+Ensure superset examples are loaded:
+ cmd.run:
+   - name: /usr/local/bin/service_venv superset load_examples
+   - cwd: /srv/service/current
+
+{% endif %}

piptools_requirements.in

@@ -12,3 +12,5 @@ packaging==16.8
 # License: MIT
 # Upstream url: http://pypi.python.org/pypi/six/
 six==1.10.0
+
+cryptography==1.9

piptools_requirements.txt

@@ -2,12 +2,13 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    control run piptools.compile servicename
+#    control run piptools.compile superset-private
 #
 appdirs==1.4.3
+asn1crypto==0.23.0        # via cryptography
 certifi==2017.7.27.1
 cffi==1.9.1               # via cryptography
-cryptography==1.7.2
+cryptography==1.9
 enum34==1.1.6             # via cryptography
 idna==2.5
 ipaddress==1.0.18         # via cryptography
@@ -21,4 +22,4 @@ six==1.10.0
 
 # The following packages are considered to be unsafe in a requirements file:
 pip==9.0.1
-# setuptools
+setuptools==36.6.0

piptools_requirements3.txt

@@ -0,0 +1,25 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    control run piptools.compile superset-private
+#
+appdirs==1.4.3
+asn1crypto==0.23.0        # via cryptography
+certifi==2017.7.27.1
+cffi==1.9.1               # via cryptography
+cryptography==1.9
+enum34==1.1.6             # via cryptography
+idna==2.5
+ipaddress==1.0.18         # via cryptography
+ndg-httpsclient==0.4.2
+packaging==16.8
+pyasn1==0.2.2
+pycparser==2.17           # via cffi
+pyopenssl==16.2.0
+pyparsing==2.1.10         # via packaging
+six==1.10.0
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==9.0.1
+setuptools==36.6.0

requirements.in

@@ -0,0 +1,125 @@
+# piptools: python3
+
+# Note that lyft-python in base also requires >= some version and your repo
+# must always require == a version greater than lyft-python base.
+# If your repo doesn't meet that requirement your tests will fail.
+
+
+# Dispatching system for parties to subscribe to events
+# Supports per-endpoint timing in flask
+# License: MIT
+# Upstream url: https://pypi.python.org/pypi/blinker
+blinker==1.4
+
+# AWS client library
+# License: MIT
+# Upstream url: https://github.com/boto/boto/
+boto==2.47.0
+
+# The modular source code checker: pep8, pyflakes and co
+# License: MIT
+# Upstream url: http://bitbucket.org/tarek/flake8
+flake8==3.3.0
+
+# Coroutine-based network library
+# License: MIT
+# Upstream url: http://www.gevent.org/
+gevent==1.2.1
+
+# Lightweight in-process concurrent programming
+# License: MIT
+# Upstream url: https://github.com/python-greenlet/greenlet
+greenlet==0.4.12
+
+# WSGI HTTP Server for UNIX
+# License: MIT
+# Upstream url: https://github.com/benoitc/gunicorn
+gunicorn==19.7.1
+
+## A Python source code analyzer.  Used to enforce Python 3 compatibility.
+# License: GPL
+# Upstream url: https://github.com/PyCQA/pylint
+-e git+git://github.com/lyft/pylint@lyft_pylint#egg=pylint
+
+# Testing library
+# License: MIT
+# Upstream url: https://github.com/pytest-dev/pytest
+pytest==3.1.1
+
+# Thin-wrapper around the mock package for easier use with py.test
+# License: MIT
+# Upstream url: https://github.com/pytest-dev/pytest-mock/
+pytest-mock==1.1
+
+# A flake8 plugin that helps you write tidier imports.
+# License: ISCL
+# Upstream url: https://pypi.python.org/pypi/flake8-tidy-imports
+flake8-tidy-imports==1.0.6
+
+# Flask service framework
+# License: BSD
+# Upstream url: http://github.com/mitsuhiko/flask/
+Flask==0.12.2
+
+# Scripting support for Flask
+# License: BSD
+# Upstream url: http://github.com/techniq/flask-script
+Flask-Script==2.0.5
+
+# A Python library of common idioms used in Lyft microservices
+# License: N/A
+# Upstream url: https://github.com/lyft/python-lyft-stdlib/
+lyft-stdlib==113.0.0
+
+# A Python library of request client and helper modules for Lyft
+# License: N/A
+# Upstream url: https://github.com/lyft/python-lyft-requests
+lyft-requests==0.16.0
+
+## Uncomment the following for mongo support
+#mongoengine==0.8.7
+#flask-mongoengine==0.7.1
+#pymongo==2.7.2
+#lyft-mongo==1.0.4
+
+# Sanctioned DynamoDB ODM, uncomment for dynamo support
+# A pythonic interface to Amazon's DynamoDB
+# License: MIT
+# Upstream url: https://github.com/jlafon/PynamoDB
+#pynamodb==3.2.0
+
+# Code coverage plugin for py.test
+# License: MIT
+# Upstream url: https://github.com/pytest-dev/pytest-cov
+pytest-cov==2.5.1
+
+# YAML parser and emitter for Python
+# License: MIT
+# Upstream url: http://pyyaml.org/wiki/PyYAML
+PyYAML==3.12
+
+# A wrapper around gunicorn to handle graceful restarts correctly
+# License: BSD
+# Upstream url: https://github.com/lyft/rainbow-saddle
+rainbow-saddle>=0.4.0
+
+# A simple statsd client.
+# License: MIT
+# Upstream url: https://github.com/jsocol/pystatsd
+statsd==3.2.1
+
+# Type Hints for Python
+# License: PSF
+# Upstream url: https://github.com/python/typing
+typing==3.6.1
+
+# Superset Dependencies
+wtforms==2.1
+amqp==1.4.9
+anyjson==0.3.3
+psycopg2 # Postgres driver
+sqlalchemy-redshift # Redshift driver
+pyhive # Presto driver
+
+# Superset github module
+superset==0.20.4