diff --git a/README.md b/README.md index 50f2fa75863d0..b2919eff3262d 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,22 @@ -# superset +# Superset -internal visualization tool for interactively exploring lyft data (apache/incubator-superset) +Internal visualization tool for interactively exploring lyft data using [apache/incubator-superset](https://github.com/apache/incubator-superset) -================== REMOVE THIS BLOCK ================== -# INITIAL COMMIT INSTRUCTIONS -These instructions are **only** meant for the person doing the initial commit to the repo. -This section of the README should **not** be checked in. -If you see this in GitHub create a PR to remove it #MakeItHappen +## Running -Congratulations on creating your Lyft service! Thank you for contributing to Lyft's codebase. Now that the infrastructure for your service has been set up there are a few more steps required in order to get your awesome new service deployed. +To start the service, run `control start -g superset.dev` While in development, the service loads examples and creates an admin with credentials username and password equal to `admin`. -1. As part of the script you ran to create this library it modified the [metaservice](https://github.com/lyft/metaservice/) repository to have a new definition for your new repository. Get someone in #provisioning to +1 your new repo using pull request -https://github.com/lyft/metaservice/compare/add_superset_definition?expand=1 -1. After you get a +1 on that pull request, merge it, and deploy your repository should be available at https://github.com/lyft/superset. -1. If you chose to add your service to the ops repo you can find a pull request to add -your service here: https://github.com/lyft/ops/compare/add_superset_service?expand=1 -* after merging, wait for a base deploy (monitor #base-train), after which the Jenkins pipeline for the service should be available -================== END REMOVE BLOCK ================== +## Intialization + +Create a new admin user +```bash +make create_admin +``` + +Load examples +```bash +make init_examples +``` ## Requirements * `requirements.txt` for Python dependencies: remember to update `lyft-stdlib` to the [latest version](https://github.com/lyft/python-lyft-stdlib/releases) @@ -37,7 +37,6 @@ To include UI in your project, merge in the [feature-include-ui](https://github.com/lyft/python-service-template/tree/feature-include-ui) branch - ### Develop and test superset in Devbox See [devbox](https://github.com/lyft/devbox). diff --git a/manifest.yaml b/manifest.yaml index dd0a1f32a8ca4..00716b0b79b38 100644 --- a/manifest.yaml +++ b/manifest.yaml @@ -8,7 +8,7 @@ containers: provision: /code/ops/base/starters/post_starters.sh exports: - name: api - port: 8088 + port: 80 groups: - name: dev members: diff --git a/ops/config/pillar/env.sls b/ops/config/pillar/env.sls index a431fd47c6fa6..665d6d502ef26 100644 --- a/ops/config/pillar/env.sls +++ b/ops/config/pillar/env.sls @@ -1,4 +1,7 @@ environment: + common: + LC_ALL: C.UTF-8 + LANG: C.UTF-8 development: PORT: 80 APPLICATION_MONGODB_DBNAME: {{ grains.service_group }} @@ -9,8 +12,6 @@ environment: DEBUG: true REDIS_URL: redis://local-development-iad-onebox.lyft.net:6379 REDIS_KEY_PREFIX: '{{ grains.service_name }}-' - LC_ALL: C.UTF-8 - LANG: C.UTF-8 staging: PORT: 80 APPLICATION_ENV: {{ grains.service_instance }} diff --git a/ops/config/pillar/superset.sls b/ops/config/pillar/superset.sls index 9e9bc68e29752..5b6da3a532698 100644 --- a/ops/config/pillar/superset.sls +++ b/ops/config/pillar/superset.sls @@ -9,7 +9,7 @@ python_executable: python3.6 workers: - web: gunicorn -w 2 --timeout 120 -b 0.0.0.0:8088 --limit-request-line 0 --limit-request-field_size 0 superset:app + web: gunicorn superset:app --workers={{ workers }} --forwarded-allow-ips="*" -k gevent --timeout 120 --worker-connections=1000 -c /etc/gunicorn/gunicorn.conf envoy_with_gunicorn: True diff --git a/ops/config/states/superset/dependencies.sls b/ops/config/states/superset/dependencies.sls index 6a36f64cb7ae7..b5bbe2d778a03 100644 --- a/ops/config/states/superset/dependencies.sls +++ b/ops/config/states/superset/dependencies.sls @@ -14,7 +14,6 @@ Ensure pip dependencies for superset are installed: pkg.installed: - pkgs: - build-essential - - libssl-dev - libffi-dev - libsasl2-dev - libldap2-dev diff --git a/ops/config/states/superset/init.sls b/ops/config/states/superset/init.sls index 22545de370bd1..fbb8471199e6a 100644 --- a/ops/config/states/superset/init.sls +++ b/ops/config/states/superset/init.sls @@ -23,3 +23,21 @@ Ensure basic roles and permissions are defined: cmd.run: - name: /usr/local/bin/service_venv superset init - cwd: /srv/service/current + +{% if grains.service_instance == 'development' %} + +Ensure load mock admin user: + file.managed: + - name: /etc/starter/post/9-ensure-repos + - makedirs: True + - mode: 755 + - contents: | + #!/bin/bash + /usr/local/bin/service_venv service_venv fabmanager create-admin --app superset --username admin --firstname Superset --lastname Lyft --password password --email superset@lyft.com + +Ensure superset examples are loaded: + cmd.run: + - name: /usr/local/bin/service_venv superset load_examples + - cwd: /srv/service/current + +{% endif %} diff --git a/piptools_requirements.in b/piptools_requirements.in index 6a7c7bfeb0c05..14833443910bc 100644 --- a/piptools_requirements.in +++ b/piptools_requirements.in @@ -12,3 +12,5 @@ packaging==16.8 # License: MIT # Upstream url: http://pypi.python.org/pypi/six/ six==1.10.0 + +cryptography==1.9 diff --git a/piptools_requirements.txt b/piptools_requirements.txt index 324cc1ab7adcc..913a4a454231f 100644 --- a/piptools_requirements.txt +++ b/piptools_requirements.txt @@ -2,12 +2,13 @@ # This file is autogenerated by pip-compile # To update, run: # -# control run piptools.compile servicename +# control run piptools.compile superset-private # appdirs==1.4.3 +asn1crypto==0.23.0 # via cryptography certifi==2017.7.27.1 cffi==1.9.1 # via cryptography -cryptography==1.7.2 +cryptography==1.9 enum34==1.1.6 # via cryptography idna==2.5 ipaddress==1.0.18 # via cryptography @@ -21,4 +22,4 @@ six==1.10.0 # The following packages are considered to be unsafe in a requirements file: pip==9.0.1 -# setuptools +setuptools==36.6.0 diff --git a/piptools_requirements3.txt b/piptools_requirements3.txt new file mode 100644 index 0000000000000..913a4a454231f --- /dev/null +++ b/piptools_requirements3.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# control run piptools.compile superset-private +# +appdirs==1.4.3 +asn1crypto==0.23.0 # via cryptography +certifi==2017.7.27.1 +cffi==1.9.1 # via cryptography +cryptography==1.9 +enum34==1.1.6 # via cryptography +idna==2.5 +ipaddress==1.0.18 # via cryptography +ndg-httpsclient==0.4.2 +packaging==16.8 +pyasn1==0.2.2 +pycparser==2.17 # via cffi +pyopenssl==16.2.0 +pyparsing==2.1.10 # via packaging +six==1.10.0 + +# The following packages are considered to be unsafe in a requirements file: +pip==9.0.1 +setuptools==36.6.0 diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000000000..2cb4a3d2d2801 --- /dev/null +++ b/requirements.in @@ -0,0 +1,125 @@ +# piptools: python3 + +# Note that lyft-python in base also requires >= some version and your repo +# must always require == a version greater than lyft-python base. +# If your repo doesn't meet that requirement your tests will fail. + + +# Dispatching system for parties to subscribe to events +# Supports per-endpoint timing in flask +# License: MIT +# Upstream url: https://pypi.python.org/pypi/blinker +blinker==1.4 + +# AWS client library +# License: MIT +# Upstream url: https://github.com/boto/boto/ +boto==2.47.0 + +# The modular source code checker: pep8, pyflakes and co +# License: MIT +# Upstream url: http://bitbucket.org/tarek/flake8 +flake8==3.3.0 + +# Coroutine-based network library +# License: MIT +# Upstream url: http://www.gevent.org/ +gevent==1.2.1 + +# Lightweight in-process concurrent programming +# License: MIT +# Upstream url: https://github.com/python-greenlet/greenlet +greenlet==0.4.12 + +# WSGI HTTP Server for UNIX +# License: MIT +# Upstream url: https://github.com/benoitc/gunicorn +gunicorn==19.7.1 + +## A Python source code analyzer. Used to enforce Python 3 compatibility. +# License: GPL +# Upstream url: https://github.com/PyCQA/pylint +-e git+git://github.com/lyft/pylint@lyft_pylint#egg=pylint + +# Testing library +# License: MIT +# Upstream url: https://github.com/pytest-dev/pytest +pytest==3.1.1 + +# Thin-wrapper around the mock package for easier use with py.test +# License: MIT +# Upstream url: https://github.com/pytest-dev/pytest-mock/ +pytest-mock==1.1 + +# A flake8 plugin that helps you write tidier imports. +# License: ISCL +# Upstream url: https://pypi.python.org/pypi/flake8-tidy-imports +flake8-tidy-imports==1.0.6 + +# Flask service framework +# License: BSD +# Upstream url: http://github.com/mitsuhiko/flask/ +Flask==0.12.2 + +# Scripting support for Flask +# License: BSD +# Upstream url: http://github.com/techniq/flask-script +Flask-Script==2.0.5 + +# A Python library of common idioms used in Lyft microservices +# License: N/A +# Upstream url: https://github.com/lyft/python-lyft-stdlib/ +lyft-stdlib==113.0.0 + +# A Python library of request client and helper modules for Lyft +# License: N/A +# Upstream url: https://github.com/lyft/python-lyft-requests +lyft-requests==0.16.0 + +## Uncomment the following for mongo support +#mongoengine==0.8.7 +#flask-mongoengine==0.7.1 +#pymongo==2.7.2 +#lyft-mongo==1.0.4 + +# Sanctioned DynamoDB ODM, uncomment for dynamo support +# A pythonic interface to Amazon's DynamoDB +# License: MIT +# Upstream url: https://github.com/jlafon/PynamoDB +#pynamodb==3.2.0 + +# Code coverage plugin for py.test +# License: MIT +# Upstream url: https://github.com/pytest-dev/pytest-cov +pytest-cov==2.5.1 + +# YAML parser and emitter for Python +# License: MIT +# Upstream url: http://pyyaml.org/wiki/PyYAML +PyYAML==3.12 + +# A wrapper around gunicorn to handle graceful restarts correctly +# License: BSD +# Upstream url: https://github.com/lyft/rainbow-saddle +rainbow-saddle>=0.4.0 + +# A simple statsd client. +# License: MIT +# Upstream url: https://github.com/jsocol/pystatsd +statsd==3.2.1 + +# Type Hints for Python +# License: PSF +# Upstream url: https://github.com/python/typing +typing==3.6.1 + +# Superset Dependencies +wtforms==2.1 +amqp==1.4.9 +anyjson==0.3.3 +psycopg2 # Postgres driver +sqlalchemy-redshift # Redshift driver +pyhive # Presto driver + +# Superset github module +superset==0.20.4 diff --git a/requirements.txt b/requirements.txt index 734c7a4b226aa..673b6db83ff80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,124 +1,120 @@ -# piptools: python3 - -# Note that lyft-python in base also requires >= some version and your repo -# must always require == a version greater than lyft-python base. -# If your repo doesn't meet that requirement your tests will fail. - - -# Dispatching system for parties to subscribe to events -# Supports per-endpoint timing in flask -# License: MIT -# Upstream url: https://pypi.python.org/pypi/blinker +# +# This file is autogenerated by pip-compile +# To update, run: +# +# control run piptools.compile superset-private +# +-e git+git://github.com/lyft/pylint@lyft_pylint#egg=pylint +alembic==0.9.6 # via flask-migrate +amqp==1.4.9 +anyjson==0.3.3 +appdirs==1.4.3 +argh==0.26.2 # via watchdog +asn1crypto==0.23.0 +astroid==1.4.9 +babel==2.5.1 # via flask-babel, flower +backports.functools-lru-cache==1.4 +billiard==3.3.0.23 # via celery blinker==1.4 - -# AWS client library -# License: MIT -# Upstream url: https://github.com/boto/boto/ +boto3==1.4.4 # via superset boto==2.47.0 - -# The modular source code checker: pep8, pyflakes and co -# License: MIT -# Upstream url: http://bitbucket.org/tarek/flake8 +botocore==1.5.95 # via boto3, s3transfer +celery==3.1.25 # via flower, superset +certifi==2017.7.27.1 +cffi==1.9.1 +chardet==3.0.4 # via requests +click==6.7 # via flask, flask-appbuilder +colorama==0.3.9 # via flask-appbuilder, superset +configparser==3.5.0 +coverage==4.4.1 # via pytest-cov +cryptography==1.9 +defusedxml==0.5.0 # via python3-openid +docutils==0.14 # via botocore +enum34==1.1.6 +flake8-tidy-imports==1.0.6 flake8==3.3.0 - -# Coroutine-based network library -# License: MIT -# Upstream url: http://www.gevent.org/ +flask-appbuilder==1.9.4 # via superset +flask-babel==0.11.1 # via flask-appbuilder +flask-cache==0.13.1 # via superset +flask-login==0.2.11 # via flask-appbuilder +flask-migrate==2.0.3 # via superset +flask-openid==1.2.5 # via flask-appbuilder +flask-script==2.0.5 +flask-sqlalchemy==2.1 # via flask-appbuilder, flask-migrate, superset +flask-testing==0.6.2 # via superset +flask-wtf==0.14.2 # via flask-appbuilder, superset +flask==0.12.2 +flower==0.9.1 # via superset +future==0.16.0 # via pyhive, superset gevent==1.2.1 - -# Lightweight in-process concurrent programming -# License: MIT -# Upstream url: https://github.com/python-greenlet/greenlet greenlet==0.4.12 - -# WSGI HTTP Server for UNIX -# License: MIT -# Upstream url: https://github.com/benoitc/gunicorn gunicorn==19.7.1 - -## A Python source code analyzer. Used to enforce Python 3 compatibility. -# License: GPL -# Upstream url: https://github.com/PyCQA/pylint --e git+git://github.com/lyft/pylint@lyft_pylint#egg=pylint - -# Testing library -# License: MIT -# Upstream url: https://github.com/pytest-dev/pytest -pytest==3.1.1 - -# Thin-wrapper around the mock package for easier use with py.test -# License: MIT -# Upstream url: https://github.com/pytest-dev/pytest-mock/ -pytest-mock==1.1 - -# A flake8 plugin that helps you write tidier imports. -# License: ISCL -# Upstream url: https://pypi.python.org/pypi/flake8-tidy-imports -flake8-tidy-imports==1.0.6 - -# Flask service framework -# License: BSD -# Upstream url: http://github.com/mitsuhiko/flask/ -Flask==0.12.2 - -# Scripting support for Flask -# License: BSD -# Upstream url: http://github.com/techniq/flask-script -Flask-Script==2.0.5 - -# A Python library of common idioms used in Lyft microservices -# License: N/A -# Upstream url: https://github.com/lyft/python-lyft-stdlib/ -lyft-stdlib==113.0.0 - -# A Python library of request client and helper modules for Lyft -# License: N/A -# Upstream url: https://github.com/lyft/python-lyft-requests +humanize==0.5.1 # via superset +idna==2.5 +ipaddress==1.0.18 +isort==4.2.15 +itsdangerous==0.24 # via flask +jinja2==2.9.6 # via flask, flask-babel +jmespath==0.9.3 # via boto3, botocore +kombu==3.0.37 # via celery +lazy-object-proxy==1.3.1 # via astroid +lyft-exceptions==0.1.0 # via lyft-requests lyft-requests==0.16.0 - -## Uncomment the following for mongo support -#mongoengine==0.8.7 -#flask-mongoengine==0.7.1 -#pymongo==2.7.2 -#lyft-mongo==1.0.4 - -# Sanctioned DynamoDB ODM, uncomment for dynamo support -# A pythonic interface to Amazon's DynamoDB -# License: MIT -# Upstream url: https://github.com/jlafon/PynamoDB -#pynamodb==3.2.0 - -# Code coverage plugin for py.test -# License: MIT -# Upstream url: https://github.com/pytest-dev/pytest-cov +lyft-stdlib==113.0.0 +lyftsettings==0.1.0 # via lyft-requests +mako==1.0.7 # via alembic +markdown==2.6.8 # via superset +markupsafe==1.0 # via jinja2, mako +mccabe==0.6.1 # via flake8 +ndg-httpsclient==0.4.2 +numpy==1.13.3 # via pandas +packaging==16.8 +pandas==0.20.3 # via superset +parsedatetime==2.0.0 # via superset +pathtools==0.1.2 # via watchdog +protobuf==3.2.0 # via lyft-requests +psutil==5.4.0 # via rainbow-saddle +psycopg2==2.7.3.2 +py==1.4.34 # via pytest +pyasn1==0.2.2 +pycodestyle==2.3.1 # via flake8 +pycparser==2.17 +pydruid==0.3.1 # via superset +pyflakes==1.5.0 # via flake8 +pyhive==0.5.0 +pyopenssl==16.2.0 +pyparsing==2.1.10 pytest-cov==2.5.1 - -# YAML parser and emitter for Python -# License: MIT -# Upstream url: http://pyyaml.org/wiki/PyYAML -PyYAML==3.12 - -# A wrapper around gunicorn to handle graceful restarts correctly -# License: BSD -# Upstream url: https://github.com/lyft/rainbow-saddle -rainbow-saddle>=0.4.0 - -# A simple statsd client. -# License: MIT -# Upstream url: https://github.com/jsocol/pystatsd +pytest-mock==1.1 +pytest==3.1.1 +python-dateutil==2.6.0 # via alembic, botocore, pandas, superset +python-editor==1.0.3 # via alembic +python3-openid==3.1.0 # via flask-openid +pytz==2017.3 # via babel, celery, flower, pandas +pyyaml==3.12 +rainbow-saddle==0.4.0 +requests==2.17.3 # via lyft-requests, superset +s3transfer==0.1.11 # via boto3 +sasl==0.2.1 # via thrift-sasl +simplejson==3.10.0 # via superset +six==1.10.0 +sqlalchemy-redshift==0.7.0 +sqlalchemy-utils==0.32.16 # via superset +sqlalchemy==1.1.9 # via alembic, flask-sqlalchemy, sqlalchemy-redshift, sqlalchemy-utils, superset +sqlparse==0.2.3 # via superset statsd==3.2.1 - -# Type Hints for Python -# License: PSF -# Upstream url: https://github.com/python/typing +superset==0.20.4 +thrift-sasl==0.3.0 # via superset +thrift==0.10.0 # via superset, thrift-sasl +tornado==4.2.0 # via flower typing==3.6.1 - -# Superset Dependencies +ujson==1.35 # via lyft-stdlib +urllib3==1.21.1 # via requests +watchdog==0.8.3 # via lyft-stdlib +werkzeug==0.12.2 # via flask +wrapt==1.10.11 # via astroid wtforms==2.1 -amqp==1.4.9 -anyjson==0.3.3 -psycopg2 -sqlalchemy-redshift -# Superset github module -superset==0.20.4 +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/requirements3.txt b/requirements3.txt new file mode 100644 index 0000000000000..673b6db83ff80 --- /dev/null +++ b/requirements3.txt @@ -0,0 +1,120 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# control run piptools.compile superset-private +# +-e git+git://github.com/lyft/pylint@lyft_pylint#egg=pylint +alembic==0.9.6 # via flask-migrate +amqp==1.4.9 +anyjson==0.3.3 +appdirs==1.4.3 +argh==0.26.2 # via watchdog +asn1crypto==0.23.0 +astroid==1.4.9 +babel==2.5.1 # via flask-babel, flower +backports.functools-lru-cache==1.4 +billiard==3.3.0.23 # via celery +blinker==1.4 +boto3==1.4.4 # via superset +boto==2.47.0 +botocore==1.5.95 # via boto3, s3transfer +celery==3.1.25 # via flower, superset +certifi==2017.7.27.1 +cffi==1.9.1 +chardet==3.0.4 # via requests +click==6.7 # via flask, flask-appbuilder +colorama==0.3.9 # via flask-appbuilder, superset +configparser==3.5.0 +coverage==4.4.1 # via pytest-cov +cryptography==1.9 +defusedxml==0.5.0 # via python3-openid +docutils==0.14 # via botocore +enum34==1.1.6 +flake8-tidy-imports==1.0.6 +flake8==3.3.0 +flask-appbuilder==1.9.4 # via superset +flask-babel==0.11.1 # via flask-appbuilder +flask-cache==0.13.1 # via superset +flask-login==0.2.11 # via flask-appbuilder +flask-migrate==2.0.3 # via superset +flask-openid==1.2.5 # via flask-appbuilder +flask-script==2.0.5 +flask-sqlalchemy==2.1 # via flask-appbuilder, flask-migrate, superset +flask-testing==0.6.2 # via superset +flask-wtf==0.14.2 # via flask-appbuilder, superset +flask==0.12.2 +flower==0.9.1 # via superset +future==0.16.0 # via pyhive, superset +gevent==1.2.1 +greenlet==0.4.12 +gunicorn==19.7.1 +humanize==0.5.1 # via superset +idna==2.5 +ipaddress==1.0.18 +isort==4.2.15 +itsdangerous==0.24 # via flask +jinja2==2.9.6 # via flask, flask-babel +jmespath==0.9.3 # via boto3, botocore +kombu==3.0.37 # via celery +lazy-object-proxy==1.3.1 # via astroid +lyft-exceptions==0.1.0 # via lyft-requests +lyft-requests==0.16.0 +lyft-stdlib==113.0.0 +lyftsettings==0.1.0 # via lyft-requests +mako==1.0.7 # via alembic +markdown==2.6.8 # via superset +markupsafe==1.0 # via jinja2, mako +mccabe==0.6.1 # via flake8 +ndg-httpsclient==0.4.2 +numpy==1.13.3 # via pandas +packaging==16.8 +pandas==0.20.3 # via superset +parsedatetime==2.0.0 # via superset +pathtools==0.1.2 # via watchdog +protobuf==3.2.0 # via lyft-requests +psutil==5.4.0 # via rainbow-saddle +psycopg2==2.7.3.2 +py==1.4.34 # via pytest +pyasn1==0.2.2 +pycodestyle==2.3.1 # via flake8 +pycparser==2.17 +pydruid==0.3.1 # via superset +pyflakes==1.5.0 # via flake8 +pyhive==0.5.0 +pyopenssl==16.2.0 +pyparsing==2.1.10 +pytest-cov==2.5.1 +pytest-mock==1.1 +pytest==3.1.1 +python-dateutil==2.6.0 # via alembic, botocore, pandas, superset +python-editor==1.0.3 # via alembic +python3-openid==3.1.0 # via flask-openid +pytz==2017.3 # via babel, celery, flower, pandas +pyyaml==3.12 +rainbow-saddle==0.4.0 +requests==2.17.3 # via lyft-requests, superset +s3transfer==0.1.11 # via boto3 +sasl==0.2.1 # via thrift-sasl +simplejson==3.10.0 # via superset +six==1.10.0 +sqlalchemy-redshift==0.7.0 +sqlalchemy-utils==0.32.16 # via superset +sqlalchemy==1.1.9 # via alembic, flask-sqlalchemy, sqlalchemy-redshift, sqlalchemy-utils, superset +sqlparse==0.2.3 # via superset +statsd==3.2.1 +superset==0.20.4 +thrift-sasl==0.3.0 # via superset +thrift==0.10.0 # via superset, thrift-sasl +tornado==4.2.0 # via flower +typing==3.6.1 +ujson==1.35 # via lyft-stdlib +urllib3==1.21.1 # via requests +watchdog==0.8.3 # via lyft-stdlib +werkzeug==0.12.2 # via flask +wrapt==1.10.11 # via astroid +wtforms==2.1 + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools