Setup EKS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python Version 3.6.5 |Anaconda, Inc.| (default, Apr 29 2018, 16:14:56) \n",
      "[GCC 7.2.0]\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "print('Python Version %s' % sys.version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package                            Version\r\n",
      "---------------------------------- ----------\r\n",
      "alabaster                          0.7.10\r\n",
      "anaconda-client                    1.6.14\r\n",
      "anaconda-project                   0.8.2\r\n",
      "asn1crypto                         0.24.0\r\n",
      "astroid                            1.6.3\r\n",
      "astropy                            3.0.2\r\n",
      "attrs                              18.1.0\r\n",
      "Automat                            0.3.0\r\n",
      "autovizwidget                      0.15.0\r\n",
      "awscli                             1.18.45\r\n",
      "Babel                              2.5.3\r\n",
      "backcall                           0.1.0\r\n",
      "backports.shutil-get-terminal-size 1.0.0\r\n",
      "bcrypt                             3.1.7\r\n",
      "beautifulsoup4                     4.6.0\r\n",
      "bitarray                           0.8.1\r\n",
      "bkcharts                           0.2\r\n",
      "blaze                              0.11.3\r\n",
      "bleach                             2.1.3\r\n",
      "bokeh                              1.0.4\r\n",
      "boto                               2.48.0\r\n",
      "boto3                              1.12.45\r\n",
      "botocore                           1.15.45\r\n",
      "Bottleneck                         1.2.1\r\n",
      "cached-property                    1.5.1\r\n",
      "certifi                            2019.11.28\r\n",
      "cffi                               1.11.5\r\n",
      "characteristic                     14.3.0\r\n",
      "chardet                            3.0.4\r\n",
      "click                              6.7\r\n",
      "cloudpickle                        0.5.3\r\n",
      "clyent                             1.2.2\r\n",
      "colorama                           0.3.9\r\n",
      "contextlib2                        0.5.5\r\n",
      "cryptography                       2.9.2\r\n",
      "cycler                             0.10.0\r\n",
      "Cython                             0.28.4\r\n",
      "cytoolz                            0.9.0.1\r\n",
      "dask                               0.17.5\r\n",
      "datashape                          0.5.4\r\n",
      "decorator                          4.3.0\r\n",
      "defusedxml                         0.6.0\r\n",
      "distributed                        1.21.8\r\n",
      "docker                             4.2.0\r\n",
      "docker-compose                     1.25.5\r\n",
      "dockerpty                          0.4.1\r\n",
      "docopt                             0.6.2\r\n",
      "docutils                           0.14\r\n",
      "entrypoints                        0.2.3\r\n",
      "enum34                             1.1.9\r\n",
      "environment-kernels                1.1.1\r\n",
      "et-xmlfile                         1.0.1\r\n",
      "fastcache                          1.0.2\r\n",
      "filelock                           3.0.4\r\n",
      "Flask                              1.0.2\r\n",
      "Flask-Cors                         3.0.4\r\n",
      "future                             0.18.2\r\n",
      "gevent                             1.3.0\r\n",
      "glob2                              0.6\r\n",
      "gmpy2                              2.0.8\r\n",
      "greenlet                           0.4.13\r\n",
      "h5py                               2.8.0\r\n",
      "hdijupyterutils                    0.15.0\r\n",
      "heapdict                           1.0.0\r\n",
      "html5lib                           1.0.1\r\n",
      "idna                               2.6\r\n",
      "imageio                            2.3.0\r\n",
      "imagesize                          1.0.0\r\n",
      "importlib-metadata                 1.5.0\r\n",
      "ipykernel                          4.8.2\r\n",
      "ipyparallel                        6.2.2\r\n",
      "ipython                            6.4.0\r\n",
      "ipython-genutils                   0.2.0\r\n",
      "ipywidgets                         7.4.0\r\n",
      "isort                              4.3.4\r\n",
      "itsdangerous                       0.24\r\n",
      "jdcal                              1.4\r\n",
      "jedi                               0.12.0\r\n",
      "Jinja2                             2.10\r\n",
      "jmespath                           0.9.4\r\n",
      "jsonschema                         2.6.0\r\n",
      "jupyter                            1.0.0\r\n",
      "jupyter-client                     5.2.3\r\n",
      "jupyter-console                    5.2.0\r\n",
      "jupyter-core                       4.4.0\r\n",
      "jupyterlab                         0.32.1\r\n",
      "jupyterlab-launcher                0.10.5\r\n",
      "kiwisolver                         1.0.1\r\n",
      "lazy-object-proxy                  1.3.1\r\n",
      "llvmlite                           0.23.1\r\n",
      "locket                             0.2.0\r\n",
      "lxml                               4.2.1\r\n",
      "MarkupSafe                         1.0\r\n",
      "matplotlib                         3.0.3\r\n",
      "mccabe                             0.6.1\r\n",
      "mistune                            0.8.3\r\n",
      "mkl-fft                            1.0.0\r\n",
      "mkl-random                         1.0.1\r\n",
      "mock                               4.0.1\r\n",
      "more-itertools                     4.1.0\r\n",
      "mpmath                             1.0.0\r\n",
      "msgpack                            0.6.0\r\n",
      "msgpack-python                     0.5.6\r\n",
      "multipledispatch                   0.5.0\r\n",
      "nb-conda                           2.2.1\r\n",
      "nb-conda-kernels                   2.2.2\r\n",
      "nbconvert                          5.4.1\r\n",
      "nbformat                           4.4.0\r\n",
      "networkx                           2.1\r\n",
      "nltk                               3.3\r\n",
      "nose                               1.3.7\r\n",
      "notebook                           5.5.0\r\n",
      "numba                              0.38.0\r\n",
      "numexpr                            2.6.5\r\n",
      "numpy                              1.14.3\r\n",
      "numpydoc                           0.8.0\r\n",
      "odo                                0.5.1\r\n",
      "olefile                            0.45.1\r\n",
      "opencv-python                      3.4.2.17\r\n",
      "openpyxl                           2.5.3\r\n",
      "packaging                          20.1\r\n",
      "pandas                             0.24.2\r\n",
      "pandocfilters                      1.4.2\r\n",
      "paramiko                           2.7.1\r\n",
      "parso                              0.2.0\r\n",
      "partd                              0.3.8\r\n",
      "path.py                            11.0.1\r\n",
      "pathlib2                           2.3.2\r\n",
      "patsy                              0.5.0\r\n",
      "pep8                               1.7.1\r\n",
      "pexpect                            4.5.0\r\n",
      "pickleshare                        0.7.4\r\n",
      "Pillow                             5.1.0\r\n",
      "pip                                20.1\r\n",
      "pkginfo                            1.4.2\r\n",
      "plotly                             4.5.2\r\n",
      "pluggy                             0.6.0\r\n",
      "ply                                3.11\r\n",
      "prompt-toolkit                     1.0.15\r\n",
      "protobuf                           3.6.1\r\n",
      "protobuf3-to-dict                  0.1.5\r\n",
      "psutil                             5.4.5\r\n",
      "psycopg2                           2.7.5\r\n",
      "ptyprocess                         0.5.2\r\n",
      "py                                 1.5.3\r\n",
      "py4j                               0.10.7\r\n",
      "pyasn1                             0.4.8\r\n",
      "PyAthena                           1.8.0\r\n",
      "pycodestyle                        2.4.0\r\n",
      "pycosat                            0.6.3\r\n",
      "pycparser                          2.18\r\n",
      "pycrypto                           2.6.1\r\n",
      "pycurl                             7.43.0.1\r\n",
      "pyflakes                           1.6.0\r\n",
      "pygal                              2.4.0\r\n",
      "Pygments                           2.2.0\r\n",
      "pykerberos                         1.2.1\r\n",
      "pylint                             1.8.4\r\n",
      "PyNaCl                             1.3.0\r\n",
      "pyodbc                             4.0.23\r\n",
      "pyOpenSSL                          18.0.0\r\n",
      "pyparsing                          2.2.0\r\n",
      "PySocks                            1.6.8\r\n",
      "pyspark                            2.3.4\r\n",
      "pytest                             3.5.1\r\n",
      "pytest-arraydiff                   0.2\r\n",
      "pytest-astropy                     0.3.0\r\n",
      "pytest-doctestplus                 0.1.3\r\n",
      "pytest-openfiles                   0.3.0\r\n",
      "pytest-remotedata                  0.2.1\r\n",
      "python-dateutil                    2.7.3\r\n",
      "pytz                               2018.4\r\n",
      "PyWavelets                         0.5.2\r\n",
      "PyYAML                             5.3.1\r\n",
      "pyzmq                              17.0.0\r\n",
      "QtAwesome                          0.4.4\r\n",
      "qtconsole                          4.3.1\r\n",
      "QtPy                               1.4.1\r\n",
      "requests                           2.20.0\r\n",
      "requests-kerberos                  0.12.0\r\n",
      "retrying                           1.3.3\r\n",
      "rope                               0.10.7\r\n",
      "rsa                                3.4.2\r\n",
      "ruamel-yaml                        0.15.35\r\n",
      "s3fs                               0.1.5\r\n",
      "s3transfer                         0.3.3\r\n",
      "sagemaker                          1.55.4\r\n",
      "sagemaker-pyspark                  1.3.1\r\n",
      "scikit-image                       0.13.1\r\n",
      "scikit-learn                       0.20.3\r\n",
      "scipy                              1.1.0\r\n",
      "seaborn                            0.8.1\r\n",
      "Send2Trash                         1.5.0\r\n",
      "setuptools                         39.1.0\r\n",
      "simplegeneric                      0.8.1\r\n",
      "singledispatch                     3.4.0.3\r\n",
      "six                                1.11.0\r\n",
      "smdebug-rulesconfig                0.1.2\r\n",
      "snowballstemmer                    1.2.1\r\n",
      "sortedcollections                  0.6.1\r\n",
      "sortedcontainers                   1.5.10\r\n",
      "sparkmagic                         0.12.5\r\n",
      "Sphinx                             1.7.4\r\n",
      "sphinxcontrib-websupport           1.0.1\r\n",
      "spyder                             3.2.8\r\n",
      "SQLAlchemy                         1.2.11\r\n",
      "statsmodels                        0.9.0\r\n",
      "sympy                              1.1.1\r\n",
      "tables                             3.4.3\r\n",
      "tabulate                           0.8.7\r\n",
      "TBB                                0.1\r\n",
      "tblib                              1.3.2\r\n",
      "tenacity                           6.2.0\r\n",
      "terminado                          0.8.1\r\n",
      "testpath                           0.3.1\r\n",
      "texttable                          1.6.2\r\n",
      "toolz                              0.9.0\r\n",
      "tornado                            5.0.2\r\n",
      "traitlets                          4.3.2\r\n",
      "typing                             3.6.4\r\n",
      "unicodecsv                         0.14.1\r\n",
      "urllib3                            1.23\r\n",
      "wcwidth                            0.1.7\r\n",
      "webencodings                       0.5.1\r\n",
      "websocket-client                   0.57.0\r\n",
      "Werkzeug                           0.14.1\r\n",
      "wheel                              0.31.1\r\n",
      "widgetsnbextension                 3.4.2\r\n",
      "wrapt                              1.10.11\r\n",
      "xlrd                               1.1.0\r\n",
      "XlsxWriter                         1.0.4\r\n",
      "xlwt                               1.3.0\r\n",
      "zict                               0.1.3\r\n",
      "zipp                               3.0.0\r\n"
     ]
    }
   ],
   "source": [
    "!pip list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# packages in environment at /home/ec2-user/anaconda3/envs/python3:\n",
      "#\n",
      "# Name                    Version                   Build  Channel\n",
      "_libgcc_mutex             0.1                        main  \n",
      "alabaster                 0.7.10           py36h306e16b_0  \n",
      "anaconda-client           1.6.14                   py36_0  \n",
      "anaconda-project          0.8.2            py36h44fb852_0  \n",
      "argparse                  1.4.0                     <pip>\n",
      "asn1crypto                0.24.0                   py36_0  \n",
      "astroid                   1.6.3                    py36_0  \n",
      "astropy                   3.0.2            py36h3010b51_1  \n",
      "attrs                     18.1.0                   py36_0  \n",
      "Automat                   0.3.0                     <pip>\n",
      "autovizwidget             0.15.0                    <pip>\n",
      "awscli                    1.18.45                   <pip>\n",
      "babel                     2.5.3                    py36_0  \n",
      "backcall                  0.1.0                    py36_0  \n",
      "backports                 1.0              py36hfa02d7e_1  \n",
      "backports.shutil_get_terminal_size 1.0.0            py36hfea85ff_2  \n",
      "bcrypt                    3.1.7                     <pip>\n",
      "beautifulsoup4            4.6.0            py36h49b8c8c_1  \n",
      "bitarray                  0.8.1            py36h14c3975_1  \n",
      "bkcharts                  0.2              py36h735825a_0  \n",
      "blas                      1.0                         mkl  \n",
      "blaze                     0.11.3           py36h4e06776_0  \n",
      "bleach                    2.1.3                    py36_0  \n",
      "blosc                     1.14.3               hdbcaa40_0  \n",
      "bokeh                     1.0.4                     <pip>\n",
      "bokeh                     0.12.16                  py36_0  \n",
      "boto                      2.48.0           py36h6e4cd66_1  \n",
      "boto3                     1.12.0                     py_0  \n",
      "boto3                     1.12.45                   <pip>\n",
      "botocore                  1.15.45                   <pip>\n",
      "botocore                  1.15.0                     py_0  \n",
      "bottleneck                1.2.1            py36haac1ea0_0  \n",
      "bzip2                     1.0.6                h14c3975_5  \n",
      "ca-certificates           2020.1.1                      0  \n",
      "cached-property           1.5.1                     <pip>\n",
      "cairo                     1.14.12              h7636065_2  \n",
      "certifi                   2019.11.28               py36_0  \n",
      "cffi                      1.11.5           py36h9745a5d_0  \n",
      "characteristic            14.3.0                    <pip>\n",
      "chardet                   3.0.4            py36h0f667ec_1  \n",
      "click                     6.7              py36h5253387_0  \n",
      "cloudpickle               0.5.3                    py36_0  \n",
      "clyent                    1.2.2            py36h7e57e65_1  \n",
      "colorama                  0.3.9            py36h489cec4_0  \n",
      "contextlib2               0.5.5            py36h6c84a62_0  \n",
      "cryptography              2.2.2            py36h14c3975_0  \n",
      "cryptography              2.9.2                     <pip>\n",
      "curl                      7.60.0               h84994c4_0  \n",
      "cycler                    0.10.0           py36h93f1223_0  \n",
      "cython                    0.28.2           py36h14c3975_0  \n",
      "Cython                    0.28.4                    <pip>\n",
      "cytoolz                   0.9.0.1          py36h14c3975_0  \n",
      "dask                      0.17.5                   py36_0  \n",
      "dask-core                 0.17.5                   py36_0  \n",
      "datashape                 0.5.4            py36h3ad6b5c_0  \n",
      "dbus                      1.13.2               h714fa37_1  \n",
      "decorator                 4.3.0                    py36_0  \n",
      "defusedxml                0.6.0                      py_0  \n",
      "distributed               1.21.8                   py36_0  \n",
      "docker                    4.2.0                     <pip>\n",
      "docker-compose            1.25.5                    <pip>\n",
      "dockerpty                 0.4.1                     <pip>\n",
      "docopt                    0.6.2                     <pip>\n",
      "docutils                  0.14             py36hb0f60f5_0  \n",
      "entrypoints               0.2.3            py36h1aec115_2  \n",
      "enum34                    1.1.9                     <pip>\n",
      "environment-kernels       1.1.1                     <pip>\n",
      "et_xmlfile                1.0.1            py36hd6bccc3_0  \n",
      "expat                     2.2.5                he0dffb1_0  \n",
      "fastcache                 1.0.2            py36h14c3975_2  \n",
      "filelock                  3.0.4                    py36_0  \n",
      "flask                     1.0.2                    py36_1  \n",
      "flask-cors                3.0.4                    py36_0  \n",
      "fontconfig                2.12.6               h49f89f6_0  \n",
      "freetype                  2.8                  hab7d2ae_1  \n",
      "future                    0.18.2                    <pip>\n",
      "get_terminal_size         1.0.0                haa9412d_0  \n",
      "gevent                    1.3.0            py36h14c3975_0  \n",
      "glib                      2.56.1               h000015b_0  \n",
      "glob2                     0.6              py36he249c77_0  \n",
      "gmp                       6.1.2                h6c8ec71_1  \n",
      "gmpy2                     2.0.8            py36hc8893dd_2  \n",
      "graphite2                 1.3.11               h16798f4_2  \n",
      "graphviz                  2.40.1               h25d223c_0  \n",
      "greenlet                  0.4.13           py36h14c3975_0  \n",
      "gst-plugins-base          1.14.0               hbbd80ab_1  \n",
      "gstreamer                 1.14.0               hb453b48_1  \n",
      "h5py                      2.8.0            py36h989c5e5_3  \n",
      "harfbuzz                  1.7.6                h5f0a787_1  \n",
      "hdf5                      1.10.2               hba1933b_1  \n",
      "hdijupyterutils           0.15.0                    <pip>\n",
      "heapdict                  1.0.0                    py36_2  \n",
      "html5lib                  1.0.1            py36h2f9c1c0_0  \n",
      "icu                       58.2                 h9c2bf20_1  \n",
      "idna                      2.6              py36h82fb2a8_1  \n",
      "imageio                   2.3.0                    py36_0  \n",
      "imagesize                 1.0.0                    py36_0  \n",
      "importlib-metadata        1.5.0                     <pip>\n",
      "intel-openmp              2018.0.0                      8  \n",
      "ipykernel                 4.8.2                    py36_0  \n",
      "ipyparallel               6.2.2                     <pip>\n",
      "ipython                   6.4.0                    py36_0  \n",
      "ipython_genutils          0.2.0            py36hb52b0d5_0  \n",
      "ipywidgets                7.2.1                    py36_0  \n",
      "ipywidgets                7.4.0                     <pip>\n",
      "isort                     4.3.4                    py36_0  \n",
      "itsdangerous              0.24             py36h93cc618_1  \n",
      "jbig                      2.1                  hdba287a_0  \n",
      "jdcal                     1.4                      py36_0  \n",
      "jedi                      0.12.0                   py36_1  \n",
      "jinja2                    2.10             py36ha16c418_0  \n",
      "jmespath                  0.9.4                      py_0  \n",
      "jpeg                      9b                   h024ee3a_2  \n",
      "jsonschema                2.6.0            py36h006f8b5_0  \n",
      "jupyter                   1.0.0                    py36_4  \n",
      "jupyter_client            5.2.3                    py36_0  \n",
      "jupyter_console           5.2.0            py36he59e554_1  \n",
      "jupyter_core              4.4.0            py36h7c827e3_0  \n",
      "jupyterlab                0.32.1                   py36_0  \n",
      "jupyterlab_launcher       0.10.5                   py36_0  \n",
      "kiwisolver                1.0.1            py36h764f252_0  \n",
      "krb5                      1.14.2               hcdc1b81_6  \n",
      "lazy-object-proxy         1.3.1            py36h10fcdad_0  \n",
      "ld_impl_linux-64          2.33.1               h53a641e_7  \n",
      "libcurl                   7.60.0               h1ad7b7a_0  \n",
      "libedit                   3.1.20170329         h6b74fdf_2  \n",
      "libffi                    3.2.1                hd88cf55_4  \n",
      "libgcc-ng                 9.1.0                hdf63c60_0  \n",
      "libgfortran               3.0.0                         1    conda-forge\n",
      "libgfortran-ng            7.2.0                hdf63c60_3  \n",
      "libpng                    1.6.34               hb9fc6fc_0  \n",
      "libsodium                 1.0.16               h1bed415_0  \n",
      "libssh2                   1.8.0                h9cfc8f7_4  \n",
      "libstdcxx-ng              9.1.0                hdf63c60_0  \n",
      "libtiff                   4.0.9                he85c1e1_1  \n",
      "libtool                   2.4.6                h544aabb_3  \n",
      "libxcb                    1.13                 h1bed415_1  \n",
      "libxml2                   2.9.8                h26e45fe_1  \n",
      "libxslt                   1.1.32               h1312cb7_0  \n",
      "llvmlite                  0.23.1           py36hdbcaa40_0  \n",
      "locket                    0.2.0            py36h787c0ad_1  \n",
      "lxml                      4.2.1            py36h23eabaa_0  \n",
      "lzo                       2.10                 h49e0be7_2  \n",
      "markupsafe                1.0              py36hd9260cd_1  \n",
      "matplotlib                3.0.3                     <pip>\n",
      "matplotlib                2.2.2            py36h0e671d2_1  \n",
      "mccabe                    0.6.1            py36h5ad9710_1  \n",
      "mistune                   0.8.3            py36h14c3975_1  \n",
      "mkl                       2018.0.2                      1  \n",
      "mkl-service               1.1.2            py36h17a0993_4  \n",
      "mkl_fft                   1.0.1            py36h3010b51_0  \n",
      "mkl_random                1.0.1            py36h629b387_0  \n",
      "mock                      4.0.1                     <pip>\n",
      "more-itertools            4.1.0                    py36_0  \n",
      "mpc                       1.0.3                hec55b23_5  \n",
      "mpfr                      3.1.5                h11a74b3_2  \n",
      "mpi                       1.0                     openmpi    conda-forge\n",
      "mpmath                    1.0.0            py36hfeacd6b_2  \n",
      "msgpack                   0.6.0                     <pip>\n",
      "msgpack-python            0.5.6            py36h6bb024c_0  \n",
      "multipledispatch          0.5.0                    py36_0  \n",
      "nb_conda                  2.2.1                    py36_0  \n",
      "nb_conda_kernels          2.2.2                    py36_0  \n",
      "nbconvert                 5.4.1                    py36_3  \n",
      "nbformat                  4.4.0            py36h31c9010_0  \n",
      "ncurses                   6.1                  hf484d3e_0  \n",
      "networkx                  2.1                      py36_0  \n",
      "nltk                      3.3.0                    py36_0  \n",
      "nose                      1.3.7            py36hcdf7029_2  \n",
      "notebook                  5.5.0                    py36_0  \n",
      "numba                     0.38.0           py36h637b7d7_0  \n",
      "numexpr                   2.6.5            py36h7bf3b9c_0  \n",
      "numpy                     1.14.3           py36hcd700cb_1  \n",
      "numpy-base                1.14.3           py36h9be14a7_1  \n",
      "numpydoc                  0.8.0                    py36_0  \n",
      "odo                       0.5.1            py36h90ed295_0  \n",
      "olefile                   0.45.1                   py36_0  \n",
      "opencv-python             3.4.2.17                  <pip>\n",
      "openmpi                   3.1.0                h26a2512_3    conda-forge\n",
      "openpyxl                  2.5.3                    py36_0  \n",
      "openssl                   1.0.2u               h7b6447c_0  \n",
      "packaging                 20.1                      <pip>\n",
      "packaging                 17.1                     py36_0  \n",
      "pandas                    0.24.2           py36he6710b0_0  \n",
      "pandoc                    1.19.2.1             hea2e7c5_1  \n",
      "pandocfilters             1.4.2            py36ha6701b7_1  \n",
      "pango                     1.41.0               hd475d92_0  \n",
      "paramiko                  2.7.1                     <pip>\n",
      "parso                     0.2.0                    py36_0  \n",
      "partd                     0.3.8            py36h36fd896_0  \n",
      "patchelf                  0.9                  hf79760b_2  \n",
      "path.py                   11.0.1                   py36_0  \n",
      "pathlib2                  2.3.2                    py36_0  \n",
      "patsy                     0.5.0                    py36_0  \n",
      "pcre                      8.42                 h439df22_0  \n",
      "pep8                      1.7.1                    py36_0  \n",
      "pexpect                   4.5.0                    py36_0  \n",
      "pickleshare               0.7.4            py36h63277f8_0  \n",
      "pillow                    5.1.0            py36h3deb7b8_0  \n",
      "pip                       10.0.1                   py36_0  \n",
      "pip                       20.1                      <pip>\n",
      "pixman                    0.34.0               hceecf20_3  \n",
      "pkginfo                   1.4.2                    py36_1  \n",
      "plotly                    4.5.2                     <pip>\n",
      "pluggy                    0.6.0            py36hb689045_0  \n",
      "ply                       3.11                     py36_0  \n",
      "prompt_toolkit            1.0.15           py36h17d85b1_0  \n",
      "protobuf                  3.6.1                     <pip>\n",
      "protobuf3-to-dict         0.1.5                     <pip>\n",
      "psutil                    5.4.5            py36h14c3975_0  \n",
      "psycopg2                  2.7.5                     <pip>\n",
      "ptyprocess                0.5.2            py36h69acd42_0  \n",
      "py                        1.5.3                    py36_0  \n",
      "py4j                      0.10.7                    <pip>\n",
      "pyasn1                    0.4.8                     <pip>\n",
      "PyAthena                  1.8.0                     <pip>\n",
      "pycodestyle               2.4.0                    py36_0  \n",
      "pycosat                   0.6.3            py36h0a5515d_0  \n",
      "pycparser                 2.18             py36hf9f622e_1  \n",
      "pycrypto                  2.6.1            py36h14c3975_8  \n",
      "pycurl                    7.43.0.1         py36hb7f436b_0  \n",
      "pyflakes                  1.6.0            py36h7bd6a15_0  \n",
      "pygal                     2.4.0                     <pip>\n",
      "pygments                  2.2.0            py36h0d3125c_0  \n",
      "pykerberos                1.2.1            py36h14c3975_0  \n",
      "pylint                    1.8.4                    py36_0  \n",
      "PyNaCl                    1.3.0                     <pip>\n",
      "pyodbc                    4.0.23           py36hf484d3e_0  \n",
      "pyopenssl                 18.0.0                   py36_0  \n",
      "pyparsing                 2.2.0            py36hee85983_1  \n",
      "pyqt                      5.9.2            py36h751905a_0  \n",
      "pysocks                   1.6.8                    py36_0  \n",
      "pyspark                   2.3.4                     <pip>\n",
      "pytables                  3.4.3            py36h02b9ad4_2  \n",
      "pytest                    3.5.1                    py36_0  \n",
      "pytest-arraydiff          0.2                      py36_0  \n",
      "pytest-astropy            0.3.0                    py36_0  \n",
      "pytest-doctestplus        0.1.3                    py36_0  \n",
      "pytest-openfiles          0.3.0                    py36_0  \n",
      "pytest-remotedata         0.2.1                    py36_0  \n",
      "python                    3.6.5                hc3d631a_2  \n",
      "python-dateutil           2.7.3                    py36_0  \n",
      "pytz                      2018.4                   py36_0  \n",
      "pywavelets                0.5.2            py36he602eb0_0  \n",
      "pyyaml                    3.12             py36hafb9ca4_1  \n",
      "PyYAML                    5.3.1                     <pip>\n",
      "pyzmq                     17.0.0           py36h14c3975_0  \n",
      "qt                        5.9.5                h7e424d6_0  \n",
      "qtawesome                 0.4.4            py36h609ed8c_0  \n",
      "qtconsole                 4.3.1            py36h8f73b5b_0  \n",
      "qtpy                      1.4.1                    py36_0  \n",
      "readline                  7.0                  ha6073c6_4  \n",
      "requests                  2.20.0                   py36_0  \n",
      "requests-kerberos         0.12.0                    <pip>\n",
      "retrying                  1.3.3                     <pip>\n",
      "rope                      0.10.7           py36h147e2ec_0  \n",
      "rsa                       3.4.2                     <pip>\n",
      "ruamel_yaml               0.15.35          py36h14c3975_1  \n",
      "s3fs                      0.1.5                    py36_0  \n",
      "s3transfer                0.3.3                    py36_0  \n",
      "sagemaker                 1.55.4                    <pip>\n",
      "sagemaker-pyspark         1.3.1                     <pip>\n",
      "scikit-image              0.13.1           py36h14c3975_1  \n",
      "scikit-learn              0.19.1           py36h7aa7ec6_0  \n",
      "scikit-learn              0.20.3                    <pip>\n",
      "scipy                     1.1.0            py36hfc37229_0  \n",
      "seaborn                   0.8.1            py36hfad7ec4_0  \n",
      "send2trash                1.5.0                    py36_0  \n",
      "setuptools                39.1.0                   py36_0  \n",
      "simplegeneric             0.8.1                    py36_2  \n",
      "singledispatch            3.4.0.3          py36h7a266c3_0  \n",
      "sip                       4.19.8           py36hf484d3e_0  \n",
      "six                       1.11.0           py36h372c433_1  \n",
      "smdebug-rulesconfig       0.1.2                     <pip>\n",
      "snappy                    1.1.7                hbae5bb6_3  \n",
      "snowballstemmer           1.2.1            py36h6febd40_0  \n",
      "sortedcollections         0.6.1                    py36_0  \n",
      "sortedcontainers          1.5.10                   py36_0  \n",
      "sparkmagic                0.12.5                    <pip>\n",
      "sphinx                    1.7.4                    py36_0  \n",
      "sphinxcontrib             1.0              py36h6d0f590_1  \n",
      "sphinxcontrib-websupport  1.0.1            py36hb5cb234_1  \n",
      "spyder                    3.2.8                    py36_0  \n",
      "SQLAlchemy                1.2.11                    <pip>\n",
      "sqlalchemy                1.2.7            py36h6b74fdf_0  \n",
      "sqlite                    3.23.1               he433501_0  \n",
      "statsmodels               0.9.0            py36h3010b51_0  \n",
      "sympy                     1.1.1            py36hc6d1c1c_0  \n",
      "tabulate                  0.8.7                     <pip>\n",
      "tbb                       2018.0.4             h6bb024c_1  \n",
      "tbb4py                    2018.0.4         py36h6bb024c_1  \n",
      "tblib                     1.3.2            py36h34cf8b6_0  \n",
      "tenacity                  6.2.0                     <pip>\n",
      "terminado                 0.8.1                    py36_1  \n",
      "testpath                  0.3.1            py36h8cadb63_0  \n",
      "texttable                 1.6.2                     <pip>\n",
      "tk                        8.6.7                hc745277_3  \n",
      "toolz                     0.9.0                    py36_0  \n",
      "tornado                   5.0.2                    py36_0  \n",
      "traitlets                 4.3.2            py36h674d592_0  \n",
      "typing                    3.6.4                    py36_0  \n",
      "unicodecsv                0.14.1           py36ha668878_0  \n",
      "unixodbc                  2.3.6                h1bed415_0  \n",
      "urllib3                   1.23                     py36_0  \n",
      "wcwidth                   0.1.7            py36hdf4376a_0  \n",
      "webencodings              0.5.1            py36h800622e_1  \n",
      "websocket-client          0.57.0                    <pip>\n",
      "werkzeug                  0.14.1                   py36_0  \n",
      "wheel                     0.31.1                   py36_0  \n",
      "widgetsnbextension        3.2.1                    py36_0  \n",
      "widgetsnbextension        3.4.2                     <pip>\n",
      "wrapt                     1.10.11          py36h28b7045_0  \n",
      "xlrd                      1.1.0            py36h1db9f0c_1  \n",
      "xlsxwriter                1.0.4                    py36_0  \n",
      "xlwt                      1.3.0            py36h7b00a1f_0  \n",
      "xz                        5.2.4                h14c3975_4  \n",
      "yaml                      0.1.7                had09818_2  \n",
      "zeromq                    4.2.5                h439df22_0  \n",
      "zict                      0.1.3            py36h3a3bf81_0  \n",
      "zipp                      3.0.0                     <pip>\n",
      "zlib                      1.2.11               ha838bed_2  \n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "conda list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe src=\"https://aws.amazon.com/eks/\" width=\"800px\" height=\"600px\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%html\n",
    "<iframe src=\"https://aws.amazon.com/eks/\" width=\"800px\" height=\"600px\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Install the CLI tools\n",
    "### Note:  Make sure the SageMaker role has access to create EKS Cluster.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "source ~/.bash_profile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting awscli\n",
      "  Downloading awscli-1.18.55-py2.py3-none-any.whl (3.0 MB)\n",
      "\u001b[K     |████████████████████████████████| 3.0 MB 3.2 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting botocore==1.16.5\n",
      "  Downloading botocore-1.16.5-py2.py3-none-any.whl (6.2 MB)\n",
      "\u001b[K     |████████████████████████████████| 6.2 MB 24.4 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied, skipping upgrade: colorama<0.4.4,>=0.2.5; python_version != \"3.4\" in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from awscli) (0.3.9)\n",
      "Requirement already satisfied, skipping upgrade: s3transfer<0.4.0,>=0.3.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from awscli) (0.3.3)\n",
      "Requirement already satisfied, skipping upgrade: PyYAML<5.4,>=3.10; python_version != \"3.4\" in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from awscli) (5.3.1)\n",
      "Requirement already satisfied, skipping upgrade: rsa<=3.5.0,>=3.1.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from awscli) (3.4.2)\n",
      "Requirement already satisfied, skipping upgrade: docutils<0.16,>=0.10 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from awscli) (0.14)\n",
      "Requirement already satisfied, skipping upgrade: urllib3<1.26,>=1.20; python_version != \"3.4\" in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from botocore==1.16.5->awscli) (1.23)\n",
      "Requirement already satisfied, skipping upgrade: jmespath<1.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from botocore==1.16.5->awscli) (0.9.4)\n",
      "Requirement already satisfied, skipping upgrade: python-dateutil<3.0.0,>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from botocore==1.16.5->awscli) (2.7.3)\n",
      "Requirement already satisfied, skipping upgrade: pyasn1>=0.1.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from rsa<=3.5.0,>=3.1.2->awscli) (0.4.8)\n",
      "Requirement already satisfied, skipping upgrade: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from python-dateutil<3.0.0,>=2.1->botocore==1.16.5->awscli) (1.11.0)\n",
      "\u001b[31mERROR: boto3 1.12.45 has requirement botocore<1.16.0,>=1.15.45, but you'll have botocore 1.16.5 which is incompatible.\u001b[0m\n",
      "Installing collected packages: botocore, awscli\n",
      "Successfully installed awscli-1.18.55 botocore-1.16.5\n",
      "0.18.0\n"
     ]
    }
   ],
   "source": [
    "#### Install `eksctl`\n",
    "# To get started we'll first install the `awscli` and `eksctl` CLI tools. [eksctl](https://eksctl.io) simplifies the process of creating EKS clusters.\n",
    "\n",
    "!pip install awscli --upgrade --user\n",
    "\n",
    "!curl --silent --location \"https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz\" | tar xz -C /tmp\n",
    "\n",
    "!sudo mv /tmp/eksctl /usr/local/bin\n",
    "\n",
    "!eksctl version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 55.1M  100 55.1M    0     0  12.0M      0  0:00:04  0:00:04 --:--:-- 12.5M\n",
      "Client Version: v1.15.10-eks-bac369\n"
     ]
    }
   ],
   "source": [
    "#### Install `kubectl`\n",
    "# `kubectl` is a command line interface for running commands against Kubernetes clusters. \n",
    "# Run the following to install Kubectl\n",
    "\n",
    "!curl --location -o ./kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.15.10/2020-02-22/bin/linux/amd64/kubectl\n",
    "\n",
    "!chmod +x ./kubectl\n",
    "\n",
    "!sudo mv ./kubectl /usr/local/bin\n",
    "\n",
    "!kubectl version --short --client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 33.6M  100 33.6M    0     0  11.1M      0  0:00:03  0:00:03 --:--:-- 11.1M\n",
      "{\"Version\":\"v0.5.0\",\"Commit\":\"1cfe2a90f68381eacd7b6dcfa2bf689e76eb8b4b\"}\n"
     ]
    }
   ],
   "source": [
    "#### Install `aws-iam-authenticator`\n",
    "\n",
    "!curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.15.10/2020-02-22/bin/linux/amd64/aws-iam-authenticator\n",
    "\n",
    "!chmod +x ./aws-iam-authenticator\n",
    "\n",
    "!sudo mv aws-iam-authenticator /usr/local/bin\n",
    "\n",
    "!aws-iam-authenticator version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded plugins: dkms-build-requires, priorities, update-motd, upgrade-helper,\n",
      "              : versionlock\n",
      "amzn-main                                                | 2.1 kB     00:00     \n",
      "amzn-updates                                             | 2.5 kB     00:00     \n",
      "libnvidia-container/x86_64/signature                     |  455 B     00:00     \n",
      "libnvidia-container/x86_64/signature                     | 2.0 kB     00:00 !!! \n",
      "nvidia-container-runtime/x86_64/signature                |  455 B     00:00     \n",
      "nvidia-container-runtime/x86_64/signature                | 2.0 kB     00:00 !!! \n",
      "nvidia-docker/x86_64/signature                           |  455 B     00:00     \n",
      "nvidia-docker/x86_64/signature                           | 2.0 kB     00:00 !!! \n",
      "Package jq-1.5-1.2.amzn1.x86_64 already installed and latest version\n",
      "Package gettext-0.18.1.1-9.1.11.amzn1.x86_64 already installed and latest version\n",
      "Nothing to do\n"
     ]
    }
   ],
   "source": [
    "#### Install jq and envsubst (from GNU gettext utilities) \n",
    "!sudo yum -y install jq gettext"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Verify the binaries are in the path and executable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kubectl in path\n",
      "jq in path\n",
      "envsubst in path\n",
      "Completed\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "for command in kubectl jq envsubst\n",
    "   do\n",
    "    which $command &>/dev/null && echo \"$command in path\" || echo \"$command NOT FOUND\"\n",
    "   done\n",
    "\n",
    "echo \"Completed\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Configure the Environment Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "export AWS_REGION=us-east-2\n",
      "export AWS_CLUSTER_NAME=cluster\n",
      "Completed\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "\n",
    "source ~/.bash_profile\n",
    "\n",
    "export AWS_REGION=$(aws configure get region)\n",
    "echo \"export AWS_REGION=${AWS_REGION}\" | tee -a ~/.bash_profile\n",
    "\n",
    "export AWS_CLUSTER_NAME=cluster\n",
    "echo \"export AWS_CLUSTER_NAME=${AWS_CLUSTER_NAME}\" | tee -a ~/.bash_profile\n",
    "\n",
    "echo \"Completed\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Create the EKS Cluster\n",
    "\n",
    "_Note:  Creating a cluster may take about 15 minutes.  Please be patient._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "source ~/.bash_profile\n",
    "\n",
    "cat << EOF > cluster.yaml\n",
    "---\n",
    "apiVersion: eksctl.io/v1alpha5\n",
    "kind: ClusterConfig\n",
    "\n",
    "metadata:\n",
    "  name: ${AWS_CLUSTER_NAME}\n",
    "  region: ${AWS_REGION}\n",
    "\n",
    "cloudWatch:\n",
    "  clusterLogging:\n",
    "    enableTypes: [\"*\"]\n",
    "\n",
    "managedNodeGroups:\n",
    "- name: cpu-nodes\n",
    "  instanceType: c5.xlarge\n",
    "  volumeSize: 100\n",
    "  desiredCapacity: 5\n",
    "  iam:\n",
    "    withAddonPolicies:\n",
    "      albIngress: true\n",
    "\n",
    "#secretsEncryption:\n",
    "#  keyARN: ${MASTER_ARN}\n",
    "EOF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---\r\n",
      "apiVersion: eksctl.io/v1alpha5\r\n",
      "kind: ClusterConfig\r\n",
      "\r\n",
      "metadata:\r\n",
      "  name: cluster\r\n",
      "  region: us-east-2\r\n",
      "\r\n",
      "cloudWatch:\r\n",
      "  clusterLogging:\r\n",
      "    enableTypes: [\"*\"]\r\n",
      "\r\n",
      "managedNodeGroups:\r\n",
      "- name: cpu-nodes\r\n",
      "  instanceType: c5.xlarge\r\n",
      "  volumeSize: 100\r\n",
      "  desiredCapacity: 5\r\n",
      "  iam:\r\n",
      "    withAddonPolicies:\r\n",
      "      albIngress: true\r\n",
      "\r\n",
      "#secretsEncryption:\r\n",
      "#  keyARN: \r\n"
     ]
    }
   ],
   "source": [
    "!cat cluster.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "YOu may need to create role CreateStack "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ℹ]  eksctl version 0.18.0\n",
      "[ℹ]  using region us-east-2\n",
      "[ℹ]  setting availability zones to [us-east-2b us-east-2a us-east-2c]\n",
      "[ℹ]  subnets for us-east-2b - public:192.168.0.0/19 private:192.168.96.0/19\n",
      "[ℹ]  subnets for us-east-2a - public:192.168.32.0/19 private:192.168.128.0/19\n",
      "[ℹ]  subnets for us-east-2c - public:192.168.64.0/19 private:192.168.160.0/19\n",
      "[ℹ]  using Kubernetes version 1.15\n",
      "[ℹ]  creating EKS cluster \"cluster\" in \"us-east-2\" region with managed nodes\n",
      "[ℹ]  1 nodegroup (cpu-nodes) was included (based on the include/exclude rules)\n",
      "[ℹ]  will create a CloudFormation stack for cluster itself and 0 nodegroup stack(s)\n",
      "[ℹ]  will create a CloudFormation stack for cluster itself and 1 managed nodegroup stack(s)\n",
      "[ℹ]  if you encounter any issues, check CloudFormation console or try 'eksctl utils describe-stacks --region=us-east-2 --cluster=cluster'\n",
      "[ℹ]  Kubernetes API endpoint access will use default of {publicAccess=true, privateAccess=false} for cluster \"cluster\" in \"us-east-2\"\n",
      "[ℹ]  3 sequential tasks: { create cluster control plane \"cluster\", create managed nodegroup \"cpu-nodes\", update CloudWatch logging configuration }\n",
      "[ℹ]  building cluster stack \"eksctl-cluster-cluster\"\n",
      "[ℹ]  deploying stack \"eksctl-cluster-cluster\"\n",
      "[✖]  unexpected status \"ROLLBACK_IN_PROGRESS\" while waiting for CloudFormation stack \"eksctl-cluster-cluster\"\n",
      "[ℹ]  fetching stack events in attempt to troubleshoot the root cause of the failure\n",
      "[✖]  AWS::IAM::Role/ServiceRole: CREATE_FAILED – \"Resource creation cancelled\"\n",
      "[✖]  AWS::EC2::EIP/NATIP: CREATE_FAILED – \"API: ec2:allocateAddress You are not authorized to perform this operation.\"\n",
      "[✖]  AWS::EC2::InternetGateway/InternetGateway: CREATE_FAILED – \"API: ec2:CreateInternetGateway You are not authorized to perform this operation.\"\n",
      "[✖]  AWS::EC2::VPC/VPC: CREATE_FAILED – \"API: ec2:CreateVpc You are not authorized to perform this operation.\"\n",
      "[ℹ]  1 error(s) occurred and cluster hasn't been created properly, you may wish to check CloudFormation console\n",
      "[ℹ]  to cleanup resources, run 'eksctl delete cluster --region=us-east-2 --name=cluster'\n",
      "[✖]  waiting for CloudFormation stack \"eksctl-cluster-cluster\": ResourceNotReady: failed waiting for successful resource state\n",
      "Completed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Error: failed to create cluster \"cluster\"\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "\n",
    "source ~/.bash_profile\n",
    "\n",
    "eksctl create cluster -f ./cluster.yaml\n",
    "\n",
    "echo \"Completed\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}