Start_Dask_Cluster_Nebari.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# nebari.esipfed.org Dask Cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import logging\n",
    "\n",
    "try:\n",
    "    from dask_gateway import Gateway\n",
    "except ImportError:\n",
    "    logging.error(\"Unable to import Dask Gateway.  Are you running in a cloud compute environment?\\n\")\n",
    "    raise\n",
    "os.environ['DASK_DISTRIBUTED__SCHEDULER__WORKER_SATURATION'] = \"1.0\"\n",
    "\n",
    "gateway = Gateway()\n",
    "_options = gateway.cluster_options()\n",
    "_options.conda_environment='users/users-pangeo'  ##<< this is the conda environment we use on nebari.\n",
    "_options.profile = 'Small Worker'\n",
    "_env_to_add={}\n",
    "aws_env_vars=['AWS_ACCESS_KEY_ID',\n",
    "              'AWS_SECRET_ACCESS_KEY',\n",
    "              'AWS_SESSION_TOKEN',\n",
    "              'AWS_DEFAULT_REGION',\n",
    "              'AWS_S3_ENDPOINT']\n",
    "for _e in aws_env_vars:\n",
    "    if _e in os.environ:\n",
    "        _env_to_add[_e] = os.environ[_e]\n",
    "_options.environment_vars = _env_to_add    \n",
    "cluster = gateway.new_cluster(_options)          ##<< create cluster via the dask gateway\n",
    "cluster.adapt(minimum=2, maximum=30)             ##<< Sets scaling parameters. \n",
    "\n",
    "client = cluster.get_client()\n",
    "\n",
    "print(\"The 'cluster' object can be used to adjust cluster behavior.  i.e. 'cluster.adapt(minimum=10)'\")\n",
    "print(\"The 'client' object can be used to directly interact with the cluster.  i.e. 'client.submit(func)' \")\n",
    "print(f\"The link to view the client dashboard is:\\n>  {client.dashboard_link}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "users-users-pangeo",
   "language": "python",
   "name": "conda-env-users-users-pangeo-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "d7ebce313f85fb1ac8949e834c83f371584cb2422d845bf1570c1220fdedc716"
   }
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}