examples/jupyter_jobs_handle_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Jobs Handling Example\n",
    "Show the functions relates to the stopping of Jobs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import asyncio\n",
    "import time\n",
    "\n",
    "from decanter import core\n",
    "from decanter.core.core_api import TrainInput, PredictInput, SetupInput\n",
    "from decanter.core.jobs import DataUpload, Experiment\n",
    "from decanter.core.extra import CoreStatus\n",
    "from decanter.core.enums.algorithms import Algo\n",
    "from decanter.core.enums.evaluators import Evaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "core.enable_default_logger()\n",
    "\n",
    "# The Main object handles the calling of Decanter's API.\n",
    "# Create connection to Decanter server, and set up basic settings.\n",
    "# set the username, password, host\n",
    "client = core.CoreClient(username=???, password=???, host=???)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file_path = 'data/train.csv'\n",
    "test_file_path = 'data/test.csv'\n",
    "train_file = open(train_file_path , 'rb')\n",
    "test_file = open(test_file_path , 'rb')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Job.stop() in diff status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = client.upload(file=train_file, name='stop_when_running')\n",
    "\n",
    "async def stop_when_running():\n",
    "    while train_data.status == 'pending':\n",
    "        await asyncio.sleep(3)\n",
    "    train_data.stop()\n",
    "    return\n",
    "\n",
    "await stop_when_running()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Job failed if it is stopped in pending status."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = DataUpload(name='stop_when_pending')\n",
    "train_data.stop()\n",
    "print(train_data.status)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Job remain failed if it is stopped in pending status."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = DataUpload(name='stop_when_failed')\n",
    "train_data.stop()\n",
    "print(train_data.status)\n",
    "train_data.stop()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test dependency\n",
    "Stop DataUpload, then the Experiemnt and predict depends on the data will fail. Test data will keep upload till it's done."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file = open(train_file_path , 'rb')\n",
    "test_file = open(test_file_path, 'rb')\n",
    "\n",
    "train_data = client.upload(file=train_file, name='upload_train_data')\n",
    "test_data = client.upload(file=test_file, name='upload_test_data')\n",
    "\n",
    "setup_input = SetupInput(\n",
    "        data = train_data,\n",
    "        # data_id=train_data.id,\n",
    "        data_source=train_data.accessor,\n",
    "        data_columns=[\n",
    "            {\n",
    "                'id': 'Pclass',\n",
    "                'data_type': 'categorical'\n",
    "            }]\n",
    "        )\n",
    "\n",
    "train_data = client.setup(setup_input)\n",
    "    \n",
    "train_input = TrainInput(data=train_data, target='Survived', algos=[Algo.XGBoost], max_model=2, tolerance=0.9)\n",
    "exp = client.train(train_input=train_input, select_model_by=Evaluator.auc, name='exp(data_failed)')\n",
    "\n",
    "predict_input = PredictInput(data=test_data, experiment=exp)\n",
    "\n",
    "# predict result\n",
    "pred_res = client.predict(predict_input=predict_input, name='pred(exp_failed)')\n",
    "    \n",
    "async def stop():\n",
    "    while train_data.status == CoreStatus.PENDING:\n",
    "        await asyncio.sleep(3)\n",
    "    \n",
    "    train_data.stop()\n",
    "    return\n",
    "\n",
    "await stop()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test stop jobs list\n",
    "Stop list of jobs, the jobs exclude in list will keep continue till done."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file = open(train_file_path , 'rb')\n",
    "\n",
    "train_data = client.upload(file=train_file, name='upload_train_data')\n",
    "train_input = TrainInput(data=train_data, target='Survived', algos=[Algo.XGBoost], max_model=2, tolerance=0.9)\n",
    "\n",
    "exps = []\n",
    "for i in range(5):\n",
    "    exp = client.train(train_input=train_input, select_model_by=Evaluator.auc, name='exp_%d' % i)\n",
    "    exps.append(exp)\n",
    "\n",
    "time.sleep(4)\n",
    "client.stop_jobs([exps[0], exps[2], exps[4]])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get status of all the Jobs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "client.get_jobs_status()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test stop all jobs\n",
    "Stop all jobs while data is done, all the experiment should be stopped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file = open(train_file_path , 'rb')\n",
    "\n",
    "train_data = client.upload(file=train_file, name='upload_train_data')\n",
    "\n",
    "train_input = TrainInput(data=train_data, target='Survived', algos=[Algo.XGBoost], max_model=2, tolerance=0.9)\n",
    "\n",
    "exps = []\n",
    "for i in range(5):\n",
    "    exp = client.train(train_input=train_input, select_model_by=Evaluator.auc, name='exp_%d' % i)\n",
    "    exps.append(exp)\n",
    "\n",
    "async def stop():\n",
    "    while train_data.status not in CoreStatus.DONE_STATUS:\n",
    "        await asyncio.sleep(3)\n",
    "    \n",
    "    client.stop_all_jobs()\n",
    "    return\n",
    "\n",
    "await stop()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "client.get_jobs_status()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}