2019-11-17-pp-Predict.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Find new Nets by looking at highest probability candidate from the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-18T14:47:05.932556Z",
     "start_time": "2019-11-18T14:47:03.662144Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"  # specify which GPU(s) to be used\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.backends.cudnn as cudnn\n",
    "cudnn.benchmark = True\n",
    "\n",
    "import geopandas as gpd\n",
    "import glob\n",
    "from PIL import Image\n",
    "import albumentations\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import seaborn as sns\n",
    "sns.set_style(\"white\")\n",
    "\n",
    "from train import predict, SalmonDataset\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))\n",
    "\n",
    "data_path = \"/media/slowSSD/Carribean_Challenge/data/fish\"\n",
    "path_to_visuals = \"/media/austausch/popien/MachineLearning/CharlesProject/FishProject/visuals\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict new random test images and find new nets\n",
    "- Sort new random images by descending probability of fish net\n",
    "- Visualize, find new nets and copy their coordinates into \"fish_coordinates\" in crawling.py "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-18T14:42:56.372409Z",
     "start_time": "2019-11-18T14:42:56.247107Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7515 test images\n"
     ]
    }
   ],
   "source": [
    "# sz = (450, 432)\n",
    "sz = (900, 864)\n",
    "random_crop = 448\n",
    "test_images = glob.glob(\"{}/no_nets/*.png\".format(data_path = \"/media/slowSSD/Carribean_Challenge/data/fish\"))\n",
    "print(\"{:3} test images\".format(len(test_images)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-18T14:42:56.420726Z",
     "start_time": "2019-11-18T14:42:56.374390Z"
    }
   },
   "outputs": [],
   "source": [
    "epochs = 1000\n",
    "print_every_x_batches = 1000\n",
    "lr = 0.001\n",
    "arch = \"resnet18\"\n",
    "\n",
    "val_transforms = []\n",
    "val_transforms.append(albumentations.Resize(sz[1], sz[0], interpolation=cv2.INTER_CUBIC))\n",
    "val_dataset = SalmonDataset(images=test_images, test=True, transforms=albumentations.Compose(val_transforms)) \n",
    "\n",
    "val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=64, shuffle=False, num_workers=10, pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-18T14:42:56.467403Z",
     "start_time": "2019-11-18T14:42:56.422245Z"
    }
   },
   "outputs": [],
   "source": [
    "model_name = \"resnet18_94nets_6401wo_val_v1_resize_900x864_rc_448_pat3\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-18T14:35:58.358937Z",
     "start_time": "2019-11-18T14:34:07.500539Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet18_94nets_6401wo_val_v1_resize_900x864_rc_448_pat3 with loss 0.0266 at epoch 27\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:48<00:00,  1.09it/s]\n"
     ]
    }
   ],
   "source": [
    "preds = predict(model_name, val_loader, infrared=False, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-11-18T14:45:55.258Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "how_many = 30\n",
    "count = 0\n",
    "skip = 0\n",
    "\n",
    "for j in np.argsort(preds)[::-1]:\n",
    "    count += 1\n",
    "    if count < skip: continue\n",
    "    rand_int = j\n",
    "    f, axarr = plt.subplots(1, 2, figsize=(30, 20))\n",
    "#     plt.figure(figsize=(12, 12))\n",
    "    img = Image.open(test_images[rand_int])\n",
    "    axarr[0].imshow(img)\n",
    "    axarr[0].set_title(test_images[rand_int])\n",
    "    sample = val_dataset[rand_int]\n",
    "    axarr[1].imshow(np.transpose((sample[\"image\"].numpy()*255).astype(np.uint8), (1,2,0)))\n",
    "    axarr[1].set_title(\"{:.1f}% for Net\".format(preds[rand_int]*100))\n",
    "    print(test_images[rand_int].split(\"/\")[-1])\n",
    "    plt.show()\n",
    "    if count>how_many+skip:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In QGIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-18T14:47:11.643665Z",
     "start_time": "2019-11-18T14:47:11.475035Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7514"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coords = [j.split(\"/\")[-1].split(\"_\")[1] for j in glob.glob(\"{}/no_nets/*\".format(data_path))]\n",
    "no_net_coords = []\n",
    "for coord in coords:\n",
    "    lat, long = map(float, coord.split(\",\"))\n",
    "    no_net_coords.append((lat, long))\n",
    "len(no_net_coords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-18T14:47:16.049142Z",
     "start_time": "2019-11-18T14:47:15.752916Z"
    }
   },
   "outputs": [],
   "source": [
    "no_net_coords = pd.DataFrame({\"lat\": [k for k,v in no_net_coords], \"long\": [v for k,v in no_net_coords]})\n",
    "\n",
    "net_coords = pd.DataFrame({\"lat\": [k for k,v in fish_coordinates], \"long\": [v for k,v in fish_coordinates]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-18T14:47:19.715422Z",
     "start_time": "2019-11-18T14:47:16.696080Z"
    }
   },
   "outputs": [],
   "source": [
    "os.makedirs(path_to_visuals, exist_ok=True)\n",
    "gdf = gpd.GeoDataFrame({\"class\": [\"net_present\"]*len(net_coords) + [\"no_net\"]*len(no_net_coords), \"geometry\": gpd.points_from_xy(net_coords.long, net_coords.lat) + gpd.points_from_xy(no_net_coords.long, no_net_coords.lat)})\n",
    "gdf.crs = \"EPSG:4326\"\n",
    "gdf.to_file(f\"{path_to_visuals}/output.shp\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Print Net coordinates to copy paste into http://www.copypastemap.com/index.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-18T12:37:15.479595Z",
     "start_time": "2019-11-18T12:37:15.424059Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "52.591901\t-128.521975\n",
      "50.882396\t-127.49693\n",
      "50.878538\t-126.902387\n",
      "50.854201\t-126.758715\n",
      "50.850617\t-126.716857\n",
      "50.847869\t-126.319231\n",
      "50.837714\t-126.663131\n",
      "50.832302\t-127.520585\n",
      "50.831666\t-126.597147\n",
      "50.796923\t-126.495746\n",
      "50.749069\t-127.683667\n",
      "50.746123\t-127.647787\n",
      "50.743125\t-127.448952\n",
      "50.733599\t-127.444264\n",
      "50.707619\t-126.664039\n",
      "50.69787\t-126.25668\n",
      "50.670949\t-126.476713\n",
      "50.656792\t-126.665785\n",
      "50.653974\t-126.28905\n",
      "50.649603\t-126.618208\n",
      "50.618472\t-126.704306\n",
      "50.607944\t-126.363262\n",
      "50.601005\t-126.348642\n",
      "50.566306\t-126.274258\n",
      "50.594348\t-127.571985\n",
      "50.488321\t-125.35654\n",
      "50.4857\t-127.876225\n",
      "50.474602\t-127.787872\n",
      "50.473781\t-125.808549\n",
      "50.468923\t-127.835091\n",
      "50.458708\t-127.890031\n",
      "50.425624\t-125.305282\n",
      "50.414876\t-125.768263\n",
      "50.41463\t-125.659615\n",
      "50.409818\t-125.700438\n",
      "50.408549\t-125.340233\n",
      "50.392891\t-125.362241\n",
      "50.349968\t-125.34287\n",
      "50.324312\t-125.26048\n",
      "50.309412\t-125.316563\n",
      "50.302343\t-125.338198\n",
      "50.286035\t-125.349444\n",
      "50.178898\t-125.327057\n",
      "50.177378\t-125.344332\n",
      "50.046627\t-127.296477\n",
      "50.037817\t-127.176002\n",
      "50.020139\t-127.19364\n",
      "50.028678\t-124.717693\n",
      "49.886826\t-126.791493\n",
      "49.87972\t-126.760589\n",
      "49.867223\t-126.759452\n",
      "49.854614\t-124.227246\n",
      "49.854365\t-124.218791\n",
      "49.854393\t-124.231923\n",
      "49.835393\t-124.055283\n",
      "49.796591\t-124.097778\n",
      "49.779334\t-124.153914\n",
      "49.672485\t-124.086384\n",
      "49.670991\t-123.861535\n",
      "49.647667\t-123.724143\n",
      "49.64008\t-123.658476\n",
      "49.635475\t-123.782979\n",
      "49.628771\t-123.845134\n",
      "49.615316\t-123.834076\n",
      "49.598437\t-123.787204\n",
      "49.565561\t-123.781296\n",
      "49.512665\t-123.491802\n",
      "49.459149\t-126.235764\n",
      "49.404472\t-126.152111\n",
      "49.375583\t-126.091917\n",
      "49.341726\t-125.952694\n",
      "49.327123\t-125.798584\n",
      "49.324218\t-126.048529\n",
      "49.307867\t-126.007018\n",
      "49.305597\t-125.81607\n",
      "49.294125\t-126.070052\n",
      "49.279077\t-125.830551\n",
      "49.2654\t-125.812512\n",
      "49.258769\t-125.870399\n",
      "49.256889\t-125.841672\n",
      "49.247028\t-125.905806\n",
      "49.235057\t-125.751812\n",
      "49.214303\t-125.767151\n",
      "49.165746\t-125.769318\n",
      "49.133107\t-125.783026\n",
      "49.129157\t-125.753921\n",
      "49.119247\t-125.725661\n",
      "49.014239\t-125.030291\n",
      "48.943252\t-124.987273\n",
      "48.938815\t-124.991092\n",
      "48.814243\t-124.667558\n",
      "48.582611\t-124.348857\n",
      "52.798845\t-128.401589\n",
      "52.795894\t-128.31154\n",
      "52.785498\t-128.331592\n",
      "52.785389\t-128.414116\n",
      "50.965119\t-127.453544\n",
      "50.906201\t-127.341426\n",
      "50.821342\t-127.555\n",
      "50.721725\t-126.56806\n",
      "50.52159\t-128.008791\n",
      "50.533085\t-126.226017\n",
      "49.862455\t-124.224756\n",
      "49.85803\t-124.215966\n"
     ]
    }
   ],
   "source": [
    "coords = pd.DataFrame({\"lat\": [k for k,v in fish_coordinates], \"long\": [v for k,v in fish_coordinates]})\n",
    "for j in range(len(coords)):\n",
    "    print(\"{}\\t{}\".format(coords[\"lat\"].iloc[j], coords[\"long\"].iloc[j]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}