Data_extraction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Data_extraction.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/SahilSunda/ML_Project/blob/master/Data_extraction.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lNXWPzSZDQZa"
      },
      "source": [
        "import urllib\n",
        "import json\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from PIL import Image"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IzPGl0FZrStk",
        "outputId": "d39bd256-e956-4b7a-e664-5213f3b7cfcd"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iMK95WbXcHxb",
        "outputId": "daf433d3-c3d6-4805-d3b0-2723f0fdd40f"
      },
      "source": [
        "crops = {}\n",
        "crops_data = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/Khetipoint/iitk_crop.csv')\n",
        "#crops_data = pd.read_csv('/content/drive/MyDrive/iitk_crop.csv')\n",
        "#cropid = []\n",
        "cropid = crops_data['_id']\n",
        "cropname = crops_data['name.en']\n",
        "#print(len(crops_data))\n",
        "for i in range(len(crops_data)):\n",
        "  crops[cropid[i]] = cropname[i]\n",
        "print(crops)\n",
        "#print(crops_data['_id'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'5fc4fd0f00dd0a5e9ddb8abc': 'Cotton', '5fc4fd1100dd0a5e9ddb8abd': 'Cumin', '5fc4fd1200dd0a5e9ddb8abe': 'Coriander ', '5fc4fd1300dd0a5e9ddb8abf': 'Potato', '5fc4fd1500dd0a5e9ddb8ac0': 'Onion', '5fc4fd1600dd0a5e9ddb8ac1': 'Garlic', '5fc4fd1800dd0a5e9ddb8ac2': 'Gram', '5fdf686e36deb42315324603': 'Maize', '5fe21e9489d67d51b7331c41': 'Fennel', '5fe2284c89d67d51b7331c49': 'Sesame', '5fe22d1489d67d51b7331c4a': 'Groundnut', '5fe333a789d67d51b7331d4a': 'Paddy', '5fe33be589d67d51b7331d53': 'Brinjal', '5fe340ce89d67d51b7331d5b': 'Tomato', '5fe346e289d67d51b7331d77': 'Chilli', '5fe46b5889d67d51b7331e0f': 'Okra', '5fe4734789d67d51b7331e11': 'Cabbage', '5fe48c8189d67d51b7331e16': 'Cauliflower', '5fe5aa5189d67d51b7331e82': 'Pigeon Pea (Tur)', '5fe5b2a789d67d51b7331e99': 'Green Gram', '604c728fcd47936a4cb87e02': 'Bajara (Pearl millet)', '604ca272cd47936a4cb87e26': 'Castor', '604cc814cd47936a4cb87e2f': 'Sorghum', '604ccf2dcd47936a4cb87e32': 'Coconut', '604cff8fcd47936a4cb87e40': 'Water Melon', '604d0790cd47936a4cb87e43': 'Musk Melon ', '604d1320cd47936a4cb87e44': 'Black Gram', '604d889bcd47936a4cb87e5f': 'Bitter Guard', '604d95f5cd47936a4cb87e61': 'Cucumber', '604dac9fcd47936a4cb87e6c': 'Soyabean', '604dc162cd47936a4cb87e6f': 'Pea', '604de291cd47936a4cb87e7b': 'Sugarcane', '604f3c30cd47936a4cb87ebb': 'Pointed Guard'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 181
        },
        "id": "8_NM9WdtLww3",
        "outputId": "73972d49-5de8-4707-d7ec-9f83e42c085c"
      },
      "source": [
        "content1.clear()\n",
        "print(content1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-4-61eda4e40561>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcontent1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontent1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'content1' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YtqVMnUA5ilx"
      },
      "source": [
        "f = open(\"/content/drive/MyDrive/Colab Notebooks/Khetipoint/iitk_query.json\")\n",
        "#f = open('/content/drive/MyDrive/iitk_query.json')\n",
        "data = json.load(f)\n",
        "#print(len(data))\n",
        "#print(data[0]['cropIds'])\n",
        "content1 = { 'imageurl': [], 'id': [], 'createdon': [], 'description': [] }\n",
        "temp = []\n",
        "for i in range(len(data)):\n",
        "  flag = list(data[i]['medias'].values())\n",
        "  #print(i)\n",
        "  if(not bool(data[i]['medias'])==True):\n",
        "    continue\n",
        "  if(not bool(data[i]['medias'][flag[0]['fid']]['urls'])==True):\n",
        "    continue\n",
        "  if(not bool(data[i]['cropIds'])==True):\n",
        "    continue\n",
        "  d = data[i]['medias'][flag[0]['fid']]['urls'].keys()\n",
        "  temp = 'main' in d\n",
        "  if(temp==False):\n",
        "    continue\n",
        "  content1['imageurl'].append(data[i]['medias'][flag[0]['fid']]['urls']['main'])\n",
        "  content1['id'].append(data[i]['cropIds'])\n",
        "  content1['createdon'].append(data[i]['createdOn']['$date'])\n",
        "  content1['description'].append(data[i]['description'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pp_JdzqH6nQv",
        "outputId": "07753e31-7eba-4825-b920-775ee8f4811d"
      },
      "source": [
        "print(len(content1['imageurl']))\n",
        "print(len(data))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "4740\n",
            "9509\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8tVTRMDQtXb3",
        "outputId": "fd38483a-218e-4054-f1d3-52d2dff6ffbf"
      },
      "source": [
        "filename = str(content1['id'][0][0])\n",
        "filename = crops[filename]\n",
        "print(filename)\n",
        "print(len(crops_data['name.en']))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Potato\n",
            "33\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NqfziBs9yk1P"
      },
      "source": [
        "import os\n",
        "for i in range(len(crops_data['name.en'])):\n",
        "  path = '/content/drive/MyDrive/Data2/' + str(crops_data['name.en'][i])\n",
        "  os.mkdir(path)\n",
        "\n",
        "#please don't run this again and again it is for forming folder on your drive"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FFlHB-x9vA_9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 232
        },
        "outputId": "3cea7ced-ac58-43d2-e76c-a2c1a0f6b487"
      },
      "source": [
        "import urllib.request\n",
        "import requests\n",
        "flag=0\n",
        "flag1=0\n",
        "flag2=0\n",
        "flag3=0\n",
        "flag4=0\n",
        "flag5=0\n",
        "flag6=0\n",
        "#flag7=0\n",
        "flag8=0\n",
        "flag9=0\n",
        "flag10=0\n",
        "flag11=0\n",
        "flag12=0\n",
        "flag13=0\n",
        "flag14=0\n",
        "flag15=0\n",
        "flag16=0\n",
        "flag17=0\n",
        "flag18=0\n",
        "flag19=0\n",
        "flag20=0\n",
        "flag21=0\n",
        "flag22=0\n",
        "flag23=0\n",
        "flag24=0\n",
        "flag25=0\n",
        "flag26=0\n",
        "flag27=0\n",
        "flag28=0\n",
        "flag29=0\n",
        "flag30=0\n",
        "flag31=0\n",
        "flag32=0\n",
        "flag33=0\n",
        "flag34=0\n",
        "\n",
        "for i in range(0, 6002):\n",
        "  # setting filename and image URL\n",
        "  filename = str(content1['id'][i][0])\n",
        "  filename = crops[filename]\n",
        "  image_url = str(content1['imageurl'][i])\n",
        "\n",
        "  responsecode = requests.get(content1['imageurl'][i].get_attribute('href')).status_code \n",
        "  #print((content1['imageurl'][i].get_attribute('href'))+\" --- \"+str(responsecode)) \n",
        "  \n",
        "  if (responsecode==200): \n",
        "    if filename == \"Bajara (Pearl millet)\":\n",
        "      flag1 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag1) + '.jpg')\n",
        "    elif filename == \"Cotton\":\n",
        "      flag2 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag2) + '.jpg')\n",
        "    elif filename == \"Cumin\":\n",
        "      flag3 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag3) + '.jpg')\n",
        "    elif filename == \"Coriander\":\n",
        "      flag4 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag4) + '.jpg')\n",
        "    elif filename == \"Potato\":\n",
        "      flag5 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag5) + '.jpg')\n",
        "    elif filename == \"Onion\":\n",
        "      flag6 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag6) + '.jpg')\n",
        "    elif filename == \"Garlic\":\n",
        "      flag8 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag8) + '.jpg')\n",
        "    elif filename == \"Gram\":\n",
        "      flag9 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag9) + '.jpg')\n",
        "    elif filename == \"Maize\":\n",
        "      flag10 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag10) + '.jpg')\n",
        "    elif filename == \"Fennel\":\n",
        "      flag11 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag11) + '.jpg')\n",
        "    elif filename == \"Sesame\":\n",
        "      flag12 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag12) + '.jpg')\n",
        "    elif filename == \"Groundnut\":\n",
        "      flag13 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag13) + '.jpg')\n",
        "    elif filename == \"Paddy\":\n",
        "      flag14 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag14) + '.jpg')\n",
        "    elif filename == \"Brinjal\":\n",
        "      flag15 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag15) + '.jpg')\n",
        "    elif filename == \"Tomato\":\n",
        "      flag16 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag16) + '.jpg')\n",
        "    elif filename == \"Chilli\":\n",
        "      flag17 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag17) + '.jpg')\n",
        "    elif filename == \"Okra\":\n",
        "      flag18 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag18) + '.jpg')\n",
        "    elif filename == \"Cabbage\":\n",
        "      flag19 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag19) + '.jpg')\n",
        "    elif filename == \"Cauliflower\":\n",
        "      flag20 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag20) + '.jpg')\n",
        "    elif filename == \"Pigeon Pea (Tur)\":\n",
        "      flag21 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag21) + '.jpg')\n",
        "    elif filename == \"Green Gram\":\n",
        "      flag22 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag22) + '.jpg')\n",
        "    elif filename == \"Sorghum\":\n",
        "      flag23 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag23) + '.jpg')\n",
        "    elif filename == \"Castor\":\n",
        "      flag24 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag24) + '.jpg')\n",
        "    elif filename == \"Coconut\":\n",
        "      flag25 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag25) + '.jpg')\n",
        "    elif filename == \"Water Melon\":\n",
        "      flag26 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag26) + '.jpg')\n",
        "    elif filename == \"Musk Melon\":\n",
        "      flag27 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag27) + '.jpg')\n",
        "    elif filename == \"Black Gram\":\n",
        "      flag28 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag28) + '.jpg')\n",
        "    elif filename == \"Bitter Guard\":\n",
        "      flag29 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag29) + '.jpg')\n",
        "    elif filename == \"Cucumber\":\n",
        "      flag30 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag30) + '.jpg')\n",
        "    elif filename == \"Soyabean\":\n",
        "      flag31 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag31) + '.jpg')\n",
        "    elif filename == \"Pea\":\n",
        "      flag32 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag32) + '.jpg')\n",
        "    elif filename == \"Sugarcane\":\n",
        "      flag33 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag33) + '.jpg')\n",
        "    elif filename == \"Pointed Guard\":\n",
        "      flag34 += 1\n",
        "      urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2/' + str(filename) + \"/\" + str(flag34) + '.jpg')\n",
        "    else: continue\n",
        "    \n",
        "  else: \n",
        "    print(\"broken/invalid link\")\n",
        "    continue\n",
        "  \n",
        "  flag+=1\n",
        "  print(flag)\n",
        "  # calling urlretrieve function to get resource\n",
        "#urllib.request.urlretrieve(image_url,'/content/drive/MyDrive/Data2//pic1.jpg')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-efa5226ba543>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     43\u001b[0m   \u001b[0mimage_url\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontent1\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'imageurl'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m   \u001b[0mresponsecode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontent1\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'imageurl'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_attribute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'href'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus_code\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m   \u001b[0;31m#print((links[i].get_attribute('href'))+\" --- \"+str(responsecode))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'str' object has no attribute 'get_attribute'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GK2WXs6fbvx_"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}