VXVault webpage scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VX Vault Project"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VxVault is a website which keeps records of malicious contents. In this project we are extracting the data from VxVault website. First we get some proxies from 'ssl proxies' and using it we open the website through selenium webdriver. Now we crawl the required data fields and save them in a csv file. We extract geographic information of the urls extracted using ip2geotools and also download the malicious files in those urls using wget package to a local directory. We can also export the data to mongoDB database and/or to .json file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Actual deployment- Run when using for First Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "#**************STEP 1:****************\n",
    "#Open cmd and type these commands one by one for first time\n",
    "#pip install selenium\n",
    "#pip install beautifulsoup4\n",
    "#pip install pandas\n",
    "#pip install wget\n",
    "#pip install requests\n",
    "#pip install ip2geotools\n",
    "\n",
    "#*************STEP 2:******************\n",
    "#Download webdriver for Edge, chrome or firefox and paste its .exe file in C directory\n",
    "\n",
    "try:        \n",
    "    import wget\n",
    "    import requests\n",
    "    from bs4 import BeautifulSoup\n",
    "    import csv\n",
    "    import platform\n",
    "    from selenium import webdriver\n",
    "    from selenium.common.exceptions import NoSuchElementException\n",
    "    from ip2geotools.databases.noncommercial import DbIpCity\n",
    "    from random import choice\n",
    "    import pandas as pd\n",
    "    import sys\n",
    "except:\n",
    "    print(\"Error: Required Libraries not found.\")\n",
    "    sys.exit()\n",
    "\n",
    "def check_driver():   \n",
    "    x=platform.system()\n",
    "    try:\n",
    "            print(\"Checking for Microsoft Edge\")\n",
    "            driver = webdriver.Edge('MicrosoftWebDriver.exe')\n",
    "            return driver\n",
    "    except:\n",
    "            print(\"Microsoft Edge not Found\")\n",
    "            try:\n",
    "                print(\"Checking for Google Chrome\")\n",
    "                driver = webdriver.Chrome('C:\\chromedriver.exe')\n",
    "                return driver\n",
    "            except:\n",
    "                 print(\"Google Chrome not Found\")\n",
    "                 try:\n",
    "                     print(\"Checking for Mozilla Firefox\")\n",
    "                     driver = webdriver.Firefox('C:\\geckodriver.exe')\n",
    "                     return driver\n",
    "                 except:\n",
    "                     print(\"No Supported Browser found\")\n",
    "                     sys.exit()\n",
    "          \n",
    "def first_level_crawling():\n",
    "    try:\n",
    "        proxypage=requests.get(\"https://www.sslproxies.org/\")\n",
    "        soup2=BeautifulSoup(proxypage.content,\"html.parser\")\n",
    "    except:\n",
    "        print(\"Can't connect to Proxy list.\\nCheck your Internet Connnection and try again.\")\n",
    "        sys.exit()\n",
    "        \n",
    "    while True:\n",
    "        try:\n",
    "            proxy={\"https\":choice(list(map(lambda x:x[0]+':'+x[1],list(zip(map(lambda x:x.text, soup2.findAll('td')[::8]),\n",
    "                                                                           map(lambda x:x.text, soup2.findAll('td')[1::8]))))))}\n",
    "            print(\"Using Proxy: \",proxy)\n",
    "            try:\n",
    "                page=requests.get(\"http://vxvault.net/ViriList.php\",proxies=proxy,timeout=5)\n",
    "                soup=BeautifulSoup(page.content,\"html.parser\")\n",
    "            except:\n",
    "                print(\"Cannot connect to VxVault.\\nCheck your Internet Connnection and try again. \")\n",
    "                sys.exit()\n",
    "            break\n",
    "        except:\n",
    "            pass\n",
    "    return soup\n",
    "\n",
    "def second_level_crawling(soup):\n",
    "    try:\n",
    "        table=soup.find(\"table\")\n",
    "        anchors=table.find_all(\"a\")\n",
    "        mylist=[]\n",
    "        for x in anchors:\n",
    "            mylist.append(x.text)    \n",
    "        length=len(mylist)\n",
    "        date=[]\n",
    "        URL=[]\n",
    "        MD5=[]\n",
    "        IP=[]\n",
    "    \n",
    "        for x in range(4,length,7):\n",
    "            date.append(mylist[x])\n",
    "        for x in range(6,length,7):\n",
    "            URL.append(mylist[x])    \n",
    "        for x in range(7,length,7):\n",
    "            MD5.append(mylist[x])    \n",
    "        for x in range(8,length,7):\n",
    "            IP.append(mylist[x])\n",
    "    \n",
    "        vaultdict={\n",
    "            \"Date\":date,\n",
    "            \"URL\":URL,\n",
    "            \"MD5\":MD5,\n",
    "            \"IP\":IP\n",
    "        }\n",
    "    \n",
    "        vxvault=pd.DataFrame(vaultdict)\n",
    "        vxvault[\"Status\"]=\"Not Downloaded\"\n",
    "        file=open(\"D:\\VXVault.csv\",\"a\",encoding=\"utf-8\")\n",
    "        vxvault.to_csv(file, sep=',', index=False)\n",
    "        file.close()\n",
    "    except:\n",
    "        print(\"Error While Locating Web Element. Page Structure may have changed\")\n",
    "        sys.exit()\n",
    "        \n",
    "# ##### For First time crawling\n",
    "soup=first_level_crawling()\n",
    "driver=check_driver()\n",
    "url = \"http://vxvault.net/ViriList.php\"\n",
    "driver.get(url)\n",
    "page_number = 1\n",
    "while True:\n",
    "    try:     \n",
    "        second_level_crawling(soup)\n",
    "        driver.implicitly_wait(20)   \n",
    "        link = driver.find_element_by_link_text(\"Next >\")\n",
    "        driver.implicitly_wait(20)\n",
    "    except NoSuchElementException:\n",
    "        print(\"Alert: Last Page Reached!!\")\n",
    "        break\n",
    "    link.click()\n",
    "    page=requests.get(driver.current_url)\n",
    "    soup=BeautifulSoup(page.content,\"html.parser\") \n",
    "    page_number += 1\n",
    "\n",
    "print(\"Website crawled successfully!!\\n\",page_number,\" pages parsed!!\")\n",
    "\n",
    "# ##### Code to remove duplicate records and headers\n",
    "try:\n",
    "    y=pd.read_csv('D:\\VXVault.csv',encoding=\"utf-8\")\n",
    "    y.sort_values(\"MD5\",inplace=True)\n",
    "    y.drop_duplicates(subset='MD5',keep=\"first\")\n",
    "    x=pd.DataFrame(y)\n",
    "    \n",
    "    file2=open(\"D:\\VXVault.csv\",\"w\",encoding=\"utf-8\")\n",
    "    x.to_csv(file2, sep=',',index=False)\n",
    "    file2.close()\n",
    "except:\n",
    "    print(\"Couldn't Open VXVault.csv in D: directory\")\n",
    "    sys.exit()\n",
    "    \n",
    "# ##### Code to download the malicious files from the URLs of VxVault\n",
    "try:\n",
    "    urllist=[]\n",
    "    statuslist=[]\n",
    "    count=0\n",
    "    x=pd.read_csv('D:\\VXVault.csv',encoding=\"utf-8\")\n",
    "    ulist=x[\"URL\"]\n",
    "    slist=x[\"Status\"]\n",
    "    l=len(ulist)\n",
    "    for m in ulist:\n",
    "        urllist.append(m)\n",
    "        \n",
    "    for n in slist:\n",
    "        statuslist.append(n)\n",
    "        \n",
    "    print(\"Start downloading files from urls...\")\n",
    "    for i in range(1,l):\n",
    "        try:\n",
    "            if(statuslist[i]=='Downloaded'):\n",
    "                print(\"\\nURL \",i,\" Already Downloaded\")\n",
    "                count+=1\n",
    "                pass  \n",
    "            elif(statuslist[i]=='Not Downloaded'):\n",
    "                rawurl=urllist[i]\n",
    "                url=\"http://\"+rawurl\n",
    "                print(\"\\nURL \",i,\": \",url)\n",
    "                wget.download(url, 'D:\\VxVault Downloads')\n",
    "                try:\n",
    "                    f=open('D:\\VXVault.csv',encoding=\"utf-8\")\n",
    "                    r = csv.reader(f) \n",
    "                    count+=1\n",
    "                    records = list(r)\n",
    "                    records =[e for e in records if e]\n",
    "                    records[i+1][4]=\"Downloaded\"                      \n",
    "                    f.close()              \n",
    "                    f=open('D:\\VXVault.csv','w',encoding=\"utf-8\")\n",
    "                    w = csv.writer(f)\n",
    "                    w.writerows(records)   \n",
    "                    f.close()                    \n",
    "                    print(\"\\nURL\",i,\" Downloaded Successfully!\")\n",
    "                    \n",
    "                except:\n",
    "                    print(\"Couldn't Open VXVault.csv in D: directory\")\n",
    "            else:\n",
    "                print(\"Invalid Status! Not a URL:\",i)\n",
    "                     \n",
    "        except:\n",
    "            print(\"\\nError while downloading file from url:\",i,\"\\nTry again Later\")\n",
    "            pass\n",
    "        \n",
    "    print(count,\" Files downloaded from Urls out of \",l)\n",
    "    \n",
    "except:\n",
    "    print(\"Couldn't Open VXVault.csv in D: directory\")\n",
    "    sys.exit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Actual deployment- To run daily using scheduler/cronjob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "#**************STEP 1:****************\n",
    "#Open cmd and type these commands one by one for first time\n",
    "#pip install selenium\n",
    "#pip install beautifulsoup4\n",
    "#pip install pandas\n",
    "#pip install wget\n",
    "#pip install requests\n",
    "#pip install ip2geotools\n",
    "\n",
    "#*************STEP 2:******************\n",
    "#Download webdriver for Edge, chrome or firefox and paste its .exe file in C directory\n",
    "\n",
    "try:        \n",
    "    import wget\n",
    "    import requests\n",
    "    from bs4 import BeautifulSoup\n",
    "    import csv\n",
    "    import platform\n",
    "    from selenium import webdriver\n",
    "    from selenium.common.exceptions import NoSuchElementException\n",
    "    from ip2geotools.databases.noncommercial import DbIpCity\n",
    "    from random import choice\n",
    "    import pandas as pd\n",
    "    import sys\n",
    "except:\n",
    "    print(\"Error: Required Libraries not found.\")\n",
    "    sys.exit()\n",
    "\n",
    "def check_driver():   \n",
    "    x=platform.system()\n",
    "    \n",
    "    try:\n",
    "            print(\"Checking for Microsoft Edge\")\n",
    "            driver = webdriver.Edge('MicrosoftWebDriver.exe')\n",
    "            return driver\n",
    "    except:\n",
    "            print(\"Microsoft Edge not Found\")\n",
    "            try:\n",
    "                print(\"Checking for Google Chrome\")\n",
    "                driver = webdriver.Chrome('C:\\chromedriver.exe')\n",
    "                return driver\n",
    "            except:\n",
    "                 print(\"Google Chrome not Found\")\n",
    "                 try:\n",
    "                     print(\"Checking for Mozilla Firefox\")\n",
    "                     driver = webdriver.Firefox('C:\\geckodriver.exe')\n",
    "                     return driver\n",
    "                 except:\n",
    "                     print(\"No Supported Browser found\")\n",
    "                     sys.exit()\n",
    "                    \n",
    "def first_level_crawling():\n",
    "    try:\n",
    "        proxypage=requests.get(\"https://www.sslproxies.org/\")\n",
    "        soup2=BeautifulSoup(proxypage.content,\"html.parser\")\n",
    "    except:\n",
    "        print(\"Can't connect to Proxy list.\\nCheck your Internet Connnection and try again.\")\n",
    "        sys.exit()\n",
    "        \n",
    "    while True:\n",
    "        try:\n",
    "            proxy={\"https\":choice(list(map(lambda x:x[0]+':'+x[1],list(zip(map(lambda x:x.text, soup2.findAll('td')[::8]),\n",
    "                                                                           map(lambda x:x.text, soup2.findAll('td')[1::8]))))))}\n",
    "            print(\"Using Proxy: \",proxy)\n",
    "            try:\n",
    "                page=requests.get(\"http://vxvault.net/ViriList.php\",proxies=proxy,timeout=5)\n",
    "                soup=BeautifulSoup(page.content,\"html.parser\")\n",
    "            except:\n",
    "                print(\"Cannot connect to VxVault.\\nCheck your Internet Connnection and try again. \")\n",
    "                sys.exit()\n",
    "            break\n",
    "        except:\n",
    "            pass\n",
    "    return soup\n",
    "\n",
    "def second_level_crawling(soup):\n",
    "    try:\n",
    "        table=soup.find(\"table\")\n",
    "        anchors=table.find_all(\"a\")\n",
    "        mylist=[]\n",
    "        for x in anchors:\n",
    "            mylist.append(x.text)    \n",
    "        length=len(mylist)\n",
    "        date=[]\n",
    "        URL=[]\n",
    "        MD5=[]\n",
    "        IP=[]\n",
    "    \n",
    "        for x in range(4,length,7):\n",
    "            date.append(mylist[x])\n",
    "        for x in range(6,length,7):\n",
    "            URL.append(mylist[x])    \n",
    "        for x in range(7,length,7):\n",
    "            MD5.append(mylist[x])    \n",
    "        for x in range(8,length,7):\n",
    "            IP.append(mylist[x])\n",
    "    \n",
    "        vaultdict={\n",
    "            \"Date\":date,\n",
    "            \"URL\":URL,\n",
    "            \"MD5\":MD5,\n",
    "            \"IP\":IP\n",
    "        }\n",
    "    \n",
    "        vxvault=pd.DataFrame(vaultdict)\n",
    "        vxvault[\"Status\"]=\"Not Downloaded\"\n",
    "        file=open(\"D:\\VXVault.csv\",\"a\",encoding=\"utf-8\")\n",
    "        vxvault.to_csv(file, sep=',', index=False)\n",
    "        file.close()\n",
    "    except:\n",
    "        print(\"Error While Locating Web Element. Page Structure may have changed\")\n",
    "        sys.exit()\n",
    "        \n",
    "# ##### If Not first time then only recent pages needed to be crawled\n",
    "soup=first_level_crawling()\n",
    "driver=check_driver()\n",
    "url = \"http://vxvault.net/ViriList.php\"\n",
    "driver.get(url)\n",
    "page_number = 1\n",
    "for i in range(5):\n",
    "    try:     \n",
    "        second_level_crawling(soup)\n",
    "        driver.implicitly_wait(20)   \n",
    "        link = driver.find_element_by_link_text(\"Next >\")\n",
    "        driver.implicitly_wait(20)\n",
    "    except NoSuchElementException:\n",
    "        print(\"Alert: Last Page Reached!!\")\n",
    "        break\n",
    "    link.click()\n",
    "    page=requests.get(driver.current_url)\n",
    "    soup=BeautifulSoup(page.content,\"html.parser\")\n",
    "    page_number += 1\n",
    "\n",
    "print(\"Website crawled successfully!!\\n\",page_number-1,\" pages parsed!!\")\n",
    "\n",
    "# ##### Code to remove duplicate records and headers\n",
    "try:\n",
    "    y=pd.read_csv('D:\\VXVault.csv',encoding=\"utf-8\")\n",
    "    y.sort_values(\"MD5\",inplace=True)\n",
    "    y.drop_duplicates(subset='MD5',keep=\"first\")\n",
    "    x=pd.DataFrame(y)\n",
    "    \n",
    "    file2=open(\"D:\\VXVault.csv\",\"w\",encoding=\"utf-8\")\n",
    "    x.to_csv(file2, sep=',',index=False)\n",
    "    file2.close()\n",
    "except:\n",
    "    print(\"Couldn't Open VXVault.csv in D: directory\")\n",
    "    sys.exit()\n",
    "    \n",
    "# ##### Code to download the malicious files from the URLs of VxVault\n",
    "try:\n",
    "    urllist=[]\n",
    "    statuslist=[]\n",
    "    count=0\n",
    "    x=pd.read_csv('D:\\VXVault.csv',encoding=\"utf-8\")\n",
    "    ulist=x[\"URL\"]\n",
    "    slist=x[\"Status\"]\n",
    "    l=len(ulist)\n",
    "    for m in ulist:\n",
    "        urllist.append(m)\n",
    "        \n",
    "    for n in slist:\n",
    "        statuslist.append(n)\n",
    "        \n",
    "    print(\"Start downloading files from urls...\")\n",
    "    for i in range(1,l):\n",
    "        try:\n",
    "            if(statuslist[i]=='Downloaded'):\n",
    "                print(\"\\nURL \",i,\" Already Downloaded\")\n",
    "                count+=1\n",
    "                pass  \n",
    "            elif(statuslist[i]=='Not Downloaded'):\n",
    "                rawurl=urllist[i]\n",
    "                url=\"http://\"+rawurl\n",
    "                print(\"\\nURL \",i,\": \",url)\n",
    "                wget.download(url, 'D:\\VxVault Downloads')\n",
    "                try:\n",
    "                    f=open('D:\\VXVault.csv')\n",
    "                    r = csv.reader(f) \n",
    "                    count+=1\n",
    "                    records = list(r)\n",
    "                    records =[e for e in records if e]\n",
    "                    records[i][4]=\"Downloaded\"                      \n",
    "                    f.close()              \n",
    "                    f=open('D:\\VXVault.csv','w',encoding=\"utf-8\")\n",
    "                    w = csv.writer(f)\n",
    "                    w.writerows(records)   \n",
    "                    f.close()                    \n",
    "                    print(\"\\nURL\",i,\" Downloaded Successfully!\")\n",
    "                    \n",
    "                except:\n",
    "                    print(\"Couldn't Open VXVault.csv in D: directory\")\n",
    "            else:\n",
    "                print(\"Invalid Status! Not a URL:\",i)\n",
    "                     \n",
    "        except:\n",
    "            print(\"\\nError while downloading file from url:\",i,\"\\nTry again Later\")\n",
    "            pass\n",
    "        \n",
    "    print(count,\" Files downloaded from Urls out of \",l)\n",
    "    \n",
    "except:\n",
    "    print(\"Couldn't Open VXVault.csv in D: directory\")\n",
    "    sys.exit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Following is the Testing code blocks of above script :-"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Required Libraries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wget                              #To download malicious files from urls\n",
    "import warnings                          #To filter warnings while extracting info of ips and urls\n",
    "import requests\n",
    "import time\n",
    "from bs4 import BeautifulSoup\n",
    "import csv\n",
    "import os\n",
    "import platform                           #To get info about the platform of system/to know the os\n",
    "from selenium import webdriver\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "from random import choice                 #To randomize the proxy ip list\n",
    "import pandas as pd\n",
    "from pymongo import MongoClient           #For exporting to mongoDB\n",
    "from ip2geotools.databases.noncommercial import DbIpCity                         #To get info about IP \n",
    "import sys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ******For information about ip and Url [Only for python2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ipinfo(ip):\n",
    "    for i in ip:\n",
    "        warnings.filterwarnings(\"ignore\", category=UserWarning)\n",
    "        warnings.filterwarnings(\"ignore\", category=DeprecationWarning) \n",
    "        obj = IPWhois(i)\n",
    "        results = obj.lookup_whois()\n",
    "        match = geolite2.lookup(i)\n",
    "        try:\n",
    "            nf ={\"IP\":i, \"asn no\": results[\"asn\"],\"ip city\":results[\"nets\"][0]['city'],\"country\":results[\"nets\"][0]['country'] , \"net name \": str(results[\"nets\"][0]['name']),\"COUNTRY\":match.country,\"CONTINENT\":match.continent,\"TIMEZONE\":match.timezone}\n",
    "            print(nf)\n",
    "        except:\n",
    "            continue\n",
    "\n",
    "def urlinfo(url):\n",
    "    for j in url:\n",
    "        warnings.filterwarnings(\"ignore\", category=UserWarning)\n",
    "        warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "        w = whois.whois('j')\n",
    "        try:\n",
    "            nf1={\"domain\":j,\"name\":w[\"name\"],\"updated date\":w[\"updated_date\"],\"expiry date\":w[\"expiration_date\"],\"domain name\":w[\"domain_name\"],\"registrar\":w[\"registrar\"],\"country\":w[\"country\"],\"email\":w[\"emails\"]}\n",
    "            print(nf1)\n",
    "        except:\n",
    "            continue"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Code to Crawl through Proxy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def first_level_crawling():         #Opening Vx vault using proxy\n",
    "    proxypage=requests.get(\"https://www.sslproxies.org/\")       #Taking proxy ips from this website\n",
    "    soup2=BeautifulSoup(proxypage.content,\"html.parser\")\n",
    "    while True:\n",
    "        try:\n",
    "            proxy={\"https\":choice(list(map(lambda x:x[0]+':'+x[1],list(zip(map(lambda x:x.text, soup2.findAll('td')[::8]),\n",
    "                                                                           map(lambda x:x.text, soup2.findAll('td')[1::8]))))))}\n",
    "            print(\"Using Proxy: \",proxy)\n",
    "            page=requests.get(\"http://vxvault.net/ViriList.php\",proxies=proxy,timeout=5)    #opening vxvault webpage using proxy\n",
    "            soup=BeautifulSoup(page.content,\"html.parser\")\n",
    "            break\n",
    "        except:\n",
    "            pass\n",
    "    return soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def second_level_crawling(soup):                        #Getting necessary fields from the website\n",
    "    table=soup.find(\"table\")\n",
    "    anchors=table.find_all(\"a\")\n",
    "    mylist=[]\n",
    "    for x in anchors:\n",
    "        mylist.append(x.text)    \n",
    "    length=len(mylist)\n",
    "    date=[]\n",
    "    URL=[]\n",
    "    MD5=[]\n",
    "    IP=[]                                              #Making list of all the fields we need and making dictionary \n",
    "                                                       #and using pandas dump it to csv file\n",
    "    for x in range(4,length,7):\n",
    "        date.append(mylist[x])\n",
    "    for x in range(6,length,7):\n",
    "        URL.append(mylist[x])    \n",
    "    for x in range(7,length,7):\n",
    "        MD5.append(mylist[x])    \n",
    "    for x in range(8,length,7):\n",
    "        IP.append(mylist[x])\n",
    "\n",
    "    vaultdict={\n",
    "        \"Date\":date,\n",
    "        \"URL\":URL,\n",
    "        \"MD5\":MD5,\n",
    "        \"IP\":IP\n",
    "    }\n",
    "\n",
    "    vxvault=pd.DataFrame(vaultdict)\n",
    "    vxvault[\"Country\"]=\"Unknown\"\n",
    "    vxvault[\"Region\"]=\"Unknown\"\n",
    "    vxvault[\"City\"]=\"Unknown\"\n",
    "    vxvault[\"Longitude\"]=\"Unknown\"\n",
    "    vxvault[\"Latitude\"]=\"Unknown\"\n",
    "    vxvault[\"IPinfo\"]=\"Incomplete\"\n",
    "    vxvault[\"Status\"]=\"Not Downloaded\"\n",
    "    #print(vxvault)\n",
    "    file=open(\"D:EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.csv\",\"a\")\n",
    "    vxvault.to_csv(file, sep=',', index=False)\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### For First time crawling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#==================================================================================\n",
    "soup=first_level_crawling()\n",
    "driver = webdriver.Edge('C:\\Windows\\System32\\MicrosoftWebDriver.exe')            #Using selenium to open webbrowser and website\n",
    "url = \"http://vxvault.net/ViriList.php\"\n",
    "driver.get(url)\n",
    "page_number = 1\n",
    "while True:\n",
    "    try:     \n",
    "        second_level_crawling(soup)\n",
    "        driver.implicitly_wait(20)   \n",
    "        link = driver.find_element_by_link_text(\"Next >\")                        #Click on NEXT to get data from all pages\n",
    "        driver.implicitly_wait(20)\n",
    "    except NoSuchElementException:\n",
    "        print(\"Alert: Last Page Reached!!\")\n",
    "        break\n",
    "    link.click()\n",
    "    #page=requests.get(driver.current_url,proxies=proxy,timeout=5)\n",
    "    page=requests.get(driver.current_url)\n",
    "    soup=BeautifulSoup(page.content,\"html.parser\")\n",
    "    page_number += 1\n",
    "\n",
    "print(\"Website crawled successfully!!\\n\",page_number,\" pages parsed!!\")\n",
    "#======================================================================================="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### If Not first time then only recent pages needed to be crawled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup=first_level_crawling()\n",
    "driver = webdriver.Edge('MicrosoftWebDriver.exe')\n",
    "url = \"http://vxvault.net/ViriList.php\"\n",
    "driver.get(url)\n",
    "page_number = 1\n",
    "for i in range(5):                                                       #Only retrieving data of last 5 pages daily\n",
    "    try:     \n",
    "        second_level_crawling(soup)\n",
    "        driver.implicitly_wait(20)   \n",
    "        link = driver.find_element_by_link_text(\"Next >\")\n",
    "        driver.implicitly_wait(20)\n",
    "    except NoSuchElementException:\n",
    "        print(\"Alert: Last Page Reached!!\")\n",
    "        break\n",
    "    link.click()\n",
    "    page=requests.get(driver.current_url)\n",
    "    soup=BeautifulSoup(page.content,\"html.parser\")\n",
    "    page_number += 1\n",
    "\n",
    "print(\"Website crawled successfully!!\\n\",page_number-1,\" pages parsed!!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Finding ip information on a noncommercial database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y=pd.read_csv('D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.csv',encoding=\"utf-8\")\n",
    "y.sort_values(\"MD5\",inplace=True)\n",
    "y.drop_duplicates(subset='MD5',keep=\"first\")        #Removing duplicates from csv. MD5 is unique so using it as subset.\n",
    "x=pd.DataFrame(y)\n",
    "\n",
    "file2=open(\"D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.csv\",\"w\",encoding=\"utf-8\")\n",
    "x.to_csv(file2, sep=',',index=False)\n",
    "file2.close()\n",
    "\n",
    "iplist=[]\n",
    "region=[]\n",
    "country=[]\n",
    "city=[]\n",
    "longitude=[]\n",
    "latitude=[]\n",
    "IPinfo=[]\n",
    "\n",
    "x=pd.read_csv('D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.csv',encoding=\"utf-8\")\n",
    "ipls=x[\"IP\"]               \n",
    "rels=x[\"Region\"]\n",
    "cils=x[\"City\"]\n",
    "lols=x[\"Longitude\"]                  #Making lists for region,city,country,longitude and latitude\n",
    "lals=x[\"Latitude\"]\n",
    "ctls=x[\"Country\"]\n",
    "ipinfols=x[\"IPinfo\"]\n",
    "\n",
    "for n in ipinfols:\n",
    "    IPinfo.append(n)\n",
    "    \n",
    "for n in ipls:\n",
    "    iplist.append(n)\n",
    "    \n",
    "for n in rels:\n",
    "    region.append(n)\n",
    "    \n",
    "for n in cils:\n",
    "    city.append(n)\n",
    "    \n",
    "for n in lols:\n",
    "    longitude.append(n)\n",
    "    \n",
    "for n in lals:\n",
    "    latitude.append(n)\n",
    "\n",
    "for n in ctls:\n",
    "    country.append(n)\n",
    "    \n",
    "iplen=len(iplist)\n",
    "print(\"Length= \",iplen)\n",
    "print(\"\\nIP LIST:\\n\",iplist)\n",
    "print(\"*****-----*****\")\n",
    "print(\"\\nCountry:\\n\",country)\n",
    "print(\"*****-----*****\")\n",
    "print(\"\\nCity:\\n\",city)\n",
    "print(\"*****-----*****\")\n",
    "print(\"\\nLongitude:\\n\",longitude)\n",
    "print(\"*****-----*****\")\n",
    "print(\"\\nLatitude:\\n\",latitude)\n",
    "print(\"*****-----*****\")\n",
    "print(\"\\nRegion:\\n\",region)\n",
    "print(\"*****-----*****\")\n",
    "print(\"\\nIP info:\\n\",IPinfo)\n",
    "print(\"*****-----*****\")\n",
    "for i in range(iplen):\n",
    "    print(\"\\ni=\",i)\n",
    "    if(IPinfo[i]==\"Incomplete\"):           #Checking status of ipinfo and if information is incomplete or complete\n",
    "        try:\n",
    "            try:\n",
    "                f=open('D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.csv',encoding=\"utf-8\")\n",
    "            except:\n",
    "                print(\"VxVault Csv File not found\")\n",
    "                sys.exit()\n",
    "            print(\"\\nIP: \",iplist[i])    \n",
    "            r = csv.reader(f)            \n",
    "            records = list(r)\n",
    "            records =[e for e in records if e]            #Removing blank entries we get during conversion of csv to list\n",
    "            response = DbIpCity.get(iplist[i], api_key='free')  #Getting information corresponding to an ip\n",
    "            records[i+1][4]=response.country\n",
    "            records[i+1][5]=response.region\n",
    "            records[i+1][6]=response.city\n",
    "            records[i+1][7]=response.longitude\n",
    "            records[i+1][8]=response.latitude\n",
    "            records[i+1][9]=\"Complete\"                    #Updating the status of ipinfo field\n",
    "            f.close()            \n",
    "            f=open('D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.csv','w',encoding=\"utf-8\")           \n",
    "            w = csv.writer(f)\n",
    "            w.writerows(records)                          #Writing it to csv file again with updated records\n",
    "            f.close()          \n",
    "            print(\"-->Details of IP: \",iplist[i],\" Successfully Retrieved\")            \n",
    "        except:\n",
    "            pass\n",
    "    else:\n",
    "        pass\n",
    "print(\"operation done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Code to remove duplicate records and headers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Removing duplicates using MD5 as subset and then updating the csv\n",
    "y=pd.read_csv('D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.csv',encoding=\"utf-8\")\n",
    "y.sort_values(\"MD5\",inplace=True)\n",
    "y.drop_duplicates(subset='MD5',keep=\"first\")\n",
    "x=pd.DataFrame(y)\n",
    "\n",
    "file2=open(\"D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.csv\",\"w\",encoding=\"utf-8\")\n",
    "x.to_csv(file2, sep=',',index=False)\n",
    "file2.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Code to download the malicious files from the URLs of VxVault"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "urllist=[]\n",
    "statuslist=[]\n",
    "count=0\n",
    "x=pd.read_csv('D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.csv',encoding=\"utf-8\")\n",
    "ulist=x[\"URL\"]       #retrieving URLs and their corresponding status\n",
    "slist=x[\"Status\"]\n",
    "l=len(ulist)\n",
    "# print(ulist,\"***********\")\n",
    "# print(slist)\n",
    "for m in ulist:                     #Making list of urls\n",
    "    urllist.append(m)\n",
    "    \n",
    "for n in slist:                     #Making list of corr. status\n",
    "    statuslist.append(n)\n",
    "    \n",
    "print(\"Start downloading files from urls...\")\n",
    "for i in range(0,l):\n",
    "    print(\"\\n-->URL is:\",urllist[i])\n",
    "    try:\n",
    "        if(statuslist[i]=='Not Downloaded'):\n",
    "            rawurl=urllist[i]\n",
    "            url=\"http://\"+rawurl                      #Appending \"http:\" to the rawurls we had in list\n",
    "            #print(url)\n",
    "            wget.download(url, 'D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VxVault Downloads') #Download malicious file in url  \n",
    "            f=open('D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.csv',encoding=\"utf-8\")  #using wget to a destination folder\n",
    "            r = csv.reader(f)            \n",
    "            count+=1\n",
    "            records = list(r)\n",
    "            records =[e for e in records if e]       #Removing blank entries in the list\n",
    "            records[i+1][10]=\"Downloaded\"            #Updating status to Downloaded if operation successful\n",
    "            f.close()\n",
    "            f=open('D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.csv','w',encoding=\"utf-8\")\n",
    "            w = csv.writer(f)                        #Writng the records with updated info back to csv\n",
    "            w.writerows(records)   \n",
    "            f.close()\n",
    "            print(\"\\nURL\",i+1,\" Downloaded Successfully\")\n",
    "        else:\n",
    "            print(\"URL \",i+1,\" Already Downloaded\")\n",
    "            count+=1\n",
    "            pass  \n",
    "    except:\n",
    "        print(\"Error while downloading file from url:\",i+1)     #Throw exception in case of any problem while downloading \n",
    "        pass\n",
    "    \n",
    "print(count,\" Files downloaded from Urls out of \",l)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Inserting the records into MongoDB database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dump the records in csv to mongoDB database(in employee)\n",
    "client = MongoClient()\n",
    "db=client.test\n",
    "employee = db.employee\n",
    "df = pd.read_csv(\"D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.csv\",encoding=\"utf-8\")\n",
    "records_ = df.to_dict(orient = 'records')\n",
    "#print(records_)\n",
    "result = db.employee.insert_many(records_ )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Exporting to .json file for indexing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dumping the records in csv to a file in json format \n",
    "df = pd.read_csv(\"D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.csv\",encoding=\"utf-8\")\n",
    "df.to_json(path_or_buf=\"D:\\EDUCATIONAL\\CERT-In\\VxVault Project\\VXVault.json\",orient='records')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}