build_data-reddit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import json\n",
    "import pickle\n",
    "import copy\n",
    "import whois\n",
    "import re\n",
    "import pandas as pd\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import collections \n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "import os\n",
    "from urllib.parse import urlsplit, urlunsplit\n",
    "import IP2Location\n",
    "from urlextract import URLExtract\n",
    "import geoip2.webservice\n",
    "import urllib, sys\n",
    "import torch\n",
    "from sklearn.metrics import classification_report\n",
    "from torch import nn\n",
    "import shutil\n",
    "import requests\n",
    "import random\n",
    "from bs4 import BeautifulSoup\n",
    "from tqdm import tqdm\n",
    "from copy import deepcopy\n",
    "from pprint import pprint\n",
    "import socket\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.manifold import TSNE\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from nltk.corpus import words\n",
    "from libs.scanner.dataset_utils import create_features_single\n",
    "from libs.scanner.social_media_infogetter import InfoGetter\n",
    "import numpy as np\n",
    "from os import listdir\n",
    "import time\n",
    "from os.path import isfile, join\n",
    "from progressbar import progressbar\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from transformers import pipeline\n",
    "from xml.dom import minidom\n",
    "import praw\n",
    "import sqlite3\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import cycle\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from scipy import interp\n",
    "from bs4 import BeautifulSoup\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from urllib.parse import urlparse\n",
    "from captum.attr import (\n",
    "    GradientShap,\n",
    "    DeepLift,\n",
    "    DeepLiftShap,\n",
    "    IntegratedGradients,\n",
    "    LayerConductance,\n",
    "    NeuronConductance,\n",
    "    NoiseTunnel,\n",
    "    \n",
    ")\n",
    "from pprint import pprint"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Useful functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU :(\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def create_connection(db_file):\n",
    "    \"\"\" create a database connection to the SQLite database\n",
    "        specified by the db_file\n",
    "    :param db_file: database file\n",
    "    :return: Connection object or None\n",
    "    \"\"\"\n",
    "    conn = None\n",
    "    try:\n",
    "        conn = sqlite3.connect(db_file)\n",
    "    except Error as e:\n",
    "        print(e)\n",
    "\n",
    "    return conn\n",
    "\n",
    "\n",
    "device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n",
    "print( torch.cuda.get_device_name(device) if device == 'cuda:0' else 'CPU :(' )\n",
    "\n",
    "# create a new SM service!\n",
    "info_getter = InfoGetter()\n",
    "\n",
    "def get_source(url):\n",
    "    for schema in ['', 'https://', 'http://']:\n",
    "        try:\n",
    "            u = schema + url\n",
    "            response = requests.get(u, verify=False, timeout=5, headers={\n",
    "                'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36 RuxitSynthetic/1.0 v5553715026974570137 t6281935149377429786'\n",
    "            })\n",
    "            return response.text, u\n",
    "        except Exception as e:\n",
    "            pass\n",
    "    return None\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO for users: load your URL/labels into bot_data \n",
    "bot_data = [\n",
    "    ['www.anntaylor.com', 0],\n",
    "    ['bing.com', 0],\n",
    "    ['google.com', 1],\n",
    "    ['https://incomation.kartra.com', 1],\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/4\r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/pytorch/lib/python3.8/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'www.anntaylor.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/4\r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/pytorch/lib/python3.8/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'bing.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n",
      "/opt/miniconda3/envs/pytorch/lib/python3.8/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'www.bing.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error trying to connect to socket: closing socket\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/pytorch/lib/python3.8/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'google.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n",
      "/opt/miniconda3/envs/pytorch/lib/python3.8/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'www.google.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4/4\r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/envs/pytorch/lib/python3.8/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'incomation.kartra.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n",
      "/opt/miniconda3/envs/pytorch/lib/python3.8/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'app.kartra.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n",
      "/opt/miniconda3/envs/pytorch/lib/python3.8/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'app.kartra.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# create the bot data features based on ONLY saved pages!\n",
    "X = []\n",
    "Y = []\n",
    "collected_urls = []\n",
    "whois_data = {}\n",
    "countries = list(json.load(open('./data/country.json', 'r', encoding='utf-8')).keys())\n",
    "\n",
    "for ind_data, (url, label) in enumerate(bot_data):\n",
    "    print('%d/%d' %(ind_data + 1, len(bot_data)), end='\\r', flush=True)\n",
    "        \n",
    "    if 'http' in url:\n",
    "        u = urlparse(url).netloc\n",
    "    else:\n",
    "        u = url\n",
    "\n",
    "    # get whois data\n",
    "    try:\n",
    "        if url in whois_data:\n",
    "            icann_data = whois_data[url]\n",
    "        elif u in whois_data:\n",
    "            icann_data = whois_data[u]\n",
    "        else:\n",
    "            icann_data = whois.whois(url)\n",
    "            whois_data[url] = icann_data\n",
    "\n",
    "        if icann_data == {} or 'creation_date' not in icann_data or icann_data['creation_date'] == None:\n",
    "            continue\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        continue\n",
    "\n",
    "    # get page source (from cache or DL)\n",
    "    if os.path.exists(os.path.join('dataset', 'saved_pages', url.replace('/', '').replace('?', '').replace('!', '').replace('@', '').replace(':', '')) + '.html'):\n",
    "        fin = open(os.path.join('dataset', 'saved_pages', url.replace('/', '').replace('?', '').replace('!', '').replace('@', '').replace(':', '') + '.html'), 'r', encoding='utf-8')\n",
    "        page_content = fin.read()\n",
    "        fin.close()\n",
    "    elif os.path.exists(os.path.join('dataset', 'saved_pages', 'http' + url.replace('/', '').replace('?', '').replace('!', '').replace('@', '').replace(':', '')) + '.html'):\n",
    "        fin = open(os.path.join('dataset', 'saved_pages', 'http' + url.replace('/', '').replace('?', '').replace('!', '').replace('@', '').replace(':', '') + '.html'), 'r', encoding='utf-8')\n",
    "        page_content = fin.read()\n",
    "        fin.close()\n",
    "    elif os.path.exists(os.path.join('dataset', 'saved_pages', 'https' + url.replace('/', '').replace('?', '').replace('!', '').replace('@', '').replace(':', '')) + '.html'):\n",
    "        fin = open(os.path.join('dataset', 'saved_pages', 'https' + url.replace('/', '').replace('?', '').replace('!', '').replace('@', '').replace(':', '') + '.html'), 'r', encoding='utf-8')\n",
    "        page_content = fin.read()\n",
    "        fin.close()\n",
    "    elif os.path.exists(os.path.join('dataset', 'saved_pages', u.replace('/', '').replace('?', '').replace('!', '').replace('@', '').replace(':', '')) + '.html'):\n",
    "        fin = open(os.path.join('dataset', 'saved_pages', u.replace('/', '').replace('?', '').replace('!', '').replace('@', '').replace(':', '') + '.html'), 'r', encoding='utf-8')\n",
    "        page_content = fin.read()\n",
    "        fin.close()\n",
    "    elif os.path.exists(os.path.join('dataset', 'saved_pages', 'http' + u.replace('/', '').replace('?', '').replace('!', '').replace('@', '').replace(':', '')) + '.html'):\n",
    "        fin = open(os.path.join('dataset', 'saved_pages', 'http' + u.replace('/', '').replace('?', '').replace('!', '').replace('@', '').replace(':', '') + '.html'), 'r', encoding='utf-8')\n",
    "        page_content = fin.read()\n",
    "        fin.close()\n",
    "    elif os.path.exists(os.path.join('dataset', 'saved_pages', 'https' + u.replace('/', '').replace('?', '').replace('!', '').replace('@', '').replace(':', '')) + '.html'):\n",
    "        fin = open(os.path.join('dataset', 'saved_pages', 'https' + u.replace('/', '').replace('?', '').replace('!', '').replace('@', '').replace(':', '') + '.html'), 'r', encoding='utf-8')\n",
    "        page_content = fin.read()\n",
    "        fin.close()\n",
    "    else:\n",
    "        try:\n",
    "            page_content, url = get_source(url)\n",
    "            with open('./data/saved_pages/' + url.replace('/', '').replace('?', '').replace('!', '').replace('@', '').replace(':', '') + '.html', 'w', encoding='utf-8') as fin:\n",
    "                fin.write(page_content)\n",
    "                fin.close()\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            pass\n",
    "\n",
    "    # create features\n",
    "    if page_content != None:\n",
    "        X.append(create_features_single({url: icann_data}, countries, page_content, url, info_getter, './data/IP2LOCATION-LITE-DB1.BIN'))\n",
    "        Y.append(label)\n",
    "        collected_urls.append(u)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save collected data\n",
    "pickle.dump(whois_data, open('./cache/whois-cache.pkl', 'wb'))\n",
    "pickle.dump((X, Y, collected_urls), open('./cache/dataset.pkl', 'wb'))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert JSONL format data to features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_to_feature(icann_data, countries, html_string, url, data_file=\"IP2LOCATION-LITE-DB1.BIN\"):\n",
    "    ip_location = IP2Location.IP2Location(data_file)\n",
    "    u = urlsplit(url)\n",
    "    if 'http' not in url:\n",
    "        u = u.path\n",
    "    else:\n",
    "        u = u.netloc\n",
    "    X = []\n",
    "\n",
    "    # range\n",
    "    try:\n",
    "        created_date = datetime.datetime.strptime(icann_data[url]['creation_date'].split(' ')[0], '%Y-%m-%d')\n",
    "        exp_date = datetime.datetime.strptime(icann_data[url]['expiration_date'].split(' ')[0], '%Y-%m-%d')\n",
    "        total_age = (exp_date - created_date).days / 365.0\n",
    "    except:\n",
    "        total_age = -1\n",
    "\n",
    "    # set country\n",
    "    country_feature = [0 for i in range(255)]\n",
    "    if 'country' in icann_data[url] and icann_data[url]['country'] in countries:\n",
    "        country = countries.index(\n",
    "            icann_data[url]['country']\n",
    "        )\n",
    "        country_feature[country] = 1\n",
    "    else:\n",
    "        country = -1\n",
    "\n",
    "    # whois guard is used or not\n",
    "    names = icann_data[url]['registrar'] if 'registrar' in icann_data[url] else None\n",
    "    if type(names) is list:\n",
    "        names = names[0]\n",
    "\n",
    "    whois_guard_keywords = ['WhoisGuard'.lower(), 'REDACTED FOR PRIVACY'.lower(), 'Private Whois'.lower(), 'DOMAIN PRIVACY'.lower()]\n",
    "\n",
    "    for kw in whois_guard_keywords:\n",
    "        if names != None and (kw in names.lower() or 'priva' in names.lower()):\n",
    "            guard = 1\n",
    "            break\n",
    "        elif names == None:\n",
    "            guard = -1\n",
    "        else:\n",
    "            guard = 0\n",
    "\n",
    "    # get features from a single page\n",
    "    # parse DOM tree\n",
    "    parsed_tree = BeautifulSoup(html_string, 'html.parser')\n",
    "    script_tags = parsed_tree.find_all('script', src=True)\n",
    "    \n",
    "    link_tags = parsed_tree.find_all('a', href=True)\n",
    "    num_external_links = 0\n",
    "\n",
    "    if link_tags != None:\n",
    "        for link_tag in link_tags:\n",
    "            link = link_tag['href'] # if 'href' in link_tag else ''\n",
    "            # check if it's external link\n",
    "            if link.find('http') == 0 and u not in link:\n",
    "                num_external_links += 1\n",
    "    \n",
    "    # Shopping sites only filter: filter sites with payment\n",
    "    # if 'payment' not in html_string.lower() and 'cart' not in html_string.lower():\n",
    "    #     continue\n",
    "    \n",
    "    # check social media links\n",
    "    total_social_medias = [-1, -1, -1]\n",
    "    social_media_regex = [\n",
    "        r'instagram\\.com\\/[a-zA-Z0-9_\\-]+',\n",
    "        r'facebook\\.com\\/[a-zA-Z0-9_\\-]+',\n",
    "        r'twitter\\.com\\/[a-zA-Z0-9_\\-]+',\n",
    "    ]\n",
    "    \n",
    "    for i, r in enumerate(social_media_regex):\n",
    "        found = re.findall(r, html_string)\n",
    "        if len(found) > 0: # and 'login' not in found[0]:\n",
    "            if found[0].split('/')[-1] in url:\n",
    "                total_social_medias[i] = 1\n",
    "            else:\n",
    "                total_social_medias[i] = 0\n",
    "\n",
    "    # Host country\n",
    "    host_country_feature = [0 for i in range(255)]\n",
    "    try:\n",
    "        host_ip = socket.gethostbyname(u)\n",
    "        response = ip_location.get_all(host_ip)\n",
    "        host_country = response.country_short.decode()\n",
    "        host_domain_same = 1 if countries.index(host_country) == country else 0\n",
    "        host_country = countries.index(host_country)\n",
    "        host_country_feature[host_country] = 1\n",
    "    except:\n",
    "        host_domain_same = -1\n",
    "        host_country = -1\n",
    "\n",
    "    has_digit = False\n",
    "    for i in url:\n",
    "        if i.isdigit():\n",
    "            has_digit = True\n",
    "            break\n",
    "\n",
    "    is_cheap = 0\n",
    "    cheap_registrars = ['Namecheap', 'GoDaddy', 'Porkbun', 'NameSilo', 'Danesco', 'Hostinger']\n",
    "    \n",
    "    for cr in cheap_registrars:\n",
    "        if 'registrar' in i and i['registrar'].lower() is not None and cr.lower() in i['registrar'].lower():\n",
    "            is_cheap = 1\n",
    "            break\n",
    "    \n",
    "    top_cheap_domains = ['club', 'buzz', 'xyz', 'ua', 'icu', 'space', 'agency', 'monster', 'pw', 'click', 'website', 'site', 'club', 'online', 'link', 'shop', 'feedback', 'uno', 'press', 'best', 'fun', 'host', 'store', 'tech', 'top', 'it']\n",
    "    uses_cheap_domain = 0\n",
    "    for tld in top_cheap_domains:\n",
    "        if tld in url:\n",
    "            uses_cheap_domain = 1\n",
    "            \n",
    "    # domain in text\n",
    "    if parsed_tree.find('body') is not None:\n",
    "        domain_in_text = parsed_tree.find('body').text.count(u)\n",
    "    else:\n",
    "        domain_in_text = -1\n",
    "        \n",
    "    # append feature vector to dataset\n",
    "    domain_name = '.'.join(u.split('.')[:-1])\n",
    "    \n",
    "    X.append([\n",
    "        guard, \n",
    "        total_social_medias[0], \n",
    "        total_social_medias[1], \n",
    "        total_social_medias[2], \n",
    "        num_external_links, \n",
    "        host_domain_same,\n",
    "        len(script_tags),\n",
    "        1 if '-' in url else 0,\n",
    "        domain_name.count('.'),\n",
    "        1 if has_digit else 0,\n",
    "        is_cheap,\n",
    "        uses_cheap_domain,\n",
    "        domain_in_text,\n",
    "        0,\n",
    "        1 if u.split('.')[-1] not in ['com', 'net', 'org', 'uk', 'gov', 'au'] else 0,\n",
    "        total_age,\n",
    "    ])\n",
    "\n",
    "    X[-1].extend(country_feature)\n",
    "    X[-1].extend(host_country_feature)\n",
    "\n",
    "    # count missing features\n",
    "    missings = 0\n",
    "\n",
    "    for i in X[-1]:\n",
    "        if i == -1:\n",
    "            missings += 1\n",
    "    if country == -1:\n",
    "        missings += 1\n",
    "    if host_country == -1:\n",
    "        missings += 1\n",
    "\n",
    "    X[-1].append(missings)\n",
    "\n",
    "    return X\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['AF', 'AX', 'AL', 'DZ', 'AS', 'AD', 'AO', 'AI', 'AQ', 'AG']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 116/10087 [00:28<20:46,  8.00it/s] "
     ]
    }
   ],
   "source": [
    "whois_data = {}\n",
    "bot_data = []\n",
    "processed_dataset = []\n",
    "\n",
    "countries = list(json.load(open('./data/country.json', 'r', encoding='utf-8')).keys())\n",
    "countries_pd = pd.read_csv('./data/country.csv')\n",
    "print(countries[:10])\n",
    "countries_pd.head()\n",
    "countries_dict = {}\n",
    "for row in countries_pd.iterrows():\n",
    "    countries_dict[row[1]['value'].lower()] = row[1]['id']\n",
    "\n",
    "with open('./dataset.jsonl', 'r', encoding='utf-8') as fin:\n",
    "    for line in fin.readlines():\n",
    "        data = json.loads(line)\n",
    "        bot_data.append(data)\n",
    "\n",
    "for ind_data, info in tqdm(enumerate(bot_data), total=len(bot_data)):\n",
    "    sample_features = convert_to_feature(\n",
    "        {info['url']: info['whois']},\n",
    "        countries,\n",
    "        info['content'],\n",
    "        info['url'],\n",
    "        data_file='./data/IP2LOCATION-LITE-DB1.BIN'\n",
    "    )\n",
    "    \n",
    "    processed_dataset.append((\n",
    "        sample_features, info['label'], info['url']\n",
    "    ))\n",
    "\n",
    "# save collected data\n",
    "pickle.dump(processed_dataset, open('./cache/dataset.pkl', 'wb'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}