fast-crawler-NSTL-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# -*- coding: utf-8 -*-\n",
    "\n",
    "\"\"\"\n",
    "@Author: Shuyue Jia\n",
    "@Date: Arg 19, 2020\n",
    "\n",
    "Usage Notice:\n",
    "1. Run this code after you run \"get-website-IDs.ipynb\" script and get \"current_IDs.csv\" file\n",
    "2. I highly recommend you guys use this method to reptile the dynamic websites as it is much faster\n",
    "3. This script will download all the NSTL data automatically\n",
    "4. Don't lose Internet connection when you run this code\n",
    "5. This method can be transfered to other websites and reptile other contents besides this NSTL data\n",
    "\"\"\"\n",
    "\n",
    "# Import necessary packages\n",
    "import os\n",
    "import ssl\n",
    "import json\n",
    "import time\n",
    "import requests\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from random import randint\n",
    "import urllib3\n",
    "\n",
    "# Disable all kinds of warnings\n",
    "urllib3.disable_warnings()\n",
    "\n",
    "# Avoid SSL Certificate to access the HTTP website\n",
    "ssl._create_default_https_context = ssl._create_unverified_context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_csv(csv_path: str):\n",
    "    \"\"\"\n",
    "    Read CSV Files\n",
    "    \"\"\"\n",
    "    csv_content = pd.read_csv(csv_path, header=None)\n",
    "    IDs = np.array(csv_content)\n",
    "    IDs = np.squeeze(IDs)\n",
    "    return IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def concat_csv(*params):\n",
    "    \"\"\"\n",
    "    Concatenate all the CSV files\n",
    "    \"\"\"\n",
    "    all_csv = []\n",
    "    len_params = len(params)\n",
    "    for i in range(len_params):\n",
    "        csv_path = params[i]\n",
    "        ID = read_csv(csv_path=csv_path)\n",
    "        all_csv = np.concatenate([all_csv, ID], axis=0)\n",
    "    all_csv = np.squeeze(all_csv)\n",
    "    return all_csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_url(ID: str) -> str:\n",
    "    \"\"\"\n",
    "    Read the website and return the contents of the website\n",
    "    :param ID: The ID of the website\n",
    "    :return soup.text: The contents of the website\n",
    "    \"\"\"\n",
    "    # URL of the website + ID for every word website\n",
    "    url = 'https://www.nstl.gov.cn/execute?target=nstl4.search4&function=paper/pc/detail&id=' + ID\n",
    "\n",
    "    # A fake device to avoid the Anti reptile\n",
    "    USER_AGENTS = [\n",
    "        \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)\",\n",
    "        \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)\",\n",
    "        \"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)\",\n",
    "        \"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)\",\n",
    "        \"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)\",\n",
    "        \"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)\",\n",
    "        \"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)\",\n",
    "        \"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)\",\n",
    "        \"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6\",\n",
    "        \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1\",\n",
    "        \"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0\",\n",
    "        \"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5\",\n",
    "        \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6\",\n",
    "        \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11\",\n",
    "        \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20\",\n",
    "        \"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52\",\n",
    "    ]\n",
    "    \n",
    "    random_agent = USER_AGENTS[randint(0, len(USER_AGENTS) - 1)]\n",
    "    headers = {\n",
    "        'User-Agent': random_agent,\n",
    "    }\n",
    "    \n",
    "    # Change 10 smaller if you need faster reptile speed\n",
    "    for j in range(10):\n",
    "        try:\n",
    "            res = requests.get(url, headers=headers, verify=False, timeout=(5, 10))\n",
    "            contents = res.text\n",
    "        except Exception as e:\n",
    "            if j >= 9:\n",
    "                print('The exception has happened', '-' * 100)\n",
    "            else:\n",
    "                time.sleep(0.5)\n",
    "        else:\n",
    "            time.sleep(0.5)\n",
    "            break\n",
    "    return contents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_English_term(content: str):\n",
    "    \"\"\"\n",
    "    Find the English Term from the contents\n",
    "    :param content: The contents of the website\n",
    "    :return Eng_term: The found English term\n",
    "    :return content: The contents that cut the English term part\n",
    "    \"\"\"\n",
    "    mark = content.find('范畴号') + len('范畴号')\n",
    "    temp_cont = content[mark:mark + 100]\n",
    "\n",
    "    # START and END of the English term\n",
    "    START = temp_cont.find('[\"')\n",
    "    END = temp_cont.find('\"]')\n",
    "    Eng_term = temp_cont[START + 2:END]\n",
    "\n",
    "    # Cut the English term part from the contents\n",
    "    content = content[content.find('名称') + len('名称'):]\n",
    "    return Eng_term, content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_Chinese_term(content: str):\n",
    "    \"\"\"\n",
    "    Find the Chinese Term from the contents\n",
    "    :param content: The contents of the website\n",
    "    :return Chi_term: The found Chinese Term\n",
    "    :return content: The contents that cut the Chinese term part\n",
    "    \"\"\"\n",
    "    # If there is no Chinese Term available, then continue\n",
    "    if '中文名称' not in content:\n",
    "        Chi_term = ''\n",
    "    else:\n",
    "        # START and END of the Chinese Term\n",
    "        START = content.find('[\"') + len('[\"')\n",
    "        END = content.find('中文名称') - len('\"],\"n\":\"')\n",
    "        Chi_term = content[START:END]\n",
    "\n",
    "        # Cut the Chinese term part from the contents\n",
    "        content = content[content.find('中文名称') + len('中文名称'):]\n",
    "    return Chi_term, content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_English_definition(content: str):\n",
    "    \"\"\"\n",
    "    Find the English Definition from the content\n",
    "    :param content: The contents of the website\n",
    "    :return Eng_def: The found English definition\n",
    "    :return content: The contents that cut the English definition part\n",
    "    \"\"\"\n",
    "    # If there is no English definition available, then continue\n",
    "    if '释义' not in content:\n",
    "        Eng_def = ''\n",
    "    else:\n",
    "        # START and END of the English Definition\n",
    "        START = content.find('\"f\":\"def\",\"v\"') + len('\"f\":\"def\",\"v\":[\"')\n",
    "        END = content.find('释义')\n",
    "        Eng_def = content[START:END - len('\"],\"n\":\"')]\n",
    "\n",
    "        # Cut the English Definition part from the contents\n",
    "        content = content[END + len('释义'):]\n",
    "    return Eng_def, content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def synonym(content: str):\n",
    "    \"\"\"\n",
    "    Find all the Synonym words w.r.t. the English term\n",
    "    :param content: The contents of the website\n",
    "    :return synonym_words: The found synonym words\n",
    "    \"\"\"\n",
    "    # If there is no Synonym Words available, then continue\n",
    "    if '同义词' not in content:\n",
    "        synonym_words = ''\n",
    "    else:\n",
    "        # Find the Synonym words' mark from the content\n",
    "        mark = content.find('linkToBaTeleva') + len('linkToBaTeleva')\n",
    "        new_content = content[mark:]\n",
    "\n",
    "        # START and END of the Synonym words\n",
    "        START = new_content.find('[\"') + len('[')\n",
    "        END = new_content.find('名称') - len('],\"n\":\"')\n",
    "        synonym_words = new_content[START:END]\n",
    "    return synonym_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def field(ID: str):\n",
    "    \"\"\"\n",
    "    Find and save all the Fields of this particular term\n",
    "    :param ID: The ID of a particular website (word)\n",
    "    :return content: The Fields contents\n",
    "    \"\"\"\n",
    "    # URL of the Fields contents\n",
    "    url = 'https://www.nstl.gov.cn/execute?target=nstl4.search4&function=stkos/pc/detail/ztree&id=' + ID\n",
    "\n",
    "    # A fake device to avoid the Anti reptile\n",
    "    USER_AGENTS = [\n",
    "        \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)\",\n",
    "        \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)\",\n",
    "        \"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)\",\n",
    "        \"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)\",\n",
    "        \"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)\",\n",
    "        \"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)\",\n",
    "        \"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)\",\n",
    "        \"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)\",\n",
    "        \"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6\",\n",
    "        \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1\",\n",
    "        \"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0\",\n",
    "        \"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5\",\n",
    "        \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6\",\n",
    "        \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11\",\n",
    "        \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20\",\n",
    "        \"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52\",\n",
    "    ]\n",
    "\n",
    "    random_agent = USER_AGENTS[randint(0, len(USER_AGENTS) - 1)]\n",
    "    headers = {\n",
    "        'User-Agent': random_agent,\n",
    "    }\n",
    "    \n",
    "    for j in range(10):\n",
    "        try:\n",
    "            res = requests.get(url, headers=headers, verify=False, timeout=(5, 10))\n",
    "        except Exception as e:\n",
    "            if j >= 9:\n",
    "                print('The exception has happened', '-' * 100)\n",
    "            else:\n",
    "                time.sleep(0.5)\n",
    "        else:\n",
    "            time.sleep(0.1)\n",
    "            break\n",
    "    \n",
    "    content = res.text\n",
    "    \n",
    "    # Remove some useless contents from the Fields contents\n",
    "    # e.g., \"total\":1,\"code\":0,\n",
    "    # ,\"value\":\"180911\",\"font\":{\"color\":\"#999\"}}\n",
    "    START = content.find('code') + len('code\":0,')\n",
    "    content = content[START:]\n",
    "    content = content.replace(',\"font\":{\"color\":\"#999\"}', '')\n",
    "    content = content.replace('\"data\"', '\"Fields\"')\n",
    "\n",
    "    while '\"value\"' in content:\n",
    "        mark = content.find('\"value\"')\n",
    "        temp_cont = content[mark:mark + 100]\n",
    "        end = temp_cont.find('\"}')\n",
    "\n",
    "        true_start = mark - 1\n",
    "        true_end = mark + end + 1\n",
    "\n",
    "        content = content.replace(content[true_start:true_end], '')\n",
    "    return content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyEncoder(json.JSONEncoder):\n",
    "    \"\"\"\n",
    "    Used to save the numpy array into JSON file\n",
    "    \"\"\"\n",
    "    def default(self, obj):\n",
    "        if isinstance(obj, np.integer):\n",
    "            return int(obj)\n",
    "        elif isinstance(obj, np.floating):\n",
    "            return float(obj)\n",
    "        elif isinstance(obj, np.ndarray):\n",
    "            return obj.tolist()\n",
    "        else:\n",
    "            return super(MyEncoder, self).default(obj)\n",
    "\n",
    "\n",
    "def save_json(saved_data: list, save_name: str):\n",
    "    '''\n",
    "    Save the data (np.array) into JSON file\n",
    "    :param saved_data: the dataset which should be saved\n",
    "    :param save_name: the saved path of the JSON file\n",
    "    :return: saved file\n",
    "    '''\n",
    "    file = open(save_name, 'w', encoding='utf-8')\n",
    "    json.dump(saved_data, file, ensure_ascii=False, indent=4, cls=MyEncoder)\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_JSON(ID: str, save_year:str):\n",
    "    JSON_file = ''\n",
    "\n",
    "    # Get the contents of the website\n",
    "    contents = read_url(ID=ID)\n",
    "\n",
    "    # Find the English Term from the contents\n",
    "    Eng_term, con_cut_eng = find_English_term(content=contents)\n",
    "\n",
    "    # Find the Chinese Term from the contents\n",
    "    Chi_term, con_cut_chi = find_Chinese_term(content=con_cut_eng)\n",
    "\n",
    "    # Find the English Definition from the contents\n",
    "    Eng_def, con_cut_def = find_English_definition(content=con_cut_chi)\n",
    "\n",
    "    # Find the Synonym Words from the contents\n",
    "    synonym_word = synonym(content=con_cut_chi)\n",
    "\n",
    "    # Find the Fields from another contents\n",
    "    field_names = field(ID=i)\n",
    "\n",
    "    # Combine all the found data and make string for JSON\n",
    "    JSON_file += '{'\n",
    "    JSON_file += '\"English Term\": [\"'\n",
    "    JSON_file += Eng_term\n",
    "    JSON_file += '\"], '\n",
    "    JSON_file += '\"Chinese Term\": [\"'\n",
    "    JSON_file += Chi_term\n",
    "    JSON_file += '\"], '\n",
    "    JSON_file += '\"English Definition\": [\"'\n",
    "    JSON_file += Eng_def\n",
    "    JSON_file += '\"], '\n",
    "    JSON_file += '\"Synonym Words\": ['\n",
    "    JSON_file += synonym_word\n",
    "    JSON_file += '], '\n",
    "    JSON_file += field_names\n",
    "    \n",
    "    # Save the JSON File for each word\n",
    "    save_json(eval(JSON_file), save_path + save_year + '%s_word.json' % id)\n",
    "    print('The %s word of %s has been successfully saved!' % (id, save_year))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The main function\n",
    "if __name__ == '__main__':\n",
    "    index = 0\n",
    "    \n",
    "    # The saved path for the JSON and Excel files\n",
    "    save_path = 'NSTD-data-left/'\n",
    "    if not os.path.exists(save_path):\n",
    "        os.mkdir(save_path)\n",
    "    \n",
    "    # The CSV file containing all the website IDs from the last \"get-website-IDs.ipynb\" script\n",
    "    IDs = concat_csv('1_9999_pages.csv',\n",
    "                    '10000_19999_pages.csv',\n",
    "                    '20000_29999_pages.csv',\n",
    "                    '30000_39999_pages.csv',\n",
    "                    '40000_49999_pages.csv',\n",
    "                    '50000_59999_pages.csv',\n",
    "                    '60000_61496_pages.csv')\n",
    "    \n",
    "    print('The shape of all the IDs is %s' % np.shape(IDs)[0])\n",
    "    \n",
    "    for i in IDs:\n",
    "        prefix = i[0:4]\n",
    "        \n",
    "        # Find different year website and save the data respectively\n",
    "        # YEAR 2018\n",
    "        if prefix == 'C018':\n",
    "            save_year = 'YEAR-18/'\n",
    "            get_JSON(ID=i, save_year=save_year)\n",
    "            index += 1\n",
    "        \n",
    "        # YEAR 2019\n",
    "        elif prefix == 'C019':\n",
    "            save_year = 'YEAR-19/'\n",
    "            get_JSON(ID=i, save_year=save_year)\n",
    "            index += 1\n",
    "        \n",
    "        # YEAR 2020\n",
    "        elif prefix == 'C020':\n",
    "            save_year = 'YEAR-20/'\n",
    "            get_JSON(ID=i, save_year=save_year)\n",
    "            index += 1\n",
    "    \n",
    "    # print the number of words saved\n",
    "    print('Cheers! All the NSTL data (%s terms) has been successfully saved!' % str(index))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}