1. Selenium - scrape news.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set and start browser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "\n",
    "import os\n",
    "import time\n",
    "\n",
    "# Options\n",
    "chrome_options = Options()\n",
    "\n",
    "# Set window size for convenience\n",
    "WINDOW_SIZE = \"1280,720\"\n",
    "chrome_options.add_argument(\"--window-size=%s\" % WINDOW_SIZE)\n",
    "\n",
    "# Settings for files download\n",
    "path = os.getcwd()\n",
    "prefs = {\"profile.default_content_setting_values.notifications\" : 2, \n",
    "            \"download.default_directory\" : os.path.join(path, 'download'),\n",
    "            \"disable-popup-blocking\": \"true\"}\n",
    "chrome_options.add_experimental_option(\"prefs\",prefs)\n",
    "\n",
    "# Start browser\n",
    "driver = webdriver.Chrome(options=chrome_options)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load webpage with \"usd\" query\n",
    "driver.get(\"https://uk.investing.com/search/?q=usd&tab=news\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Work with webpage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium.webdriver.common.keys import Keys\n",
    "\n",
    "from selenium.webdriver.common.by import By\n",
    "\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "\n",
    "from selenium.webdriver.support import expected_conditions as EC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<selenium.webdriver.remote.webelement.WebElement (session=\"b0c3f0e23659b45fdd4d418dcd76171a\", element=\"CAD24E7D68FCDAA7A242DA9BEF4C35CC_element_22\")>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Wait for sing in window to pop in\n",
    "appeared = WebDriverWait(driver, 20).until(\n",
    "    EC.presence_of_element_located((By.XPATH, '//*[@id=\"PromoteSignUpPopUp\"]/div[2]/i'))\n",
    "    )\n",
    "appeared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Accept cookies\n",
    "button_cookies = driver.find_element(by=By.XPATH, value='//*[@id=\"onetrust-accept-btn-handler\"]')\n",
    "try:\n",
    "    button_cookies.click()\n",
    "except Exception as e:\n",
    "    pass\n",
    "\n",
    "# Close window to sign in\n",
    "button_sign_in = driver.find_element(by=By.XPATH, value='//*[@id=\"PromoteSignUpPopUp\"]/div[2]/i')\n",
    "try:\n",
    "    button_sign_in.click()\n",
    "except Exception as e:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open webpage calendar\n",
    "button_open_calendar = driver.find_element(by=By.XPATH, value='//*[@id=\"fullColumn\"]/div/div[4]/div[2]/div[1]/div[1]/div[2]')\n",
    "button_open_calendar.click()\n",
    "\n",
    "# Put start date to search bar\n",
    "search_start_date = driver.find_element(by=By.XPATH, value='//*[@id=\"startDate\"]')\n",
    "search_start_date.clear()\n",
    "search_start_date.send_keys(\"01/06/23\")\n",
    "\n",
    "# Put end date to search bar\n",
    "search_end_date = driver.find_element(by=By.XPATH, value='//*[@id=\"endDate\"]')\n",
    "search_end_date.clear()\n",
    "search_end_date.send_keys(\"30/06/23\")\n",
    "time.sleep(5)\n",
    "\n",
    "# Key closes search window\n",
    "search_end_date = driver.find_element(by=By.XPATH, value='//*[@id=\"endDate\"]')\n",
    "search_end_date.send_keys(Keys.ENTER)\n",
    "time.sleep(5)\n",
    "\n",
    "# Open webpage calendar\n",
    "button_open_calendar = driver.find_element(by=By.XPATH, value='//*[@id=\"fullColumn\"]/div/div[4]/div[2]/div[1]/div[1]/div[2]')\n",
    "button_open_calendar.click()\n",
    "time.sleep(1)\n",
    "\n",
    "# Apply dates\n",
    "button_apply = driver.find_element(by=By.XPATH, value='//*[@id=\"applyBtn\"]')\n",
    "button_apply.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "button_apply = driver.find_element(by=By.XPATH, value='//*[@id=\"applyBtn\"]')\n",
    "button_apply.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scroll webpage to get all news\n",
    "\n",
    "SCROLL_PAUSE_TIME = 1.5\n",
    "\n",
    "# Get scroll height\n",
    "last_height = 0\n",
    "new_height = driver.execute_script(\"return document.body.scrollHeight\")\n",
    "\n",
    "while last_height != new_height:\n",
    "    last_height = new_height\n",
    "    # Scroll down to bottom\n",
    "    driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "\n",
    "    # Wait to load page\n",
    "    time.sleep(SCROLL_PAUSE_TIME)\n",
    "\n",
    "    # Calculate new scroll height and compare with last scroll height\n",
    "    new_height = driver.execute_script(\"return document.body.scrollHeight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n<a class=\"title\" href=\"/news/forex-news/dollar-edges-lower-russian-tensions-remain-3064359\">Dollar edges lower; Russian tensions remain</a>\\n<div class=\"articleDetails\">\\n<span>By Investing.com</span>\\n<time class=\"date\">16 hours ago</time>\\n</div>\\n<p class=\"js-news-item-content\">Investing.com - The U.S. dollar weakened in early European trade Tuesday, but losses are minor as tensions in Russia remained fraught while traders await the release of economic data that may...</p>\\n'"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Take a look on HTML\n",
    "driver.find_element(By.XPATH, '//*[@id=\"fullColumn\"]/div/div[4]/div[3]/div/div[1]/div').get_attribute('innerHTML')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Dollar edges lower; Russian tensions remain',\n",
       " 'https://uk.investing.com/news/forex-news/dollar-edges-lower-russian-tensions-remain-3064359')"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get news titles and urls\n",
    "\n",
    "news_html = driver.find_elements(By.XPATH, '//*[@id=\"fullColumn\"]/div/div[4]/div[3]/div/div')\n",
    "\n",
    "news_tuples = [(html.find_element(By.XPATH, 'div/a').text, html.find_element(By.XPATH, 'div/a').get_attribute('href')) for html in news_html]\n",
    "\n",
    "news_tuples[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(driver.title)\n",
    "driver.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}