scrap-covid19-news.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "floppy-return",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "from pandas import DataFrame\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "standing-remainder",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "indonesian-concern",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://covid19.go.id/p/berita?page=1'\n",
    "response = requests.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "collected-maldives",
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(response.content, \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "spread-demand",
   "metadata": {},
   "outputs": [],
   "source": [
    "title = soup.findAll(\"a\", {\"class\": \"text-color-dark\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "floating-movie",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://covid19.go.id/p/berita/pelarangan-wna-masuk-ke-indonesia-diperpanjang',\n",
       " 'https://covid19.go.id/p/berita/pasien-sembuh-terus-meningkat-menjadi-711205-orang',\n",
       " 'https://covid19.go.id/p/berita/salah-jokowi-menggunakan-vaksin-buatan-eropa',\n",
       " 'https://covid19.go.id/p/berita/kriteria-penetapan-emergency-use-authorization-eua-untuk-vaksin-covid-19',\n",
       " 'https://covid19.go.id/p/berita/salah-rakyat-aceh-menolak-vaksin-covid19-karena-menurut-para-ulama-aceh-itu-haram',\n",
       " 'https://covid19.go.id/p/berita/kesembuhan-covid-19-bertambah-menjadi-703464-orang',\n",
       " 'https://covid19.go.id/p/berita/analisis-data-covid-19-indonesia-update-10-januari-2021',\n",
       " 'https://covid19.go.id/p/berita/monitoring-kepatuhan-protokol-kesehatan-di-34-provinsi-indonesia-update-10-januari-2021',\n",
       " 'https://covid19.go.id/p/berita/persetujuan-penggunaan-obat-dalam-kondisi-darurat',\n",
       " 'https://covid19.go.id/p/berita/fatwa-majelis-ulama-indonesia-nomor-02-tahun-2021',\n",
       " 'https://covid19.go.id/p/berita/salah-kemasan-vaksin-sinovac-ga-pake-ampulan-didalam-box-vaksin-sudah-ada-spuit-khusus-yg-sdh-ada-vaksin-nya',\n",
       " 'https://covid19.go.id/p/berita/disiplin-3m-hal-sederhana-yang-dapat-menyelamatkan-nyawa']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[item['href'] for item in title]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "accepting-consolidation",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Page1\n",
      "Page2\n",
      "Page3\n",
      "Page4\n",
      "Page5\n",
      "Page6\n",
      "Page7\n",
      "Page8\n",
      "Page9\n",
      "Page10\n",
      "Page11\n",
      "Page12\n",
      "Page13\n",
      "Page14\n",
      "Page15\n",
      "Page16\n",
      "Page17\n",
      "Page18\n",
      "Page19\n",
      "Page20\n",
      "Page21\n",
      "Page22\n",
      "Page23\n",
      "Page24\n",
      "Page25\n",
      "Page26\n",
      "Page27\n",
      "Page28\n",
      "Page29\n",
      "Page30\n",
      "Page31\n",
      "Page32\n",
      "Page33\n",
      "Page34\n",
      "Page35\n",
      "Page36\n",
      "Page37\n",
      "Page38\n",
      "Page39\n",
      "Page40\n",
      "Page41\n",
      "Page42\n",
      "Page43\n",
      "Page44\n",
      "Page45\n",
      "Page46\n",
      "Page47\n",
      "Page48\n",
      "Page49\n",
      "Page50\n",
      "Page51\n",
      "Page52\n",
      "Page53\n",
      "Page54\n",
      "Page55\n",
      "Page56\n",
      "Page57\n",
      "Page58\n",
      "Page59\n",
      "Page60\n",
      "Page61\n",
      "Page62\n",
      "Page63\n",
      "Page64\n",
      "Page65\n",
      "Page66\n",
      "Page67\n",
      "Page68\n",
      "Page69\n",
      "Page70\n",
      "Page71\n",
      "Page72\n",
      "Page73\n",
      "Page74\n",
      "Page75\n",
      "Page76\n",
      "Page77\n",
      "Page78\n",
      "Page79\n",
      "Page80\n",
      "Page81\n",
      "Page82\n",
      "Page83\n",
      "Page84\n",
      "Page85\n",
      "Page86\n",
      "Page87\n",
      "Page88\n",
      "Page89\n",
      "Page90\n",
      "Page91\n",
      "Page92\n",
      "Page93\n",
      "Page94\n",
      "Page95\n",
      "Page96\n",
      "Page97\n",
      "Page98\n",
      "Page99\n",
      "Page100\n",
      "Page101\n",
      "Page102\n",
      "Page103\n",
      "Page104\n",
      "Page105\n",
      "Page106\n",
      "Page107\n",
      "Page108\n",
      "Page109\n",
      "Page110\n",
      "Page111\n",
      "Page112\n",
      "Page113\n",
      "Page114\n",
      "Page115\n",
      "Page116\n",
      "Page117\n",
      "Page118\n",
      "Page119\n",
      "Page120\n",
      "Page121\n",
      "Page122\n",
      "Page123\n",
      "Page124\n",
      "Page125\n",
      "Page126\n",
      "Page127\n",
      "Page128\n",
      "Page129\n",
      "Page130\n",
      "Page131\n",
      "Page132\n",
      "Page133\n",
      "Page134\n",
      "Page135\n",
      "Page136\n",
      "Page137\n",
      "Page138\n",
      "Page139\n",
      "Page140\n",
      "Page141\n",
      "Page142\n",
      "Page143\n",
      "Page144\n",
      "Page145\n",
      "Page146\n",
      "Page147\n",
      "Page148\n",
      "Page149\n",
      "Page150\n",
      "Page151\n",
      "Page152\n",
      "Page153\n",
      "Page154\n",
      "Page155\n",
      "Page156\n",
      "Page157\n"
     ]
    }
   ],
   "source": [
    "URL = 'https://covid19.go.id/p/berita?page='\n",
    "title_list = []\n",
    "for x in range(157):\n",
    "    response = requests.get(URL+str(x+1))\n",
    "    soup = BeautifulSoup(response.content, \"html.parser\")\n",
    "    titles = soup.findAll(\"a\", {\"class\": \"text-color-dark\"})\n",
    "    title_list = title_list + [title.string for title in titles]\n",
    "    print(\"Page\" + str(x+1))\n",
    "    time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "similar-investment",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "original-tours",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('snapshot.json', 'w') as fp:\n",
    "    json.dump(title_list, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "encouraging-hormone",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(title_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "common-project",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Pelarangan WNA Masuk Ke Indonesia Diperpanjang',\n",
       " 'Pasien Sembuh Terus Meningkat Menjadi 711.205 Orang',\n",
       " '[SALAH] Jokowi Menggunakan Vaksin Buatan Eropa',\n",
       " 'Kriteria Penetapan Emergency Use Authorization (EUA) untuk Vaksin COVID-19',\n",
       " '[SALAH] “Rakyat Aceh menolak vaksin covid19 karena menurut para ulama Aceh itu haram”',\n",
       " 'Kesembuhan COVID-19 Bertambah Menjadi 703.464 Orang',\n",
       " 'Analisis Data COVID-19 Indonesia (Update Per 10 Januari 2021)',\n",
       " 'Monitoring Kepatuhan Protokol Kesehatan di 34 Provinsi Indonesia (Update Per 10 Januari 2021)',\n",
       " 'Persetujuan Penggunaan Obat dalam Kondisi Darurat',\n",
       " 'Fatwa Majelis Ulama Indonesia Nomor : 02 Tahun 2021']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "title_list[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ordered-rhythm",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>hoax</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [text, hoax]\n",
       "Index: []"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "directed-delay",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_title = json.load(open('snapshot.json'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "wound-witness",
   "metadata": {},
   "outputs": [],
   "source": [
    "target = []\n",
    "title_copy = new_title.copy()\n",
    "for i, judul in enumerate(title_copy):\n",
    "    if judul.find('[SALAH]') == 0:\n",
    "        target.append(1)\n",
    "        title_copy[i] = judul.replace('[SALAH]', '').replace('“', '').replace('”', '').strip()\n",
    "    else:\n",
    "        target.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "applicable-necessity",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1879"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "lightweight-interim",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1879"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(title_copy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "respected-petroleum",
   "metadata": {},
   "outputs": [],
   "source": [
    "initial = DataFrame(columns = ['text', 'hoax'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bronze-sympathy",
   "metadata": {},
   "outputs": [],
   "source": [
    "initial['text'] = title_copy\n",
    "initial['hoax'] = target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "terminal-muslim",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>hoax</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Pelarangan WNA Masuk Ke Indonesia Diperpanjang</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pasien Sembuh Terus Meningkat Menjadi 711.205 ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Jokowi Menggunakan Vaksin Buatan Eropa</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Kriteria Penetapan Emergency Use Authorization...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Rakyat Aceh menolak vaksin covid19 karena menu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1874</th>\n",
       "      <td>Donald Trump Umumkan Roche Medical Company aka...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1875</th>\n",
       "      <td>Gugus Tugas Luncurkan Covid19.go.id</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1876</th>\n",
       "      <td>MUI Ajak Masyarakat Berikhtiar dan Berkontribu...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1877</th>\n",
       "      <td>China Akui Dokter Palestina Penemu Vaksin Coro...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1878</th>\n",
       "      <td>UNICEF meminta sekolah agar memiliki rencana p...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1879 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  hoax\n",
       "0        Pelarangan WNA Masuk Ke Indonesia Diperpanjang     0\n",
       "1     Pasien Sembuh Terus Meningkat Menjadi 711.205 ...     0\n",
       "2                Jokowi Menggunakan Vaksin Buatan Eropa     1\n",
       "3     Kriteria Penetapan Emergency Use Authorization...     0\n",
       "4     Rakyat Aceh menolak vaksin covid19 karena menu...     1\n",
       "...                                                 ...   ...\n",
       "1874  Donald Trump Umumkan Roche Medical Company aka...     1\n",
       "1875                Gugus Tugas Luncurkan Covid19.go.id     0\n",
       "1876  MUI Ajak Masyarakat Berikhtiar dan Berkontribu...     0\n",
       "1877  China Akui Dokter Palestina Penemu Vaksin Coro...     1\n",
       "1878  UNICEF meminta sekolah agar memiliki rencana p...     0\n",
       "\n",
       "[1879 rows x 2 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "initial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "harmful-meditation",
   "metadata": {},
   "outputs": [],
   "source": [
    "initial.to_csv('initial.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "satisfied-royalty",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove duplicate data\n",
    "clean = initial.drop_duplicates().reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "directed-lighter",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>hoax</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Pelarangan WNA Masuk Ke Indonesia Diperpanjang</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pasien Sembuh Terus Meningkat Menjadi 711.205 ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Jokowi Menggunakan Vaksin Buatan Eropa</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Kriteria Penetapan Emergency Use Authorization...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Rakyat Aceh menolak vaksin covid19 karena menu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1860</th>\n",
       "      <td>Donald Trump Umumkan Roche Medical Company aka...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1861</th>\n",
       "      <td>Gugus Tugas Luncurkan Covid19.go.id</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1862</th>\n",
       "      <td>MUI Ajak Masyarakat Berikhtiar dan Berkontribu...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1863</th>\n",
       "      <td>China Akui Dokter Palestina Penemu Vaksin Coro...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1864</th>\n",
       "      <td>UNICEF meminta sekolah agar memiliki rencana p...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1865 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  hoax\n",
       "0        Pelarangan WNA Masuk Ke Indonesia Diperpanjang     0\n",
       "1     Pasien Sembuh Terus Meningkat Menjadi 711.205 ...     0\n",
       "2                Jokowi Menggunakan Vaksin Buatan Eropa     1\n",
       "3     Kriteria Penetapan Emergency Use Authorization...     0\n",
       "4     Rakyat Aceh menolak vaksin covid19 karena menu...     1\n",
       "...                                                 ...   ...\n",
       "1860  Donald Trump Umumkan Roche Medical Company aka...     1\n",
       "1861                Gugus Tugas Luncurkan Covid19.go.id     0\n",
       "1862  MUI Ajak Masyarakat Berikhtiar dan Berkontribu...     0\n",
       "1863  China Akui Dokter Palestina Penemu Vaksin Coro...     1\n",
       "1864  UNICEF meminta sekolah agar memiliki rencana p...     0\n",
       "\n",
       "[1865 rows x 2 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "brilliant-pursuit",
   "metadata": {},
   "outputs": [],
   "source": [
    "clean.to_csv('data.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}