2021-06-23 1_멜론 Top 100 3년치 크롤링.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "64ed787f",
   "metadata": {},
   "source": [
    "# 기초 작업"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37f0504b",
   "metadata": {},
   "source": [
    "### 모듈 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ea010f7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import selenium\n",
    "from selenium import webdriver as wd\n",
    "import time\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "from itertools import repeat\n",
    "import sqlite3\n",
    "from pymongo import MongoClient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b820d7e1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "f52b03fa",
   "metadata": {},
   "source": [
    "### 크롬드라이버가 url 접속하도록 하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "62105353",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-2-264a2b05d9a8>:6: DeprecationWarning: use options instead of chrome_options\n",
      "  driver = wd.Chrome('chromedriver', chrome_options=options) # 크롬드라이버 경로\n"
     ]
    }
   ],
   "source": [
    "# 창 실행 옵션x\n",
    "options = wd.ChromeOptions()\n",
    "options.add_argument(\"headless\") \n",
    "\n",
    "# 크롬드라이버 열기\n",
    "driver = wd.Chrome('chromedriver', chrome_options=options) # 크롬드라이버 경로\n",
    "\n",
    "# 드라이버가 해당 url 접속\n",
    "url = 'https://www.melon.com/chart/index.htm' # 멜론차트 페이지\n",
    "driver.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17d9518f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10499903",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27c0d504",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e06f39b3",
   "metadata": {},
   "source": [
    "# 함수 만들기"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f397bd0",
   "metadata": {},
   "source": [
    "### 클릭 함수 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "727e11ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 연대선택 era_num=1 : 2020년대, era_num=2 : 2010년대\n",
    "def click_era(era_num) :    \n",
    "    driver.find_element_by_xpath('//*[@id=\"d_chart_search\"]/div/div/div[1]/div[1]/ul/li['+str(era_num)+']/span/label').click()\n",
    "    time.sleep(0.1)\n",
    "    \n",
    "# 연도선택 year_num : 최근 -> 과거\n",
    "def click_year(year_num) :\n",
    "    driver.find_element_by_xpath('//*[@id=\"d_chart_search\"]/div/div/div[2]/div[1]/ul/li['+str(year_num)+']/span/label').click()\n",
    "    time.sleep(0.1)\n",
    "\n",
    "# 월간선택 \n",
    "def click_month(month_year) :\n",
    "    driver.find_element_by_xpath('//*[@id=\"d_chart_search\"]/div/div/div[3]/div[1]/ul/li['+str(month_year)+']/span/label').click()\n",
    "    time.sleep(0.1)\n",
    "\n",
    "# 주간 선택\n",
    "def click_week(week_num) :\n",
    "    driver.find_element_by_xpath('//*[@id=\"d_chart_search\"]/div/div/div[4]/div[1]/ul/li['+str(week_num)+']/span/label').click()\n",
    "    time.sleep(0.1)\n",
    "\n",
    "# 장르선택 ('장르 종합'으로 선택)\n",
    "def click_genre() :\n",
    "    driver.find_element_by_xpath('//*[@id=\"d_chart_search\"]/div/div/div[5]/div[1]/ul/li[1]/span/label').click()\n",
    "    time.sleep(0.1)\n",
    "    \n",
    "    \n",
    "# 검색버튼 클릭\n",
    "def click_search() :\n",
    "    driver.find_element_by_xpath('//*[@id=\"d_srch_form\"]/div[2]/button/span/span').click()\n",
    "    time.sleep(0.1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a14a5503",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "f60ddc7a",
   "metadata": {},
   "source": [
    "### 선택 개수 찾기 함수 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "103590a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 연대선택 개수 찾기\n",
    "def find_era_length() :\n",
    "    length = len(driver.find_elements_by_css_selector('div.box_chic.nth1.view > div.list_value > ul > li'))\n",
    "    return length    \n",
    "    \n",
    "# 연도선택 개수 찾기\n",
    "def find_year_length() :\n",
    "    length = len(driver.find_elements_by_css_selector('div.box_chic.nth2.view > div.list_value > ul > li'))\n",
    "    return length\n",
    "\n",
    "# 월간선택 개수 찾기\n",
    "def find_month_length() :\n",
    "    length = len(driver.find_elements_by_css_selector('div.box_chic.nth3.view > div.list_value > ul > li'))\n",
    "    return length\n",
    "\n",
    "# 주간선택 개수 찾기\n",
    "def find_week_length() :\n",
    "    length = len(driver.find_elements_by_css_selector('div.box_chic.nth4.view > div.list_value > ul > li'))\n",
    "    return length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "314957dc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "5c125b82",
   "metadata": {},
   "source": [
    "### 크롤링 및 저장 함수 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0152d5a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 크롤링 & 저장하기\n",
    "def crawling_and_save_to_csv(year_num, month_num, week_num) :\n",
    "    \n",
    "    # html 정보 가져오기\n",
    "    html = driver.page_source\n",
    "    soup = BeautifulSoup(html, 'lxml')\n",
    "    \n",
    "    #순위 가져오기\n",
    "    song = [title.find('strong').get_text() for title in soup.find_all('div', attrs={'class': 'ellipsis rank01'})]\n",
    "    ranks = []\n",
    "    for i in range(len(song)):\n",
    "        ranks.append(i+1)\n",
    "    \n",
    "    # 가수명 가져오기\n",
    "    artist_name_list = []\n",
    "    for artist_name in soup.find_all('span', attrs={'class':'checkEllipsis'}) :\n",
    "        artist_name_list.append(artist_name.get_text())\n",
    "    \n",
    "    # 곡명 가져오기\n",
    "    song_name_list = []\n",
    "    for song_name in soup.find_all('div', attrs={'class': 'ellipsis rank01'}) :\n",
    "        song_name_list.append(song_name.find('strong').get_text())\n",
    "    \n",
    "    # 곡 아이디 가져오기\n",
    "    song_id_list=[]\n",
    "    for song_id in soup.find_all('input',{'name':'input_check'}):\n",
    "        song_id_list.append(song_id.get('value'))\n",
    "        \n",
    "    # 파일 제목 정하기    \n",
    "    year = soup.find('input',{'id':'year_'+str(year_num)}).get('value')\n",
    "    month = soup.find('input',{'id':'month_'+str(month_num)}).get('value')\n",
    "    week = week_num\n",
    "    date = str(year)+'-'+str(month)+'-'+str(week)\n",
    "    \n",
    "    \n",
    "    \n",
    "    # 데이터프레임 만들기\n",
    "    df = pd.DataFrame({\"rank\" : ranks, \"artist_name\" : artist_name_list, \"song_name\" : song_name_list, \"song_id\" : song_id_list})\n",
    "    \n",
    "    # csv로 저장\n",
    "    df.to_csv('csv/song_TOP100_list/'+date+'_TOP100.csv',encoding='utf-8-sig', index=False)\n",
    "    \n",
    "    # SQLite3 DB로 저장\n",
    "    con = sqlite3.connect(\"db/song.db\")\n",
    "    df.to_sql(date+\"_TOP100\", con, if_exists=\"replace\", index=False)\n",
    "    con.close()\n",
    "    \n",
    "    # MongoDB에 저장\n",
    "    client = MongoClient('localhost', 27017)\n",
    "    db = client['song']\n",
    "    col = db[date+\"_TOP100\"]\n",
    "    for i in range(len(ranks)) :\n",
    "        doc = {\"rank\" : ranks[i], \"artist_name\" : artist_name_list[i], \"song_name\" : song_name_list[i], \"song_id\" : song_id_list[i]}\n",
    "        col.insert_one(doc)\n",
    "    \n",
    "    \n",
    "    print(date+\" \", end = \"\")\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a65adc22",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "47a297b0",
   "metadata": {},
   "source": [
    "# 크롤링"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e16de11e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-1 2021-01-2 2021-01-3 2021-01-4 2021-02-1 2021-02-2 2021-02-3 2021-02-4 2021-03-1 2021-03-2 2021-03-3 2021-03-4 2021-04-1 2021-04-2 2021-04-3 2021-04-4 2021-04-5 2021-05-1 2021-05-2 2021-05-3 2021-05-4 2021-06-1 2021-06-2 2021-06-3 2020-01-1 2020-01-2 2020-01-3 2020-01-4 2020-01-5 2020-02-1 2020-02-2 2020-02-3 2020-02-4 2020-03-1 2020-03-2 2020-03-3 2020-03-4 2020-04-1 2020-04-2 2020-04-3 2020-04-4 2020-04-5 2020-05-1 2020-05-2 2020-05-3 2020-05-4 2020-06-1 2020-06-2 2020-06-3 2020-06-4 2020-07-1 2020-07-2 2020-07-3 2020-07-4 2020-07-5 2020-08-1 2020-08-2 2020-08-3 2020-08-4 2020-09-1 2020-09-2 2020-09-3 2020-09-4 2020-10-1 2020-10-2 2020-10-3 2020-10-4 2020-10-5 2020-11-1 2020-11-2 2020-11-3 2020-11-4 2020-12-1 2020-12-2 2020-12-3 2020-12-4 2020-12-5 2019-01-1 2019-01-2 2019-01-3 2019-01-4 2019-01-5 2019-02-1 2019-02-2 2019-02-3 2019-02-4 2019-03-1 2019-03-2 2019-03-3 2019-03-4 2019-04-1 2019-04-2 2019-04-3 2019-04-4 2019-05-1 2019-05-2 2019-05-3 2019-05-4 2019-05-5 2019-06-1 2019-06-2 2019-06-3 2019-06-4 2019-07-1 2019-07-2 2019-07-3 2019-07-4 2019-08-1 2019-08-2 2019-08-3 2019-08-4 2019-08-5 2019-09-1 2019-09-2 2019-09-3 2019-09-4 2019-10-1 2019-10-2 2019-10-3 2019-10-4 2019-10-5 2019-11-1 2019-11-2 2019-11-3 2019-11-4 2019-12-1 2019-12-2 2019-12-3 2019-12-4 2018-06-1 2018-06-2 2018-06-3 2018-06-4 2018-07-1 2018-07-2 2018-07-3 2018-07-4 2018-08-1 2018-08-2 2018-08-3 2018-08-4 2018-08-5 2018-09-1 2018-09-2 2018-09-3 2018-09-4 2018-10-1 2018-10-2 2018-10-3 2018-10-4 2018-11-1 2018-11-2 2018-11-3 2018-11-4 2018-11-5 2018-12-1 2018-12-2 2018-12-3 2018-12-4 "
     ]
    }
   ],
   "source": [
    "# 차트파인더 클릭\n",
    "driver.find_element_by_xpath('//*[@id=\"gnb_menu\"]/ul[1]/li[1]/div/div/button/span').click()\n",
    "\n",
    "# 주간차트 클릭\n",
    "driver.find_element_by_xpath('//*[@id=\"d_chart_search\"]/div/h4[1]/a').click()\n",
    "time.sleep(0.1)\n",
    "\n",
    "# 나머지 멈추는 시기 결정\n",
    "stop_era = 2\n",
    "stop_year = 2\n",
    "stop_month = 6\n",
    "\n",
    "# 나머지 자동 선택\n",
    "song_id_list =[]\n",
    "len_era = find_era_length()\n",
    "for i in range(1,len_era+1) :   \n",
    "    click_era(i)\n",
    "        \n",
    "    len_year = find_year_length()\n",
    "    for j in range(1,len_year+1) :\n",
    "        click_year(j)\n",
    "                \n",
    "        len_month = find_month_length()\n",
    "        \n",
    "        if i == stop_era and j == stop_year :\n",
    "            for k in range(stop_month, len_month+1) :\n",
    "                click_month(k)\n",
    "\n",
    "                len_week = find_week_length()\n",
    "                for l in range(1, len_week+1) :\n",
    "                    click_week(l)\n",
    "                    click_genre()\n",
    "                    click_search()\n",
    "\n",
    "                    # 크롤링 및 저장\n",
    "                    crawling_and_save_to_csv(j, k, l)\n",
    "        \n",
    "        else :\n",
    "            for k in range(1, len_month+1) :\n",
    "                click_month(k)\n",
    "\n",
    "                len_week = find_week_length()\n",
    "                for l in range(1, len_week+1) :\n",
    "                    click_week(l)\n",
    "                    click_genre()\n",
    "                    click_search()\n",
    "\n",
    "                    # 크롤링 및 저장\n",
    "                    crawling_and_save_to_csv(j, k, l)\n",
    "            \n",
    "        if i == stop_era and j == stop_year : break    \n",
    "                            \n",
    "    if i == stop_era : break;\n",
    "        \n",
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a85d9ff7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6df3cdef",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63f9f62a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}