azlyrics_scrapping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "logical-gender",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, os.path\n",
    "from billboard_scrapping import get_titles_and_artists_billboard, clean_artist, clean_song\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.request import Request, urlopen, ProxyHandler, build_opener\n",
    "import time\n",
    "from whoosh.fields import Schema, TEXT, ID, STORED\n",
    "from whoosh.query import Every\n",
    "from whoosh.index import create_in, open_dir, exists_in\n",
    "from whoosh.qparser import QueryParser\n",
    "from shutil import rmtree\n",
    "import re\n",
    "from whoosh.query import Phrase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "detected-convertible",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = 'https://www.azlyrics.com/'\n",
    "headers = {'User-Agent':'Mozilla/5.0'}\n",
    "\n",
    "def get_song_data_from_url(url):\n",
    "    try:\n",
    "        req = Request(url, headers=headers)\n",
    "        #req.set_proxy(proxy_host, 'http')\n",
    "        webpage = urlopen(req).read()\n",
    "        bs_webpage = BeautifulSoup(webpage,\"lxml\")\n",
    "    except:\n",
    "        print('exception at url: {}'.format(url))\n",
    "        return None\n",
    "        \n",
    "    title = bs_webpage.find_all('b')[1].text\n",
    "    artist = bs_webpage.find('h2').find('b').text\n",
    "    lyrics = bs_webpage.find_all('div', attrs={'class': None})[1].text\n",
    "    \n",
    "    try:\n",
    "        album = bs_webpage.find('div', class_='songinalbum_title').text\n",
    "    except AttributeError:\n",
    "        album = \"\"\n",
    "    \n",
    "    cleaning = Cleaning()\n",
    "    title = cleaning.title(title)\n",
    "    lyrics = cleaning.lyrics(lyrics)\n",
    "    full_lyrics = cleaning.full_lyrics(lyrics)\n",
    "    \n",
    "    if album==\"You May Also Like\":\n",
    "        album = \"\"\n",
    "    if album!=\"\":\n",
    "        album = cleaning.album(album)\n",
    "    artist = \" \".join(artist.split(\" \")[:-1])\n",
    "    data = {\n",
    "        'title': title,\n",
    "        'artist': artist,\n",
    "        'lyrics': lyrics,\n",
    "        'full_lyrics': full_lyrics,\n",
    "        'album': album,\n",
    "        'url': url\n",
    "    }\n",
    "    \n",
    "    return data\n",
    "\n",
    "def create_schema():\n",
    "    schema = Schema(url=ID(stored=True),\n",
    "                    title=TEXT(stored=True),\n",
    "                    artist=TEXT(stored=True),\n",
    "                    full_lyrics=TEXT(stored=True, phrase=True),\n",
    "                    lyrics=TEXT(stored=True),\n",
    "                    album=TEXT(stored=True))\n",
    "    return schema\n",
    "\n",
    "def create_or_open_index(directory):\n",
    "    if not os.path.exists(directory):\n",
    "        os.mkdir(directory)\n",
    "    if exists_in(directory):\n",
    "        index = open_dir(directory)\n",
    "    else:\n",
    "        schema = create_schema()\n",
    "        index = create_in(directory, schema)\n",
    "    return index\n",
    "\n",
    "def index_song(index, song_data):\n",
    "    writer = index.writer()\n",
    "    writer.add_document(url=u'{}'.format(song_data['url']),\n",
    "                        title=u'{}'.format(song_data['title']),\n",
    "                        artist=u'{}'.format(song_data['artist']),\n",
    "                        full_lyrics=u'{}'.format(song_data['full_lyrics']),\n",
    "                        lyrics=u'{}'.format(song_data['lyrics']),\n",
    "                        album=u'{}'.format(song_data['album']))\n",
    "    writer.commit(optimize=True)\n",
    "    \n",
    "def search_song_by_title(title, index):\n",
    "    results_list = list()\n",
    "    qp = QueryParser('title', schema=index.schema)\n",
    "    q = qp.parse(u\"{}\".format(title))\n",
    "    with index.searcher() as searcher:\n",
    "        results = searcher.search(q)\n",
    "        for result in results:\n",
    "            data = {\n",
    "                'title': result['title'],\n",
    "                'artist': result['artist'],\n",
    "                'full_lyrics': result['full_lyrics'],\n",
    "                'lyrics': result['lyrics'],\n",
    "                'album': result['album']\n",
    "            }\n",
    "            results_list.append(data)\n",
    "    return results_list\n",
    "\n",
    "def search_song_by_author(author, index):\n",
    "    results_list = list()\n",
    "    qp = QueryParser('author', schema=index.schema)\n",
    "    q = qp.parse(u\"{}\".format(title))\n",
    "    with index.searcher() as searcher:\n",
    "        results = searcher.search(q)\n",
    "        for result in results:\n",
    "            data = {\n",
    "                'title': result['title'],\n",
    "                'artist': result['artist'],\n",
    "                'full_lyrics': result['full_lyrics'],\n",
    "                'lyrics': result['lyrics'],\n",
    "                'album': result['album']\n",
    "            }\n",
    "            results_list.append(data)\n",
    "    return results_list\n",
    "\n",
    "def search_song_by_lyrics(terms, index):\n",
    "    results_list = list()\n",
    "    qp = QueryParser('full_lyrics', schema=index.schema)\n",
    "    q = qp.parse(u'\"{}\"'.format(terms))\n",
    "    \n",
    "    with index.searcher() as searcher:\n",
    "        results = searcher.search(q)\n",
    "        for result in results:\n",
    "            data = {\n",
    "                'title': result['title'],\n",
    "                'artist': result['artist'],\n",
    "                'full_lyrics': result['full_lyrics'],\n",
    "                'lyrics': result['lyrics'],\n",
    "                'album': result['album']\n",
    "            }\n",
    "            results_list.append(data)\n",
    "    return results_list\n",
    "\n",
    "def get_songs_urls_by_letter(letter, limit=None):\n",
    "\n",
    "    url = base_url+'{}.html'.format(letter)\n",
    "    final_url_list = list()\n",
    "    \n",
    "    def get_urls_by_letter(url):\n",
    "        req = Request(url)\n",
    "        webpage = urlopen(req).read()\n",
    "        bs_webpage = BeautifulSoup(webpage,\"lxml\")\n",
    "        divs = bs_webpage.find_all('div', class_='col-sm-6')\n",
    "        a_list = [i.find_all('a') for i in divs]\n",
    "        a_list = [base_url+i['href'] for j in a_list for i in j]\n",
    "        return a_list\n",
    "    \n",
    "    urls = get_urls_by_letter(url)\n",
    "    \n",
    "    for u in urls[:limit]:\n",
    "        req = Request(u)\n",
    "        webpage = urlopen(req).read()\n",
    "        bs_webpage = BeautifulSoup(webpage,\"lxml\")\n",
    "        divs = bs_webpage.find_all('div', class_='listalbum-item')\n",
    "        a_list = [i.find_all('a') for i in divs]\n",
    "        final_url_list.append([base_url+i['href'][3:] for j in a_list for i in j])\n",
    "        time.sleep(20)\n",
    "    \n",
    "    final_url_list = [i for j in final_url_list for i in j]\n",
    "    \n",
    "    return final_url_list\n",
    "\n",
    "def index_songs_by_letter(letter, index, limit=None):\n",
    "    urls = get_songs_urls_by_letter(letter, limit)\n",
    "    for u in urls:\n",
    "        song_data = get_song_data_from_url(u)\n",
    "        print(song_data['title'])\n",
    "        index_song(index, song_data)\n",
    "        time.sleep(15)\n",
    "        \n",
    "def index_songs_by_artist(artist, index):\n",
    "    letter = 19 if artist[:1] not in \"abcdefghijklmnopqrstuvwxyz\" else artist[:1]\n",
    "    url = base_url+\"{}/{}.html\".format(letter, clean_artist(artist))\n",
    "    \n",
    "    req = Request(url)\n",
    "    webpage = urlopen(req).read()\n",
    "    bs_webpage = BeautifulSoup(webpage,\"lxml\")\n",
    "    divs = bs_webpage.find_all('div', class_='listalbum-item')\n",
    "    a_list = [i.find_all('a') for i in divs]\n",
    "    final_url_list = [base_url+i['href'][3:] for j in a_list for i in j]\n",
    "    time.sleep(20)\n",
    "    \n",
    "    for u in final_url_list:\n",
    "        song_data = get_song_data_from_url(u)\n",
    "        print(song_data['title'])\n",
    "        index_song(index, song_data)\n",
    "        time.sleep(15)\n",
    "        \n",
    "def index_songs_by_billboard(number, index, limit=None):\n",
    "    song_artist_tuple = get_titles_and_artists_billboard(number)\n",
    "    \n",
    "    for song, artist in song_artist_tuple:\n",
    "        song = clean_song(song)\n",
    "        artist = clean_artist(artist)\n",
    "        url = base_url+'lyrics/{}/{}.html'.format(artist, song)\n",
    "        print(url)\n",
    "        song_data = get_song_data_from_url(url)\n",
    "        if song_data is None:\n",
    "            continue\n",
    "        print(song_data['title'])\n",
    "        index_song(index, song_data)\n",
    "        time.sleep(15)\n",
    "\n",
    "class Cleaning():\n",
    "    \n",
    "    def full_lyrics(self, lyrics):\n",
    "        lyrics = lyrics.split('\\r\\n')\n",
    "        lyrics = [i.replace('\\n', ' ')\n",
    "                  for i in lyrics if i not in ['\\n', '\\r', '\\n\\r', '\\r\\n', '']]\n",
    "        \n",
    "        lyrics = ', '.join(lyrics).replace(',', '').replace('.', '').lower()\n",
    "    \n",
    "        return lyrics\n",
    "    \n",
    "    def lyrics(self, lyrics):\n",
    "        lyrics = lyrics.replace('\\r', '').replace('\\n\\n', '\\n')\n",
    "        lyrics = lyrics[1:][:-1]\n",
    "        return lyrics\n",
    "    \n",
    "    def title(self, title):\n",
    "        title = title.replace('\"', '')\n",
    "        return title\n",
    "    \n",
    "    def album(self, album):\n",
    "        print(album)\n",
    "        if (album!=\"\"):\n",
    "            album = re.findall(r'\"([^\"]*)\"', album)[0]\n",
    "        return album"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "recovered-humor",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ejemplo para una url de una cancion\n",
    "\n",
    "url=\"https://www.azlyrics.com/lyrics/pablocruise/islandwoman.html\"\n",
    "\n",
    "song_data = get_song_data_from_url(url)\n",
    "rmtree('./index')\n",
    "index = create_or_open_index('./index')\n",
    "index_song(index, song_data)\n",
    "results = search_song_by_title('Island', index)\n",
    "#results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "prospective-timeline",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ejemplo para urls por letra\n",
    "\n",
    "rmtree('./index')\n",
    "index = create_or_open_index('./index')\n",
    "for letter in \"abcdefghijklmnopqrstuvwxyz#\":\n",
    "    print(letter)\n",
    "    index_songs_by_letter(letter, index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "greater-lloyd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.azlyrics.com/lyrics/brunomars/leavethedooropen.html\n",
      "Leave The Door Open\n",
      "https://www.azlyrics.com/lyrics/dualipa/levitating.html\n",
      "album: \"Future Nostalgia\" (2020)\n",
      "Levitating\n",
      "https://www.azlyrics.com/lyrics/justinbieber/peaches.html\n",
      "album: \"Justice\" (2021)\n",
      "Peaches\n",
      "https://www.azlyrics.com/lyrics/weeknd/saveyourtears.html\n",
      "album: \"After Hours\" (2020)\n",
      "Save Your Tears\n",
      "https://www.azlyrics.com/lyrics/dojacat/kissmemore.html\n",
      "Kiss Me More\n",
      "https://www.azlyrics.com/lyrics/maskedwolf/astronautintheocean.html\n",
      "Astronaut In The Ocean\n",
      "https://www.azlyrics.com/lyrics/polog/rapstar.html\n",
      "RAPSTAR\n",
      "https://www.azlyrics.com/lyrics/jcole/interlude.html\n",
      "album: \"Cole World: The Sideline Story\" (2011)\n",
      "Interlude\n",
      "https://www.azlyrics.com/lyrics/kidlaroi/withoutyou.html\n",
      "EP: \"F*CK LOVE (SAVAGE)\" (2020)\n",
      "WITHOUT YOU\n",
      "https://www.azlyrics.com/lyrics/lilnasx/monterocallmebyyourname.html\n",
      "MONTERO (Call Me By Your Name)\n",
      "https://www.azlyrics.com/lyrics/trippieredd/misstherage.html\n",
      "Miss The Rage\n",
      "https://www.azlyrics.com/lyrics/weeknd/blindinglights.html\n",
      "album: \"After Hours\" (2020)\n",
      "Blinding Lights\n",
      "https://www.azlyrics.com/lyrics/liltjay/callingmyphone.html\n",
      "album: \"Destined 2 Win\" (2021)\n",
      "Calling My Phone\n",
      "https://www.azlyrics.com/lyrics/cardi-b/up.html\n",
      "Up\n",
      "https://www.azlyrics.com/lyrics/spotemgottem/beatbox.html\n",
      "Beat Box\n",
      "https://www.azlyrics.com/lyrics/oliviarodrigo/driverslicense.html\n",
      "album: \"SOUR\" (2021)\n",
      "drivers license\n",
      "https://www.azlyrics.com/lyrics/oliviarodrigo/dejavu.html\n",
      "album: \"SOUR\" (2021)\n",
      "deja vu\n",
      "https://www.azlyrics.com/lyrics/maroon5/beautifulmistakes.html\n",
      "Beautiful Mistakes\n",
      "https://www.azlyrics.com/lyrics/saweetie/bestfriend.html\n",
      "Best Friend\n",
      "https://www.azlyrics.com/lyrics/machinegunkelly/myexsbestfriend.html\n",
      "album: \"Tickets To My Downfall\" (2020)\n",
      "my ex's best friend\n",
      "https://www.azlyrics.com/lyrics/djkhaled/everychanceiget.html\n",
      "album: \"KHALED KHALED\" (2021)\n",
      "EVERY CHANCE I GET\n",
      "https://www.azlyrics.com/lyrics/lukecombs/foreverafterall.html\n",
      "album: \"What You See Is What You Get\" (2019)\n",
      "Forever After All\n",
      "https://www.azlyrics.com/lyrics/giveon/heartbreakanniversary.html\n",
      "album: \"TAKE TIME\" (2020)\n",
      "HEARTBREAK ANNIVERSARY\n",
      "https://www.azlyrics.com/lyrics/24kgoldn/mood.html\n",
      "album: \"El Dorado\" (2021)\n",
      "Mood\n",
      "https://www.azlyrics.com/lyrics/lilbaby/onme.html\n",
      "On Me\n",
      "https://www.azlyrics.com/lyrics/poohshiesty/backinblood.html\n",
      "mixtape: \"Shiesty Season\" (2021)\n",
      "Back In Blood\n",
      "https://www.azlyrics.com/lyrics/gabbybarrett/thegoodones.html\n",
      "album: \"Goldmine\" (2020)\n",
      "The Good Ones\n",
      "https://www.azlyrics.com/lyrics/popsmoke/whatyouknowboutlove.html\n",
      "album: \"Shoot For The Stars Aim For The Moon\" (2020)\n",
      "What You Know Bout Love\n",
      "https://www.azlyrics.com/lyrics/billieeilish/yourpower.html\n",
      "album: \"Happier Than Ever\" (2021)\n",
      "Your Power\n",
      "https://www.azlyrics.com/lyrics/ericchurch/hellofaview.html\n",
      "album: \"Soul\" (2021)\n",
      "Hell Of A View\n",
      "https://www.azlyrics.com/lyrics/dualipa/weregood.html\n",
      "album: \"Future Nostalgia\" (2020)\n",
      "We're Good\n",
      "https://www.azlyrics.com/lyrics/samhunt/breakingupwaseasyinthe90s.html\n",
      "album: \"Southside\" (2020)\n",
      "Breaking Up Was Easy In The 90's\n",
      "https://www.azlyrics.com/lyrics/glassanimals/heatwaves.html\n",
      "album: \"Dreamland\" (2020)\n",
      "Heat Waves\n",
      "https://www.azlyrics.com/lyrics/mooski/trackstar.html\n",
      "Track Star\n",
      "https://www.azlyrics.com/lyrics/jakeowen/madeforyou.html\n",
      "EP: \"Jake Owen\" (2018)\n",
      "Made For You\n",
      "https://www.azlyrics.com/lyrics/arianagrandefeatdojacat/3435.html\n",
      "exception at url: https://www.azlyrics.com/lyrics/arianagrandefeatdojacat/3435.html\n",
      "https://www.azlyrics.com/lyrics/arianagrande/positions.html\n",
      "album: \"positions\" (2020)\n",
      "positions\n",
      "https://www.azlyrics.com/lyrics/drake/wantsandneeds.html\n",
      "EP: \"Scary Hours 2\" (2021)\n",
      "Wants And Needs\n",
      "https://www.azlyrics.com/lyrics/tatemcrae/youbrokemefirst.html\n",
      "EP: \"TOO YOUNG TO BE SAD\" (2021)\n",
      "You Broke Me First\n",
      "https://www.azlyrics.com/lyrics/kaliuchis/telepatia.html\n",
      "album: \"Sin Miedo (del Amor y Otros Demonios) ∞\" (2020)\n",
      "telepatía\n",
      "https://www.azlyrics.com/lyrics/moneybaggyo/timetoday.html\n",
      "album: \"A GANGSTA'S PAIN\" (2021)\n",
      "Time Today\n",
      "https://www.azlyrics.com/lyrics/moneybaggyo/wockesha.html\n",
      "album: \"A GANGSTA'S PAIN\" (2021)\n",
      "Wockesha\n",
      "https://www.azlyrics.com/lyrics/badbunny/dakiti.html\n",
      "album: \"EL ÚLTIMO TOUR DEL MUNDO\" (2020)\n",
      "DÁKITI\n",
      "https://www.azlyrics.com/lyrics/yungbleu/youreminesstill.html\n",
      "EP: \"Love Scars: The 5 Stages Of Emotions\" (2020)\n",
      "You're Mines Still\n",
      "https://www.azlyrics.com/lyrics/billieeilish/thereforeiam.html\n",
      "album: \"Happier Than Ever\" (2021)\n",
      "Therefore I Am\n",
      "https://www.azlyrics.com/lyrics/thomasrhett/whatsyourcountrysong.html\n",
      "album: \"Country Again (Side A)\" (2021)\n",
      "What's Your Country Song\n",
      "https://www.azlyrics.com/lyrics/travisscott/goosebumps.html\n",
      "album: \"Birds In The Trap Sing McKnight\" (2016)\n",
      "Goosebumps\n",
      "https://www.azlyrics.com/lyrics/coileray/nomoreparties.html\n",
      "No More Parties\n",
      "https://www.azlyrics.com/lyrics/mirandalambert/settlingdown.html\n",
      "album: \"Wildcard\" (2019)\n",
      "Settling Down\n",
      "https://www.azlyrics.com/lyrics/arianagrande/pov.html\n",
      "album: \"positions\" (2020)\n",
      "pov\n",
      "https://www.azlyrics.com/lyrics/youngthug/ski.html\n",
      "mixtape: \"Slime Language 2\" (2021)\n",
      "Ski\n",
      "https://www.azlyrics.com/lyrics/drake/whatsnext.html\n",
      "EP: \"Scary Hours 2\" (2021)\n",
      "What's Next\n",
      "https://www.azlyrics.com/lyrics/coldplay/higherpower.html\n",
      "Higher Power\n",
      "https://www.azlyrics.com/lyrics/sza/gooddays.html\n",
      "Good Days\n",
      "https://www.azlyrics.com/lyrics/dylanscott/nobody.html\n",
      "EP: \"Nothing To Do Town\" (2019)\n",
      "Nobody\n",
      "https://www.azlyrics.com/lyrics/liltecca/neverleft.html\n",
      "Never Left\n",
      "https://www.azlyrics.com/lyrics/youngthug/solid.html\n",
      "mixtape: \"Slime Language 2\" (2021)\n",
      "Solid\n",
      "https://www.azlyrics.com/lyrics/dojacat/streets.html\n",
      "album: \"Hot Pink\" (2019)\n",
      "Streets\n",
      "https://www.azlyrics.com/lyrics/dierksbentley/gone.html\n",
      "Gone\n",
      "https://www.azlyrics.com/lyrics/nelly/lilbit.html\n",
      "Lil Bit\n",
      "https://www.azlyrics.com/lyrics/chrisyoung/famousfriends.html\n",
      "Famous Friends\n",
      "https://www.azlyrics.com/lyrics/parmalee/justtheway.html\n",
      "Just The Way\n",
      "https://www.azlyrics.com/lyrics/rodwave/tombstone.html\n",
      "album: \"SoulFly\" (2021)\n",
      "Tombstone\n",
      "https://www.azlyrics.com/lyrics/jordandavis/almostmaybes.html\n",
      "EP: \"Jordan Davis\" (2020)\n",
      "Almost Maybes\n",
      "https://www.azlyrics.com/lyrics/dan/gladyouexist.html\n",
      "exception at url: https://www.azlyrics.com/lyrics/dan/gladyouexist.html\n",
      "https://www.azlyrics.com/lyrics/morray/quicksand.html\n",
      "exception at url: https://www.azlyrics.com/lyrics/morray/quicksand.html\n",
      "https://www.azlyrics.com/lyrics/justinbieber/holdon.html\n",
      "album: \"Justice\" (2021)\n",
      "Hold On\n",
      "https://www.azlyrics.com/lyrics/imaginedragons/followyou.html\n",
      "Follow You\n",
      "https://www.azlyrics.com/lyrics/keithurbanduetwithpnk/onetoomany.html\n",
      "exception at url: https://www.azlyrics.com/lyrics/keithurbanduetwithpnk/onetoomany.html\n",
      "https://www.azlyrics.com/lyrics/tiesto/thebusiness.html\n",
      "The Business\n",
      "https://www.azlyrics.com/lyrics/jasonaldean/blameitonyou.html\n",
      "album: \"9\" (2019)\n",
      "Blame It On You\n",
      "https://www.azlyrics.com/lyrics/moneybaggyo/shottaslala.html\n",
      "album: \"A GANGSTA'S PAIN\" (2021)\n",
      "Shottas (Lala)\n",
      "https://www.azlyrics.com/lyrics/rodwave/streetrunner.html\n",
      "album: \"SoulFly\" (2021)\n",
      "Street Runner\n",
      "https://www.azlyrics.com/lyrics/pnk/alliknowsofar.html\n",
      "exception at url: https://www.azlyrics.com/lyrics/pnk/alliknowsofar.html\n",
      "https://www.azlyrics.com/lyrics/joynerlucas/ramenoj.html\n",
      "Ramen & OJ\n",
      "https://www.azlyrics.com/lyrics/jazminesullivan/pickupyourfeelings.html\n",
      "album: \"Heaux Tales\" (2021)\n",
      "Pick Up Your Feelings\n",
      "https://www.azlyrics.com/lyrics/lukebryan/downtoone.html\n",
      "album: \"Born Here Live Here Die Here\" (2020)\n",
      "Down To One\n",
      "https://www.azlyrics.com/lyrics/duncanlaurence/arcade.html\n",
      "album: \"Small Town Boy\" (2020)\n",
      "Arcade\n",
      "https://www.azlyrics.com/lyrics/brettyoung/lady.html\n",
      "album: \"Weekends Look A Little Different These Days\" (2021)\n",
      "Lady\n",
      "https://www.azlyrics.com/lyrics/coleswindell/singlesaturdaynight.html\n",
      "Single Saturday Night\n",
      "https://www.azlyrics.com/lyrics/moneybaggyo/hardforthenext.html\n",
      "album: \"A GANGSTA'S PAIN\" (2021)\n",
      "Hard For The Next\n",
      "https://www.azlyrics.com/lyrics/dontoliver/whatyouneed.html\n",
      "What You Need\n",
      "https://www.azlyrics.com/lyrics/badbunny/lanochedeanoche.html\n",
      "album: \"EL ÚLTIMO TOUR DEL MUNDO\" (2020)\n",
      "LA NOCHE DE ANOCHE\n",
      "https://www.azlyrics.com/lyrics/popsmoke/hello.html\n",
      "album: \"Shoot For The Stars Aim For The Moon\" (2020)\n",
      "Hello\n",
      "https://www.azlyrics.com/lyrics/ajr/waylesssad.html\n",
      "album: \"OK ORCHESTRA\" (2021)\n",
      "Way Less Sad\n",
      "https://www.azlyrics.com/lyrics/future/drankinnsmokin.html\n",
      "album: \"Pluto x Baby Pluto\" (2020)\n",
      "Drankin N Smokin\n",
      "https://www.azlyrics.com/lyrics/42dugg/4dagang.html\n",
      "mixtape: \"Free Dem Boyz\" (2021)\n",
      "4 Da Gang\n",
      "https://www.azlyrics.com/lyrics/djkhaled/ididit.html\n",
      "album: \"KHALED KHALED\" (2021)\n",
      "I DID IT\n",
      "https://www.azlyrics.com/lyrics/geramx/botellatrasbotella.html\n",
      "Botella Tras Botella\n",
      "https://www.azlyrics.com/lyrics/ryanhurdwithmarenmorris/chasingafteryou.html\n",
      "exception at url: https://www.azlyrics.com/lyrics/ryanhurdwithmarenmorris/chasingafteryou.html\n",
      "https://www.azlyrics.com/lyrics/cicop/tampa.html\n",
      "Tampa\n",
      "https://www.azlyrics.com/lyrics/kevingates/biggangsta.html\n",
      "EP: \"Only The Generals Gon Understand\" (2019)\n",
      "Big Gangsta\n",
      "https://www.azlyrics.com/lyrics/lildurk/hellcatstrackhawks.html\n",
      "mixtape: \"Only The Family - Lil Durk Presents: Loyal Bros\" (2021)\n",
      "Hellcats & Trackhawks\n",
      "https://www.azlyrics.com/lyrics/djkhaled/letitgo.html\n",
      "album: \"KHALED KHALED\" (2021)\n",
      "LET IT GO\n",
      "https://www.azlyrics.com/lyrics/thomasrhett/countryagain.html\n",
      "album: \"Country Again (Side A)\" (2021)\n",
      "Country Again\n",
      "https://www.azlyrics.com/lyrics/blakeshelton/minimumwage.html\n",
      "album: \"Body Language\" (2021)\n",
      "Minimum Wage\n",
      "https://www.azlyrics.com/lyrics/rodwave/richer.html\n",
      "album: \"SoulFly\" (2021)\n",
      "Richer\n",
      "https://www.azlyrics.com/lyrics/elleking/drunkandidontwannagohome.html\n",
      "Drunk (And I Don't Wanna Go Home)\n",
      "https://www.azlyrics.com/lyrics/liltjaypolog/headshot.html\n",
      "exception at url: https://www.azlyrics.com/lyrics/liltjaypolog/headshot.html\n",
      "https://www.azlyrics.com/lyrics/moneybaggyo/go.html\n",
      "album: \"A GANGSTA'S PAIN\" (2021)\n",
      "GO!\n"
     ]
    }
   ],
   "source": [
    "# Ejemplo para urls por billboard\n",
    "try:\n",
    "    rmtree('./index')\n",
    "except Exception:\n",
    "    pass\n",
    "index = create_or_open_index('./index')\n",
    "index_songs_by_billboard(100, index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "natural-things",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mixtape: \"Esmeralda\" (2017)\n",
      "Alabame\n",
      "mixtape: \"Esmeralda\" (2017)\n",
      "Dafne\n",
      "mixtape: \"Esmeralda\" (2017)\n",
      "Daga\n",
      "mixtape: \"Esmeralda\" (2017)\n",
      "Esmeralda\n",
      "mixtape: \"Esmeralda\" (2017)\n",
      "Oreen Ishi\n",
      "mixtape: \"Esmeralda\" (2017)\n",
      "Sandía\n",
      "mixtape: \"Esmeralda\" (2017)\n",
      "Kun Fu\n",
      "EP: \"La Sandunguera\" (2018)\n",
      "Estoy Triste\n",
      "EP: \"La Sandunguera\" (2018)\n",
      "Hot Butter\n",
      "EP: \"La Sandunguera\" (2018)\n",
      "Gimme Some Pizza\n",
      "EP: \"La Sandunguera\" (2018)\n",
      "La Sandunguera\n",
      "EP: \"La Sandunguera\" (2018)\n",
      "Ma Time\n",
      "EP: \"La Sandunguera\" (2018)\n",
      "La Passione\n",
      "album: \"Calambre\" (2020)\n",
      "Celebré\n",
      "album: \"Calambre\" (2020)\n",
      "Sana Sana\n",
      "album: \"Calambre\" (2020)\n",
      "Buenos Aires\n",
      "album: \"Calambre\" (2020)\n",
      "Delito\n",
      "album: \"Calambre\" (2020)\n",
      "Sugga\n",
      "album: \"Calambre\" (2020)\n",
      "Trío\n",
      "album: \"Calambre\" (2020)\n",
      "Business Woman\n",
      "album: \"Calambre\" (2020)\n",
      "Llámame\n",
      "album: \"Calambre\" (2020)\n",
      "Amor Salvaje\n",
      "album: \"Calambre\" (2020)\n",
      "Arrorró\n",
      "album: \"Calambre\" (2020)\n",
      "Puro Veneno\n",
      "album: \"Calambre\" (2020)\n",
      "Agárrate\n",
      "Copa Glasé\n",
      "Corashe\n",
      "exception at url: https://www.azlyrics.com/ps://www.azlyrics.com/lyrics/bizarrap/nathypelusobzrpmusicsessions36.html\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "'NoneType' object is not subscriptable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-1467b0261ed1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# rmtree('./index')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_or_open_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./index'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mindex_songs_by_artist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'nathy peluso'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-7-130e31d9055c>\u001b[0m in \u001b[0;36mindex_songs_by_artist\u001b[0;34m(artist, index)\u001b[0m\n\u001b[1;32m    174\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mu\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfinal_url_list\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    175\u001b[0m         \u001b[0msong_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_song_data_from_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mu\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 176\u001b[0;31m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msong_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'title'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    177\u001b[0m         \u001b[0mindex_song\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msong_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    178\u001b[0m         \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m15\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: 'NoneType' object is not subscriptable"
     ]
    }
   ],
   "source": [
    "# Ejemplo para urls por billboard\n",
    "\n",
    "# rmtree('./index')\n",
    "index = create_or_open_index('./index')\n",
    "index_songs_by_artist('nathy peluso', index)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}