news_analizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#First of all, I import all of the libreries that I will use.\n",
    "import datetime\n",
    "import feedparser\n",
    "import pandas as pd\n",
    "import re\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import ssl\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "#from umap import UMAP\n",
    "from hdbscan import HDBSCAN\n",
    "#import umap.umap_ as UMAP\n",
    "import getpass \n",
    "from bson.objectid import ObjectId\n",
    "# import pymongo module\n",
    "import pymongo\n",
    "import nltk\n",
    "\n",
    "import string\n",
    "import spacy\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "from spacy.lang.en import English\n",
    "import re\n",
    "import heapq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Now, I define some functions\n",
    "#The first functions are only meant to pares some news rss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def CNN_parser():\n",
    "    CNN_list=[]\n",
    "    CNN_URLS = {\n",
    "    \"World\": \"http://rss.cnn.com/rss/edition_world.rss\",\n",
    "    \"Africa\": \"http://rss.cnn.com/rss/edition_africa.rss\",\n",
    "    \"Americas\": \"http://rss.cnn.com/rss/edition_americas.rss\",\n",
    "    \"Asia\": \"http://rss.cnn.com/rss/edition_asia.rss\",\n",
    "    \"Europe\": \"http://rss.cnn.com/rss/edition_europe.rss\",\n",
    "    \"Middle East\": \"http://rss.cnn.com/rss/edition_meast.rss\",\n",
    "    \"U.S.\": \"http://rss.cnn.com/rss/edition_us.rss\",\n",
    "    \"Money\": \"http://rss.cnn.com/rss/money_news_international.rss\",\n",
    "    \"Technology\": \"http://rss.cnn.com/rss/edition_technology.rss\",\n",
    "    \"Science & Space\": \"http://rss.cnn.com/rss/edition_space.rss\",\n",
    "    \"Entertainment\":\"http://rss.cnn.com/rss/edition_entertainment.rss\",\n",
    "    \"World Sport\":\"http://rss.cnn.com/rss/edition_sport.rss\",\n",
    "    \"Football\" : \"http://rss.cnn.com/rss/edition_football.rss\",\n",
    "    \"Golf\": \"http://rss.cnn.com/rss/edition_golf.rss\",\n",
    "    \"Motorsport\": \"http://rss.cnn.com/rss/edition_motorsport.rss\",\n",
    "    \"Tennis\": \"http://rss.cnn.com/rss/edition_tennis.rss\",\n",
    "    \"Travel\":\"http://rss.cnn.com/rss/edition_travel.rss\",\n",
    "    \"Video\":\"http://rss.cnn.com/rss/cnn_freevideo.rss\",\n",
    "    \"Most Recent\":\"http://rss.cnn.com/rss/cnn_latest.rss\"}\n",
    "    for e in CNN_URLS:\n",
    "        \n",
    "        temp = feedparser.parse(CNN_URLS[e])\n",
    "        for i in range(len(temp['entries'])):\n",
    "            \n",
    "            CNN_list.append((str(datetime.datetime.now()),'Cable News Network',temp['entries'][i]['link'],temp['entries'][i]['title'], temp['entries'][i]['summary']))\n",
    "            \n",
    "        \n",
    "    \n",
    "    return CNN_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def FOX_parser():\n",
    "    FOX_list = []\n",
    "    FOX_URLS = {'Fox':\"https://www.foxnews.com/about/rss\"}\n",
    "    \n",
    "    \n",
    "    for e in FOX_URLS:\n",
    "        if hasattr(ssl, '_create_unverified_context'):\n",
    "            ssl._create_default_https_context = ssl._create_unverified_context\n",
    "        \n",
    "        temp = feedparser.parse(FOX_URLS[e])\n",
    "        for i in range(len(temp['entries'])):\n",
    "            \n",
    "            FOX_list.append((str(datetime.datetime.now()),'FOX News',temp['entries'][i]['link'],temp['entries'][i]['title']), temp['entries'][i]['summary'])\n",
    "            \n",
    "        \n",
    "    \n",
    "    return FOX_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def WSJ_parser():\n",
    "    WSJ_list=[]\n",
    "    WSJ_URLS = {\n",
    "    \"World\": \"https://feeds.a.dj.com/rss/RSSWorldNews.xml\",\n",
    "    \"Opinion\": \"https://feeds.a.dj.com/rss/RSSOpinion.xml\",\n",
    "    \"Business\":\"https://feeds.a.dj.com/rss/WSJcomUSBusiness.xml\",\n",
    "    \"Markets\":\"https://feeds.a.dj.com/rss/RSSMarketsMain.xml\",\n",
    "    \"Technology\":\"https://feeds.a.dj.com/rss/RSSWSJD.xml\",\n",
    "    \"Lifestyle\":\"https://feeds.a.dj.com/rss/RSSLifestyle.xml\"}\n",
    "    for e in WSJ_URLS:\n",
    "        \n",
    "        temp = feedparser.parse(WSJ_URLS[e])\n",
    "        for i in range(len(temp['entries'])):\n",
    "            \n",
    "            WSJ_list.append( (str(datetime.datetime.now()), 'The Wall Street Journal',temp['entries'][i]['link']  ,temp['entries'][i]['title'], temp['entries'][i]['summary']))\n",
    "            \n",
    "        \n",
    "    \n",
    "    return WSJ_list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def BBC_parser():\n",
    "    BBC_list=[]\n",
    "    BBC_URLS = {\n",
    "    \"World\":\"http://feeds.bbci.co.uk/news/world/rss.xml\",\n",
    "    \"UK\":\"http://feeds.bbci.co.uk/news/uk/rss.xml\",\n",
    "    \"Business\":\"http://feeds.bbci.co.uk/news/business/rss.xml\",\n",
    "    \"Politics\": \"http://feeds.bbci.co.uk/news/politics/rss.xml\",\n",
    "    \"Health\": \"http://feeds.bbci.co.uk/news/health/rss.xml\",\n",
    "    \"Education\": \"http://feeds.bbci.co.uk/news/education/rss.xml\",\n",
    "    \"Science\": \"http://feeds.bbci.co.uk/news/science_and_environment/rss.xml\",\n",
    "    \"Technology\": \"http://feeds.bbci.co.uk/news/technology/rss.xml\",\n",
    "    \"Entretainment & Arts\": \"http://feeds.bbci.co.uk/news/entertainment_and_arts/rss.xml\"}\n",
    "    for e in BBC_URLS:\n",
    "        \n",
    "        temp = feedparser.parse(BBC_URLS[e])\n",
    "        for i in range(len(temp['entries'])):\n",
    "            \n",
    "            BBC_list.append(( str(datetime.datetime.now()), 'British Broadcasting Corporation',temp['entries'][i]['link'] ,temp['entries'][i]['title'], temp['entries'][i]['summary']))\n",
    "            \n",
    "        \n",
    "    \n",
    "    return BBC_list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def USAT_parser():\n",
    "    USAT_list=[]\n",
    "    USAT_URLS = {\n",
    "    \"US\":\"http://rssfeeds.usatoday.com/UsatodaycomNation-TopStories\",\n",
    "    \"World\":\"http://rssfeeds.usatoday.com/UsatodaycomWorld-TopStories\",\n",
    "    \"Opinion\":\"http://rssfeeds.usatoday.com/News-Opinion\",\n",
    "    \"Sports\":\"http://rssfeeds.usatoday.com/UsatodaycomSports-TopStories\",\n",
    "    \"Lifestyle\":\"http://rssfeeds.usatoday.com/usatoday-LifeTopStories\",\n",
    "    \"Money\":\"http://rssfeeds.usatoday.com/UsatodaycomMoney-TopStories\",\n",
    "    \"Tech\":\"http://rssfeeds.usatoday.com/usatoday-TechTopStories\",\n",
    "    \"Travel\":\"http://rssfeeds.usatoday.com/UsatodaycomTravel-TopStories\"}\n",
    "    for e in USAT_URLS:\n",
    "        \n",
    "        temp = feedparser.parse(USAT_URLS[e])\n",
    "        for i in range(len(temp['entries'])):\n",
    "            \n",
    "            USAT_list.append( ( str(datetime.datetime.now()),'USA Today' ,temp['entries'][i]['link'] ,temp['entries'][i]['title'], temp['entries'][i]['summary']))\n",
    "            \n",
    "        \n",
    "    \n",
    "    return USAT_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def NYT_parser():\n",
    "    NYT_list=[]\n",
    "    NYT_URLS = {\n",
    "    \"World\":\"https://rss.nytimes.com/services/xml/rss/nyt/World.xml\",\n",
    "    \"Politics\":\"https://rss.nytimes.com/services/xml/rss/nyt/Politics.xml\",\n",
    "    \"Business\":\"https://rss.nytimes.com/services/xml/rss/nyt/Business.xml\",\n",
    "    \"Technology\":\"https://rss.nytimes.com/services/xml/rss/nyt/Technology.xml\",\n",
    "    \"Sports\":\"https://rss.nytimes.com/services/xml/rss/nyt/Sports.xml\"}\n",
    "    \n",
    "    for e in NYT_URLS:\n",
    "        \n",
    "        temp = feedparser.parse(NYT_URLS[e])\n",
    "        for i in range(len(temp['entries'])):\n",
    "            \n",
    "            NYT_list.append( (str(datetime.datetime.now()), 'New York Times',temp['entries'][i]['link'] ,temp['entries'][i]['title'], temp['entries'][i]['summary']))\n",
    "            \n",
    "        \n",
    "    \n",
    "    return NYT_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The next function is the summary algotithm. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def summary_fun(s, n_n = 7, lsen = 500):\n",
    "    '''s is the text to be sumarized.\n",
    "    n_n is the number of sentences that the function will return\n",
    "    lsen is the maximun number of words a sentence will include'''\n",
    "    if s == '': return ''\n",
    "    # Removing Square Brackets and Extra Spaces\n",
    "    article_text = re.sub(r'\\[[0-9]*\\]', ' ', s)\n",
    "    article_text = re.sub(r'\\s+', ' ', article_text)\n",
    "    # Removing special characters and digits\n",
    "    formatted_article_text = re.sub('[^a-zA-Z]', ' ', article_text )\n",
    "    formatted_article_text = re.sub(r'\\s+', ' ', formatted_article_text)\n",
    "    #Tokenize the text in sentences\n",
    "    sentence_list = nltk.sent_tokenize(article_text)\n",
    "    #Define the stopwords\n",
    "    stopwords = nltk.corpus.stopwords.words('english')\n",
    "    #In the next lines, we find the frecuency of every word\n",
    "    word_frequencies = {}\n",
    "    for word in nltk.word_tokenize(formatted_article_text):\n",
    "        if word not in stopwords:\n",
    "            if word not in word_frequencies.keys():\n",
    "                word_frequencies[word] = 1\n",
    "            else:\n",
    "                word_frequencies[word] += 1\n",
    "                \n",
    "                \n",
    "    #Now, we divide by the maximum to have relative frecuencies            \n",
    "    maximum_frequncy = max(word_frequencies.values())\n",
    "    for word in word_frequencies.keys():\n",
    "        word_frequencies[word] = (word_frequencies[word]/maximum_frequncy)\n",
    "    \n",
    "    #In the next step, we add the relative frecuencies of every word for every sentence, \n",
    "    #i.e. we give every sentence a score\n",
    "    sentence_scores = {}\n",
    "    for sent in sentence_list:\n",
    "        for word in nltk.word_tokenize(sent.lower()):\n",
    "            if word in word_frequencies.keys():\n",
    "                #here we choose the maximun number of words a sentence can have.\n",
    "                if len(sent.split(' ')) < lsen and len(sent.split(' ')) > 1:\n",
    "                    if sent not in sentence_scores.keys():\n",
    "                        sentence_scores[sent] = word_frequencies[word]\n",
    "                    else:\n",
    "                        sentence_scores[sent] += word_frequencies[word]\n",
    "    #Here, we create a list with the n_n sentences with higest scores\n",
    "    summary_sentences = heapq.nlargest(n_n, sentence_scores, key=sentence_scores.get)\n",
    "    #The final summary is the sum of the most important sentences\n",
    "    fin_summary = ' '.join(summary_sentences)\n",
    "    return fin_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The next step is to scrap the news of the day and save them in a list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "CNN_list = CNN_parser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "FOX_list=FOX_parser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "WSJ_list = WSJ_parser()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "BBC_list = BBC_parser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "USAT_list = USAT_parser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "NYT_list =NYT_parser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#and join them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "News = CNN_list + FOX_list+WSJ_list+BBC_list+USAT_list+NYT_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We now convert the list to a dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "Newsdf = pd.DataFrame(News, columns = ['date', 'News_fount' ,'link','Titles', 'Summary'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The sumaries inlcuded in the rss are a little dirty, so let's clean them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_sum(s):\n",
    "    #s = s\n",
    "    s = re.sub('<.{1,}>', ' ', s)\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "Newsdf['Summary'] = Newsdf['Summary'].apply(clean_sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#In this step, we make a sentiment analysis. We create a column for every sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyzer = SentimentIntensityAnalyzer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def neg(s):\n",
    "    n = analyzer.polarity_scores(s)['neg']\n",
    "    return n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def neut(s):\n",
    "    n = analyzer.polarity_scores(s)['neu']\n",
    "    return n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pos(s):\n",
    "    n = analyzer.polarity_scores(s)['pos']\n",
    "    return n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def com(s):\n",
    "    n = analyzer.polarity_scores(s)['compound']\n",
    "    return n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "Newsdf['Negative'] = Newsdf['Summary'].apply(neg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "Newsdf['Neutral'] = Newsdf['Summary'].apply(neut)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "Newsdf['Positive'] = Newsdf['Summary'].apply(pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "Newsdf['compound'] = Newsdf['Summary'].apply(com)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Now, I clusterize only taking into account the titles (because this way the clusters were more accurate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp=spacy.load('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser=English()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function to tokenize\n",
    "def spacy_tokenizer(sentence):\n",
    "    tokens=parser(sentence)\n",
    "    \n",
    "    filtered_tokens=[]\n",
    "    for word in tokens:\n",
    "        #lemmanising\n",
    "        lemma=word.lemma_.lower().strip()\n",
    "        #print (word, lemma)\n",
    "        if lemma not in STOP_WORDS and re.search('^[a-zA-Z]+$', lemma):\n",
    "            filtered_tokens.append(lemma)\n",
    "    \n",
    "    return filtered_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This model is to convert a collection of raw text to a matrix of TF-IDF features. Note that we are using the funcion of the above cell as a tokenizer.\n",
    "#TF-IDF: Term Frequency Inverse Document Frequency\n",
    "tfidf=TfidfVectorizer(tokenizer=spacy_tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Here we get the TF-IDF features matrix for the titles column.\n",
    "tfidf_matrix=tfidf.fit_transform(Newsdf['Titles'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#terms is a list of the words in the documnet\n",
    "terms=tfidf.get_feature_names()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We use the cosine_similarity to define a distance\n",
    "dist=1-cosine_similarity(tfidf_matrix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#And we cluster\n",
    "hdbscan=HDBSCAN()\n",
    "clusters=hdbscan.fit_predict(dist)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Then, we add the clusters labels to our dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "Newsdf['clusters'] = clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "#In this section, I create a dictionary to save statistics for every cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clusterdf(n_c):\n",
    "    cluster =Newsdf[Newsdf['clusters']== n_c]\n",
    "    sent = cluster.groupby('News_fount').mean()\n",
    "    sent = sent[['compound']]\n",
    "    hist = '. '.join(cluster['Titles']) + ' ' + ' '.join(cluster['Summary'])\n",
    "    return {'title':summary_fun(hist,1, 20),'sumary':summary_fun(hist), 'sent':sent}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "clust = {}\n",
    "for i in set(clusters):\n",
    "    \n",
    "    clust['cluster {}'.format(i)] = clusterdf(i)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#######################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# In next steps, we will need a list of the clusters, so we create it now.\n",
    "lab_clustes = Newsdf.clusters.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Finally, we upload the data to mongo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#I know that here we have a password, but Ulises agreed to do it, since this is only a demo."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# connection string\n",
    "client = pymongo.MongoClient(\"mongodb+srv://uaqro:root@cluster0-tmwuw.mongodb.net/test?retryWrites=true&w=majority\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We define the database\n",
    "db  =client.news_clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We define the collections\n",
    "col_articles = db.articles\n",
    "col_fonts = db.fonts\n",
    "col_media = db.media\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We create a list with the information of every clustar that will be upload to mongo\n",
    "archivos = []\n",
    "for e in lab_clustes:\n",
    "    clustdf = Newsdf[Newsdf['clusters']==e]\n",
    "    clust_e =clust['cluster {}'.format(e)]['sent'].reset_index(drop = False)\n",
    "    bias = {}\n",
    "    for i in range(len(clust_e)):\n",
    "        bias[clust_e.iloc[i]['News_fount']] = clust_e.iloc[i]['compound']\n",
    "    #words = tfidf_df[clusters==e].T.sum(axis=1).sort_values(ascending=False).index\n",
    "    #dictionew = {'headline':words[0].capitalize(), 'summary':words[0].capitalize()+' do '+words[1].capitalize(),\\\n",
    "                 #'ids':list(clustdf.id), 'link':list(clustdf.link)}\n",
    "    #dictionew = {'cluster': str(e) ,'headline':words[0].capitalize(), 'summary':words[0].capitalize()+' do '+words[1].capitalize()}\n",
    "    dictionew = {'cluster': str(e) ,'headline':clust['cluster {}'.format(e)]['title'], 'summary':clust['cluster {}'.format(e)]['sumary'],'bias':bias}\n",
    "    archivos.append(dictionew)\n",
    "    #print(clustdf.link)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We upload the clusters, but we save the ids.\n",
    "idclusts ={}\n",
    "for e in archivos:\n",
    "    documentos = e\n",
    "    #documentos['_id']=c\n",
    "    #print(documentos)\n",
    "    idennt = col_articles.insert_one(documentos).inserted_id\n",
    "    idclusts[str(e['cluster'])] = idennt\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Now we create a file to upload the individual news\n",
    "archi_fonts = []\n",
    "for i in range(len(Newsdf)):\n",
    "    #print(Newsdf['Titles'][i])\n",
    "    dictionew = {'headline':Newsdf['Titles'][i], 'subhead':Newsdf['Summary'][i], 'link':Newsdf['link'][i], 'bias':Newsdf['compound'][i]}\n",
    "    archi_fonts.append(dictionew)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Uploading the news, and saving the ids.\n",
    "mongo_ids = []\n",
    "for e in archi_fonts:\n",
    "    #documentos = e\n",
    "    #documentos['_id']=c\n",
    "    #print(Newsdf.iloc[i]['Titles'])\n",
    "    idennt = col_fonts.insert_one(e).inserted_id\n",
    "    mongo_ids.append(idennt)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We add the ids to the dataframe of news\n",
    "Newsdf['_id'] = mongo_ids\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "bias_bbc = Newsdf.groupby('News_fount').mean()['compound'].loc['British Broadcasting Corporation']\n",
    "bias_fox = 'no available'\n",
    "bias_cnn = Newsdf.groupby('News_fount').mean()['compound'].loc['Cable News Network']\n",
    "bias_nyt = Newsdf.groupby('News_fount').mean()['compound'].loc['New York Times']\n",
    "bias_wsj = Newsdf.groupby('News_fount').mean()['compound'].loc['The Wall Street Journal']\n",
    "bias_usat = Newsdf.groupby('News_fount').mean()['compound'].loc['USA Today']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creation of the datafremo of the news companies\n",
    "media = pd.DataFrame()\n",
    "media['Name'] = ['Cable News Network', 'FOX News', 'The Wall Street Journal', 'British Broadcasting Corporation','USA Today', 'New York Times']\n",
    "media['logo'] = ['http://i2.cdn.turner.com/cnn/2015/images/09/24/cnn.digital.png', 'https://upload.wikimedia.org/wikipedia/commons/6/67/Fox_News_Channel_logo.svg', 'http://online.wsj.com/img/wsj_sm_logo.gif', 'https://news.bbcimg.co.uk/nol/shared/img/bbc_news_120x60.gif', 'https://www.gannett-cdn.com/sites/usatnetwork/images/RSS_Syndication_Logo-USATN.png', 'https://static01.nyt.com/images/misc/NYT_logo_rss_250x40.png']\n",
    "media['web'] = ['https://www.cnn.com', 'https://www.foxnews.com', 'https://www.wsj.com', 'https://www.bbc.com', 'https://www.usatoday.com', 'https://www.nytimes.com']\n",
    "media['bias'] = [bias_cnn, bias_fox, bias_wsj, bias_bbc, bias_usat, bias_nyt]\n",
    "media['city'] = ['Atlanta','New York','New York','London','Jones Branch Drive McLean','New York']\n",
    "media['state'] = ['Georgia','New York','New York','Greater London','Virginia','New York']\n",
    "media['country'] = ['USA','USA','USA','UK','USA','USA']\n",
    "media['twitter'] = ['@CNN','@FoxNews','@WSJ','@BBC','@USATODAY','@nytimes']\n",
    "media['facebook'] = ['www.facebook.com/cnn/','www.facebook.com/FoxNews/','www.facebook.com/wsj/','https://www.facebook.com/bbc/','www.facebook.com/usatoday','https://www.facebook.com/nytimes/']\n",
    "media['level'] = ['International','International','International','International','International','International']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Uploadding the media and saving the ids.\n",
    "news_id = []\n",
    "for i in range(len(media)):\n",
    "    #documentos['_id']=c\n",
    "    #print(Newsdf.iloc[i]['Titles'])\n",
    "    idennt = col_media.insert_one({'name':media.iloc[i]['Name'], 'logo':media.iloc[i]['logo'],\\\n",
    "                                 'web': media.iloc[i]['web'], 'bias': media.iloc[i]['bias'], \\\n",
    "                                 'city': media.iloc[i]['city'], 'state': media.iloc[i]['state'], \\\n",
    "                                  'country': media.iloc[i]['country'], 'twitter': media.iloc[i]['twitter'], \\\n",
    "                                  'facebook': media.iloc[i]['facebook'], 'level': media.iloc[i]['level']}).inserted_id\n",
    "    news_id.append(idennt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We add the ids to the media df\n",
    "media['id'] = news_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Here, I add the id of the founts to every cluster\n",
    "for e in archivos:\n",
    "    tem = Newsdf[Newsdf['clusters'] == int(e['cluster'])]\n",
    "    col_articles.update_one({'cluster':e['cluster']}, {'$set':{'fonts':list(tem['_id'])}})\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "#I add the id for the media\n",
    "for i in range(len(media)):\n",
    "    #display(media.iloc[i]['id'])\n",
    "    #display(media.iloc[i]['Name'])\n",
    "    tem = Newsdf[Newsdf['News_fount'] == media.iloc[i]['Name']]\n",
    "    col_media.update_one({'name': media.iloc[i]['Name']}, {'$set':{'articles': list(tem['_id']) }})\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "#And finally I add the id for the fonts\n",
    "for i in range(len(Newsdf)):\n",
    "    #display(media.iloc[i]['id'])\n",
    "    #display(media.iloc[i]['Name'])\n",
    "    tem = Newsdf.iloc[i]\n",
    "    medtem = media[media['Name'] == Newsdf.iloc[i]['News_fount']]\n",
    "    #print(tem)\n",
    "    col_fonts.update_one({'headline': Newsdf.iloc[i]['Titles']}, {'$set':{'media': medtem.iloc[0]['id'] }})\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "###########################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Recomender system (no data yet) from here the code is totally a mess because we can't use it yet\n",
    "#The first step is to get the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'stop' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-76-4f76a9dad686>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'stop' is not defined"
     ]
    }
   ],
   "source": [
    "#This is just a  undefined variable to aboid run unproved code.\n",
    "stop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_users = db.users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "users = list(col_users.find())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utest=[{'_id': {\"$oid\":'5df91d16batsec6ad3c23d93'},\n",
    " 'readArticles': [\"5df92853767604819c456b2e\", \"5df92854767604819c456b33\"],\n",
    " 'commentsPosted': [{\"5df92854767604819c456b33\":'string string string'}],\n",
    " 'keepArticle': [\"5df92853767604819c456b2e\"],\n",
    " 'username': 'uaq'},  {'_id': {\"$oid\":'5df91126batsec6ad3c23d93'},\n",
    " 'readArticles': [\"5df928537676042319c456b2e\", \"5df9427604819c456b33\"],\n",
    " 'commentsPosted': [{\"5df92854404819c456b33\":'This article is so cool XD'}],\n",
    " 'keepArticle': [\"5df92853767604das6b2e\"],\n",
    " 'username': 'uaq'},   {'_id': {\"$oid\":'5df9a1d16bat34c6ad3c23d93'},\n",
    " 'readArticles': [\"5df92853767604819c45d2e\", \"hha2854767604819c456b33\"],\n",
    " 'commentsPosted': [{\"5df92854767604819cs6b33\":'I didnt like it'}],\n",
    " 'keepArticle': [\"5df92853767604819c45de\"],\n",
    " 'username': 'uaq'},   {'_id': {\"$oid\":'5df91d16b56sec6ad3c23d93'},\n",
    " 'readArticles': [\"5df92853767604819c456d2e\", \"5dfs854767604819c456b33\"],\n",
    " 'commentsPosted': [{\"5df92854767604819c456s\":'Boring'}],\n",
    " 'keepArticle': [\"5df92853767604819c456b2d\"],\n",
    " 'username': 'uaq'},   {'_id': {\"$oid\":'5df91d16b78tsec6ad3c23d93'},\n",
    " 'readArticles': [\"5df92853767604819c456b2e\", \"5df92854767604819c456b33\"],\n",
    " 'commentsPosted': [{\"5df92854767604819c456b33\":'Great'}],\n",
    " 'keepArticle': [\"5df92853767604819c456b2e\"],\n",
    " 'username': 'uaq'}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This funtion takes the information of ONE user and returns a dataframe with the user id as a column, the articles the it\n",
    "#has interacted with as indexes and the scores in the registers.\n",
    "def get_data(user):\n",
    "    col = pd.DataFrame()\n",
    "    for e in user['readArticles']:\n",
    "        col[e] = [1]\n",
    "    col.columns\n",
    "    for e in user['keepArticle']:\n",
    "        if e in col.columns:\n",
    "            col[e][0] +=1\n",
    "        else:\n",
    "            col[e] = [1]\n",
    "    coments = user['commentsPosted']\n",
    "    #Here I don't know exactly what the sintaxis will be, but the idea is to give scores according to the comments.\n",
    "    for e in coments:\n",
    "        article = list(e.keys())[0]\n",
    "        \n",
    "        if article in col.columns:\n",
    "            col[article][0] += com(e[article])\n",
    "            #print('si estuvo')\n",
    "    #        #print(col[article][0])\n",
    "    #        #print(com(e[article]))\n",
    "    #    #print(e[article])\n",
    "        else: \n",
    "            col[article] = com(e[article])\n",
    "            #print('no estuvo')\n",
    "    col['user']=[user['_id']['$oid']]\n",
    "    col.set_index(['user'], inplace = True)\n",
    "    #col = col.T\n",
    "    return col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "us_art = pd.DataFrame()\n",
    "for e in utest:\n",
    "    us_art = us_art.append(get_data(e))\n",
    "    #print(get_data(e))\n",
    "    \n",
    "   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "us_art = us_art.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "us_art = us_art.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "us_art"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We have the data, now the recomandation system."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we choose a new user with some condition, for example, date of last update\n",
    "#new_user = list(col_users.find(...))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_user = {'_id': {\"$oid\":'new'},\n",
    " 'readArticles': [\"5df92853767604819c456b2e\"],\n",
    " 'commentsPosted': [],\n",
    " 'keepArticle': [],\n",
    " 'username': 'new'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nu = get_data(new_user)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "us_art = us_art.T\n",
    "us_art = us_art.append(nu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "us_art = us_art.T\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "us_art = us_art.fillna(0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial.distance import pdist, squareform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "distancias=squareform(pdist(us_art.T, 'euclidean'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "similitud=pd.DataFrame(1/(1+distancias), index=us_art.columns, columns=us_art.columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sim_user =similitud.new.sort_values(ascending=False)[1:2].index[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sim = us_art[sim_user].reset_index(drop=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val = sim[sim[sim_user] == 2]['index']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_recom =val.reset_index(drop=True)[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_users.update_one({'id': 'new_user_id'}, {'$set':{'recomendation': id_recom }})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numb_clust = len(Newsdf.clusters.unique())\n",
    "numb_clust"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "documentos = {\"_id\":75,\"company\":\"Capital One\",\n",
    "\"city\":\"McLean\",\n",
    "\"state\":\"VA\",\n",
    "\"country\":\"USA\"}\n",
    "# insert document into collection\n",
    "idennt = colec.insert_one(documentos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "practice = top_words_cluster.keys()[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp(practice[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "top_words_cluster=tfidf_df[clusters==3].T.sum(axis=1).sort_values(ascending=False)\n",
    "top_words_cluster.index[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import datetime module\n",
    "import datetime\n",
    "# import pymongo module\n",
    "import pymongo\n",
    "#import dns\n",
    "# connection string\n",
    "client = pymongo.MongoClient(\"mongodb+srv://uaqro:Croqueta1.@cluster0-tmwuw.mongodb.net/test?retryWrites=true&w=majority\")\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "db  =client.news_clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colec = db.news\n",
    "colec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "documentos = {\"_id\":75,\"company\":\"Capital One\",\n",
    "\"city\":\"McLean\",\n",
    "\"state\":\"VA\",\n",
    "\"country\":\"USA\"}\n",
    "# insert document into collection\n",
    "idennt = colec.insert_one(documentos)\n",
    "idennt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "article = {\n",
    "  headline: \"Aquí va el nuevo headline generado por nuestra hiper chingona IA\",\n",
    "  resume: \"Aquí va el resumen que nos hemos inventado\",\n",
    "  fonts: [\"Aquí\", \"van\", \"cada\", \"uno\", \"de los ids de las artículos del cluster\"]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nouns = []\n",
    "adjectives = []\n",
    "for token in practice:\n",
    "    pass\n",
    "    if token.pos_ == 'NOUN':\n",
    "        nouns.append(token)\n",
    "    if token.pos_ == 'ADJ':\n",
    "        adjectives.append(token)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = nlp(\"We try to explicitly describe the geometry of the edges of the images.\")\n",
    "\n",
    "for np in doc.noun_chunks: # use np instead of np.text\n",
    "    #print(np)\n",
    "    pass\n",
    "\n",
    "print()\n",
    "\n",
    "# code to recursively combine nouns\n",
    "# 'We' is actually a pronoun but included in your question\n",
    "# hence the token.pos_ == \"PRON\" part in the last if statement\n",
    "# suggest you extract PRON separately like the noun-chunks above\n",
    "\n",
    "index = 0\n",
    "nounIndices = []\n",
    "for token in doc:\n",
    "    # print(token.text, token.pos_, token.dep_, token.head.text)\n",
    "    if token.pos_ == 'NOUN':\n",
    "        nounIndices.append(index)\n",
    "        print()\n",
    "    index = index + 1\n",
    "\n",
    "\n",
    "print(nounIndices)\n",
    "for idxValue in nounIndices:\n",
    "    doc = nlp(\"We try to explicitly describe the geometry of the edges of the images.\")\n",
    "    span = doc[doc[idxValue].left_edge.i : doc[idxValue].right_edge.i+1]\n",
    "    span.merge()\n",
    "\n",
    "    for token in doc:\n",
    "        if token.dep_ == 'dobj' or token.dep_ == 'pobj' or token.pos_ == \"PRON\":\n",
    "            print(token.text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}