ScrapMyMinistres.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Scrap them all !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib import request, response\n",
    "import re\n",
    "\n",
    "path_file = '/media/flo/Stockage/Data/ScrapMyMinistres/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.4/dist-packages/IPython/parallel.py:13: ShimWarning: The `IPython.parallel` package has been deprecated. You should import from ipyparallel instead.\n",
      "  \"You should import from ipyparallel instead.\", ShimWarning)\n"
     ]
    }
   ],
   "source": [
    "%qtconsole"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def clean_ministres_fonctions(fonction_ministres):\n",
    "    fonction_ministres_clean =  [x for x in fonction_ministres if any([len(x) < 5, '[' not in x ])]\n",
    "    fonction_ministres_clean = '#'.join(fonction_ministres_clean).split('##')\n",
    "    fonction_ministres_clean = [x.replace('#', ' / ') for x in fonction_ministres_clean]\n",
    "\n",
    "    return fonction_ministres_clean\n",
    "\n",
    "def clean_ministres_wiki(wiki_ministre) : \n",
    "    wiki_ministres_clean = [x for x in wiki_ministre if ('France' not in x) & \n",
    "                            ('jpg' not in x) & ('Fichier' not in x) & any(['(' in x or 'Conway' in x, len(x) <= 35]) &  # Philipe martin (homme politique, ...)\n",
    "                            ('gauche' not in x) & ('Outre-' not in x) & ('Ville' not in (x)) &\n",
    "                            ('Logement' not in x) & ('Budget' not in x)  & ('Walwari' not in x) &\n",
    "                            ('#cite' not in x) & ('tiquette' not in x) & ('cologistes' not in x) & ('cologie' not in x)]\n",
    "    print('%s ministres trouvés' % (len(wiki_ministres_clean)))\n",
    "    \n",
    "    return wiki_ministres_clean\n",
    "# le any(['(' in x or 'Conway' in x, len(x) <= 36]) sert à conserver certains wiki atypiques (Conway = très long, Philippe Martin = (homme polituque..))\n",
    "\n",
    "def get_my_gouv(link_gouv):\n",
    "    # Request\n",
    "    header = {'User-Agent': 'Mozilla/5.0'} # Needed to prevent 403 error on Wikipedia\n",
    "    page = request.Request(link_gouv, headers=header)\n",
    "    html = request.urlopen(page) \n",
    "    soup = BeautifulSoup(html, \"lxml\")\n",
    "    cells = soup.findAll(\"table\", {\"class\": \"wikitable sortable\"})\n",
    "    #  Init\n",
    "    wiki_ministres = []\n",
    "    fonction_ministres = [] \n",
    "    n_ministres = []\n",
    "    n_min = 0\n",
    "    # Scraping\n",
    "    for cell in cells: # Tableau principal\n",
    "        cells_href = cell.findAll(\"a\", href=True)\n",
    "        for c_href in cells_href: # éléments de chaque tableau\n",
    "            wiki_ministres.append(c_href['href'])\n",
    "            fonction_ministres.append(c_href.get_text())\n",
    "            n_ministres.append(n_min)\n",
    "            n_min += 1\n",
    "    wiki_ministres_clean = clean_ministres_wiki(wiki_ministres)\n",
    "    fonction_ministres_clean = clean_ministres_fonctions(fonction_ministres)\n",
    "    return (fonction_ministres_clean, wiki_ministres_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39 ministres trouvés\n",
      "32 ministres trouvés\n",
      "38 ministres trouvés\n",
      "35 ministres trouvés\n"
     ]
    }
   ],
   "source": [
    "dict_holland = {'holland_ayrault_1': \"/wiki/Gouvernement_Jean-Marc_Ayrault_(1)\",\n",
    "                'holland_ayrault_2': \"/wiki/Gouvernement_Jean-Marc_Ayrault_(2)\",\n",
    "                'holland_valls_1': \"/wiki/Gouvernement_Manuel_Valls_(1)\",\n",
    "                'holland_valls_2': \"/wiki/Gouvernement_Manuel_Valls_(2)\" }\n",
    "\n",
    "all_gouv = pd.DataFrame(columns=['gouvernement', 'wiki', 'fonctions'])\n",
    "for keys, values in dict_holland.items():\n",
    "    gouv = pd.DataFrame()\n",
    "    (fonction_ministres_clean, wiki_ministres_clean) = get_my_gouv('https://fr.wikipedia.org' + values)\n",
    "    gouv[\"wiki\"] = wiki_ministres_clean\n",
    "    gouv[\"fonctions\"] = fonction_ministres_clean\n",
    "    gouv[\"gouvernement\"] = keys\n",
    "    all_gouv = pd.concat([all_gouv, gouv])\n",
    "    \n",
    "# -- Export\n",
    "all_gouv = all_gouv.ix[:, ['gouvernement', 'wiki', 'fonctions']] # changement de l'ordre des colonnes\n",
    "all_gouv.to_csv(path_file + 'gouvernements_ministres.csv', sep=';',encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gouvernement</th>\n",
       "      <th>wiki</th>\n",
       "      <th>fonctions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>holland_valls_2</td>\n",
       "      <td>/wiki/Manuel_Valls</td>\n",
       "      <td>/ Premier ministre / Manuel Valls / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>holland_valls_2</td>\n",
       "      <td>/wiki/Jean-Marc_Ayrault</td>\n",
       "      <td>Ministre des Affaires étrangères et du Dévelop...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>holland_valls_2</td>\n",
       "      <td>/wiki/S%C3%A9gol%C3%A8ne_Royal</td>\n",
       "      <td>Ministre de l'Environnement, de l'Énergie et d...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      gouvernement                            wiki  \\\n",
       "0  holland_valls_2              /wiki/Manuel_Valls   \n",
       "1  holland_valls_2         /wiki/Jean-Marc_Ayrault   \n",
       "2  holland_valls_2  /wiki/S%C3%A9gol%C3%A8ne_Royal   \n",
       "\n",
       "                                           fonctions  \n",
       "0             / Premier ministre / Manuel Valls / PS  \n",
       "1  Ministre des Affaires étrangères et du Dévelop...  \n",
       "2  Ministre de l'Environnement, de l'Énergie et d...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_gouv.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test\n",
    "Vérifions qu'il ya bien autant de ministres que de fonction. Visuellement."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39 ministres trouvés\n"
     ]
    }
   ],
   "source": [
    "(fonction_ministres_clean, wiki_ministres_clean) = get_my_gouv(\"https://fr.wikipedia.org\" + dict_holland[\"holland_valls_2\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wiki_ministres</th>\n",
       "      <th>fonction_ministres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/wiki/Manuel_Valls</td>\n",
       "      <td>/ Premier ministre / Manuel Valls / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/wiki/Jean-Marc_Ayrault</td>\n",
       "      <td>Ministre des Affaires étrangères et du Développement international / Jean-Marc Ayrault / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/wiki/S%C3%A9gol%C3%A8ne_Royal</td>\n",
       "      <td>Ministre de l'Environnement, de l'Énergie et de la Mer, chargée des relations internationales sur le climat / Ségolène Royal / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/wiki/Najat_Vallaud-Belkacem</td>\n",
       "      <td>Ministre de l'Éducation nationale / de l'Enseignement supérieur et de la Recherche / Najat Vallaud-Belkacem / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/wiki/Michel_Sapin</td>\n",
       "      <td>Ministre des Finances et des Comptes publics / Michel Sapin / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>/wiki/Marisol_Touraine</td>\n",
       "      <td>Ministre des Affaires sociales et de la Santé / Marisol Touraine / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>/wiki/Jean-Yves_Le_Drian</td>\n",
       "      <td>Ministre de la Défense / Jean-Yves Le Drian / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>/wiki/Jean-Jacques_Urvoas</td>\n",
       "      <td>Garde des Sceaux, ministre de la Justice / Jean-Jacques Urvoas / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>/wiki/Myriam_El_Khomri</td>\n",
       "      <td>Ministre du Travail, de l'Emploi, de la Formation professionnelle et du Dialogue social / Myriam El Khomri / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>/wiki/Jean-Michel_Baylet</td>\n",
       "      <td>Ministre de l'Aménagement du territoire, de la Ruralité et des Collectivités territoriales / Jean-Michel Baylet / PRG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>/wiki/Bernard_Cazeneuve</td>\n",
       "      <td>Ministre de l'Intérieur / Bernard Cazeneuve / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>/wiki/St%C3%A9phane_Le_Foll</td>\n",
       "      <td>Ministre de l'Agriculture, de l'Agroalimentaire et de la Forêt / Porte-parole du gouvernement / Stéphane Le Foll / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>/wiki/Emmanuelle_Cosse</td>\n",
       "      <td>Ministre du Logement et de l'Habitat durable / Emmanuelle Cosse / Divers écologiste / [25]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>/wiki/Emmanuel_Macron</td>\n",
       "      <td>Ministre de l'Économie / de l'Industrie / du Numérique / Emmanuel Macron / Sans étiquette</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>/wiki/Audrey_Azoulay</td>\n",
       "      <td>Ministre de la Culture et de la Communication / Audrey Azoulay / Sans étiquette</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>/wiki/Laurence_Rossignol</td>\n",
       "      <td>Ministre des Familles, de l'Enfance / des Droits des femmes / Laurence Rossignol / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>/wiki/Annick_Girardin</td>\n",
       "      <td>Ministre de la Fonction publique / Annick Girardin / PRG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>/wiki/Patrick_Kanner</td>\n",
       "      <td>Ministre de la Ville / de la Jeunesse et des Sports / Patrick Kanner / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>/wiki/George_Pau-Langevin</td>\n",
       "      <td>Ministre des Outre-mer / George Pau-Langevin / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>/wiki/Jean-Marie_Le_Guen</td>\n",
       "      <td>Secrétaire d'État chargé des Relations avec le Parlement / [26] / Premier ministre / Jean-Marie Le Guen / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>/wiki/Ericka_Bareigts</td>\n",
       "      <td>Premier ministre / Ericka Bareigts / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>/wiki/Jean-Vincent_Plac%C3%A9</td>\n",
       "      <td>Secrétaire d'État chargé de la Réforme de l'État et de la Simplification / Premier ministre / Jean-Vincent Placé / Écologistes !</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>/wiki/Juliette_M%C3%A9adel</td>\n",
       "      <td>Secrétaire d’État chargée de l’Aide aux victimes / Premier ministre / Juliette Méadel / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>/wiki/Harlem_D%C3%A9sir</td>\n",
       "      <td>Secrétaire d'État chargé des Affaires européennes / Ministre des Affaires étrangères et du Développement international / Harlem Désir / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>/wiki/Matthias_Fekl</td>\n",
       "      <td>Secrétaire d'État chargée du Commerce extérieur, de la Promotion du tourisme et des Français de l'étranger / Ministre des Affaires étrangères et du Développement international / Matthias Fekl / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>/wiki/Andr%C3%A9_Vallini</td>\n",
       "      <td>Secrétaire d'État chargée du Développement et de la Francophonie / Ministre des Affaires étrangères et du Développement international / André Vallini / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>/wiki/Alain_Vidalies</td>\n",
       "      <td>Secrétaire d'État chargé des Transports, de la Mer et de la Pêche / Ministre de l'Environnement, de l'Énergie et de la Mer, chargée des relations internationales sur le climat / Alain Vidalies / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>/wiki/Barbara_Pompili</td>\n",
       "      <td>Secrétaire d’État chargée de la Biodiversité / Ministre de l'Environnement, de l'Énergie et de la Mer, chargée des relations internationales sur le climat / Barbara Pompili / Divers écologiste</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>/wiki/Thierry_Mandon</td>\n",
       "      <td>Secrétaire d'État chargé de l'Enseignement supérieur et de la Recherche / Ministre de l'Éducation nationale, de l'Enseignement supérieur et de la Recherche / Thierry Mandon / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>/wiki/Christian_Eckert</td>\n",
       "      <td>Secrétaire d'État chargé du Budget / Ministre des Finances et des Comptes publics / Christian Eckert / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>/wiki/S%C3%A9gol%C3%A8ne_Neuville</td>\n",
       "      <td>Secrétaire d'État chargée des Personnes handicapées et de la Lutte contre l'exclusion / Ministre des Affaires sociales et de la Santé / Ségolène Neuville / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>/wiki/Pascale_Boistard</td>\n",
       "      <td>Secrétaire d'État chargée des Personnes âgées et de l'Autonomie / Ministre des Affaires sociales et de la Santé / Pascale Boistard / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>/wiki/Jean-Marc_Todeschini</td>\n",
       "      <td>Secrétaire d'État chargé des Anciens combattants et de la Mémoire / Ministre de la Défense / Jean-Marc Todeschini / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>/wiki/Clotilde_Valter</td>\n",
       "      <td>Secrétaire d'État chargée de la Formation professionnelle et de l’Apprentissage / Ministre du Travail, de l'Emploi, de la Formation professionnelle et du Dialogue social / Clotilde Valter / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>/wiki/Estelle_Grelier</td>\n",
       "      <td>Secrétaire d'État chargée des Collectivités territoriales / Ministre de l'Aménagement du territoire, de la Ruralité et des Collectivités territoriales / Estelle Grelier / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>/wiki/Martine_Pinville</td>\n",
       "      <td>Secrétaire d'État chargée du Commerce, de l'Artisanat, de la Consommation et de l'Économie sociale et solidaire / Ministre de l'Économie, de l'Industrie et du Numérique / Martine Pinville / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>/wiki/Axelle_Lemaire</td>\n",
       "      <td>Secrétaire d'État chargée du Numérique / Ministre de l'Économie, de l'Industrie et du Numérique / Axelle Lemaire / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>/wiki/H%C3%A9l%C3%A8ne_Geoffroy</td>\n",
       "      <td>Secrétaire d'État chargée de la Ville / Ministre de la Ville / de la Jeunesse et des Sports / Hélène Geoffroy / PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>/wiki/Thierry_Braillard</td>\n",
       "      <td>Secrétaire d'État chargé des Sports / Ministre de la Ville, de la Jeunesse et des Sports / Thierry Braillard / PRG</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       wiki_ministres  \\\n",
       "0                  /wiki/Manuel_Valls   \n",
       "1             /wiki/Jean-Marc_Ayrault   \n",
       "2      /wiki/S%C3%A9gol%C3%A8ne_Royal   \n",
       "3        /wiki/Najat_Vallaud-Belkacem   \n",
       "4                  /wiki/Michel_Sapin   \n",
       "5              /wiki/Marisol_Touraine   \n",
       "6            /wiki/Jean-Yves_Le_Drian   \n",
       "7           /wiki/Jean-Jacques_Urvoas   \n",
       "8              /wiki/Myriam_El_Khomri   \n",
       "9            /wiki/Jean-Michel_Baylet   \n",
       "10            /wiki/Bernard_Cazeneuve   \n",
       "11        /wiki/St%C3%A9phane_Le_Foll   \n",
       "12             /wiki/Emmanuelle_Cosse   \n",
       "13              /wiki/Emmanuel_Macron   \n",
       "14               /wiki/Audrey_Azoulay   \n",
       "15           /wiki/Laurence_Rossignol   \n",
       "16              /wiki/Annick_Girardin   \n",
       "17               /wiki/Patrick_Kanner   \n",
       "18          /wiki/George_Pau-Langevin   \n",
       "19           /wiki/Jean-Marie_Le_Guen   \n",
       "20              /wiki/Ericka_Bareigts   \n",
       "21      /wiki/Jean-Vincent_Plac%C3%A9   \n",
       "22         /wiki/Juliette_M%C3%A9adel   \n",
       "23            /wiki/Harlem_D%C3%A9sir   \n",
       "24                /wiki/Matthias_Fekl   \n",
       "25           /wiki/Andr%C3%A9_Vallini   \n",
       "26               /wiki/Alain_Vidalies   \n",
       "27              /wiki/Barbara_Pompili   \n",
       "28               /wiki/Thierry_Mandon   \n",
       "29             /wiki/Christian_Eckert   \n",
       "30  /wiki/S%C3%A9gol%C3%A8ne_Neuville   \n",
       "31             /wiki/Pascale_Boistard   \n",
       "32         /wiki/Jean-Marc_Todeschini   \n",
       "33              /wiki/Clotilde_Valter   \n",
       "34              /wiki/Estelle_Grelier   \n",
       "35             /wiki/Martine_Pinville   \n",
       "36               /wiki/Axelle_Lemaire   \n",
       "37    /wiki/H%C3%A9l%C3%A8ne_Geoffroy   \n",
       "38            /wiki/Thierry_Braillard   \n",
       "\n",
       "                                                                                                                                                                                       fonction_ministres  \n",
       "0                                                                                                                                                                  / Premier ministre / Manuel Valls / PS  \n",
       "1                                                                                                             Ministre des Affaires étrangères et du Développement international / Jean-Marc Ayrault / PS  \n",
       "2                                                                       Ministre de l'Environnement, de l'Énergie et de la Mer, chargée des relations internationales sur le climat / Ségolène Royal / PS  \n",
       "3                                                                                        Ministre de l'Éducation nationale / de l'Enseignement supérieur et de la Recherche / Najat Vallaud-Belkacem / PS  \n",
       "4                                                                                                                                        Ministre des Finances et des Comptes publics / Michel Sapin / PS  \n",
       "5                                                                                                                                   Ministre des Affaires sociales et de la Santé / Marisol Touraine / PS  \n",
       "6                                                                                                                                                        Ministre de la Défense / Jean-Yves Le Drian / PS  \n",
       "7                                                                                                                                     Garde des Sceaux, ministre de la Justice / Jean-Jacques Urvoas / PS  \n",
       "8                                                                                         Ministre du Travail, de l'Emploi, de la Formation professionnelle et du Dialogue social / Myriam El Khomri / PS  \n",
       "9                                                                                   Ministre de l'Aménagement du territoire, de la Ruralité et des Collectivités territoriales / Jean-Michel Baylet / PRG  \n",
       "10                                                                                                                                                       Ministre de l'Intérieur / Bernard Cazeneuve / PS  \n",
       "11                                                                                  Ministre de l'Agriculture, de l'Agroalimentaire et de la Forêt / Porte-parole du gouvernement / Stéphane Le Foll / PS  \n",
       "12                                                                                                             Ministre du Logement et de l'Habitat durable / Emmanuelle Cosse / Divers écologiste / [25]  \n",
       "13                                                                                                              Ministre de l'Économie / de l'Industrie / du Numérique / Emmanuel Macron / Sans étiquette  \n",
       "14                                                                                                                        Ministre de la Culture et de la Communication / Audrey Azoulay / Sans étiquette  \n",
       "15                                                                                                                  Ministre des Familles, de l'Enfance / des Droits des femmes / Laurence Rossignol / PS  \n",
       "16                                                                                                                                               Ministre de la Fonction publique / Annick Girardin / PRG  \n",
       "17                                                                                                                              Ministre de la Ville / de la Jeunesse et des Sports / Patrick Kanner / PS  \n",
       "18                                                                                                                                                      Ministre des Outre-mer / George Pau-Langevin / PS  \n",
       "19                                                                                           Secrétaire d'État chargé des Relations avec le Parlement / [26] / Premier ministre / Jean-Marie Le Guen / PS  \n",
       "20                                                                                                                                                                Premier ministre / Ericka Bareigts / PS  \n",
       "21                                                                       Secrétaire d'État chargé de la Réforme de l'État et de la Simplification / Premier ministre / Jean-Vincent Placé / Écologistes !  \n",
       "22                                                                                                             Secrétaire d’État chargée de l’Aide aux victimes / Premier ministre / Juliette Méadel / PS  \n",
       "23                                                             Secrétaire d'État chargé des Affaires européennes / Ministre des Affaires étrangères et du Développement international / Harlem Désir / PS  \n",
       "24   Secrétaire d'État chargée du Commerce extérieur, de la Promotion du tourisme et des Français de l'étranger / Ministre des Affaires étrangères et du Développement international / Matthias Fekl / PS  \n",
       "25                                             Secrétaire d'État chargée du Développement et de la Francophonie / Ministre des Affaires étrangères et du Développement international / André Vallini / PS  \n",
       "26  Secrétaire d'État chargé des Transports, de la Mer et de la Pêche / Ministre de l'Environnement, de l'Énergie et de la Mer, chargée des relations internationales sur le climat / Alain Vidalies / PS  \n",
       "27       Secrétaire d’État chargée de la Biodiversité / Ministre de l'Environnement, de l'Énergie et de la Mer, chargée des relations internationales sur le climat / Barbara Pompili / Divers écologiste  \n",
       "28                      Secrétaire d'État chargé de l'Enseignement supérieur et de la Recherche / Ministre de l'Éducation nationale, de l'Enseignement supérieur et de la Recherche / Thierry Mandon / PS  \n",
       "29                                                                                              Secrétaire d'État chargé du Budget / Ministre des Finances et des Comptes publics / Christian Eckert / PS  \n",
       "30                                         Secrétaire d'État chargée des Personnes handicapées et de la Lutte contre l'exclusion / Ministre des Affaires sociales et de la Santé / Ségolène Neuville / PS  \n",
       "31                                                                Secrétaire d'État chargée des Personnes âgées et de l'Autonomie / Ministre des Affaires sociales et de la Santé / Pascale Boistard / PS  \n",
       "32                                                                                 Secrétaire d'État chargé des Anciens combattants et de la Mémoire / Ministre de la Défense / Jean-Marc Todeschini / PS  \n",
       "33       Secrétaire d'État chargée de la Formation professionnelle et de l’Apprentissage / Ministre du Travail, de l'Emploi, de la Formation professionnelle et du Dialogue social / Clotilde Valter / PS  \n",
       "34                          Secrétaire d'État chargée des Collectivités territoriales / Ministre de l'Aménagement du territoire, de la Ruralité et des Collectivités territoriales / Estelle Grelier / PS  \n",
       "35       Secrétaire d'État chargée du Commerce, de l'Artisanat, de la Consommation et de l'Économie sociale et solidaire / Ministre de l'Économie, de l'Industrie et du Numérique / Martine Pinville / PS  \n",
       "36                                                                                  Secrétaire d'État chargée du Numérique / Ministre de l'Économie, de l'Industrie et du Numérique / Axelle Lemaire / PS  \n",
       "37                                                                                     Secrétaire d'État chargée de la Ville / Ministre de la Ville / de la Jeunesse et des Sports / Hélène Geoffroy / PS  \n",
       "38                                                                                     Secrétaire d'État chargé des Sports / Ministre de la Ville, de la Jeunesse et des Sports / Thierry Braillard / PRG  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.options.display.max_colwidth = 300\n",
    "df = pd.DataFrame()\n",
    "df['wiki_ministres'] = wiki_ministres_clean\n",
    "df1 = pd.DataFrame()\n",
    "df1['fonction_ministres'] = pd.Series(fonction_ministres_clean)\n",
    "\n",
    "df.merge(df1, left_index=True, right_index=True, how='outer')  # changer 'left' ou 'right' pour identifier un éventuel pb\n",
    "#df['fonctions_ministres'] = fonction_ministres_clean"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  * Nettoyage sur LibreOffice * \n",
    "\n",
    "Clean des colonnes su LibreOffice car texte parfois trop non-struturé."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "all_gouv_clean = pd.read_csv(path_file + 'gouvernements_ministres_clean.csv', sep=';', header=0)\n",
    "all_gouv_clean.columns = ['gouvernement', 'wiki', 'fonctions', 'Nom', 'Parti']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gouvernement</th>\n",
       "      <th>wiki</th>\n",
       "      <th>fonctions</th>\n",
       "      <th>Nom</th>\n",
       "      <th>Parti</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>holland_valls_2</td>\n",
       "      <td>/wiki/Manuel_Valls</td>\n",
       "      <td>Premier ministre</td>\n",
       "      <td>Manuel Valls</td>\n",
       "      <td>PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>holland_valls_2</td>\n",
       "      <td>/wiki/Jean-Marc_Ayrault</td>\n",
       "      <td>Ministre des Affaires étrangères et du Développement international</td>\n",
       "      <td>Jean-Marc Ayrault</td>\n",
       "      <td>PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>holland_valls_2</td>\n",
       "      <td>/wiki/S%C3%A9gol%C3%A8ne_Royal</td>\n",
       "      <td>Ministre de l'Environnement, de l'Énergie et de la Mer, chargée des relations internationales sur le climat</td>\n",
       "      <td>Ségolène Royal</td>\n",
       "      <td>PS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      gouvernement                            wiki  \\\n",
       "0  holland_valls_2              /wiki/Manuel_Valls   \n",
       "1  holland_valls_2         /wiki/Jean-Marc_Ayrault   \n",
       "2  holland_valls_2  /wiki/S%C3%A9gol%C3%A8ne_Royal   \n",
       "\n",
       "                                                                                                      fonctions  \\\n",
       "0                                                                                             Premier ministre    \n",
       "1                                           Ministre des Affaires étrangères et du Développement international    \n",
       "2  Ministre de l'Environnement, de l'Énergie et de la Mer, chargée des relations internationales sur le climat    \n",
       "\n",
       "                   Nom Parti  \n",
       "0        Manuel Valls     PS  \n",
       "1   Jean-Marc Ayrault     PS  \n",
       "2      Ségolène Royal     PS  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_gouv_clean.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Looping the ministres"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## select a ministre\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dict_holland_char = {'holland_ayrault_1': \"/wiki/Gouvernement_Jean-Marc_Ayrault_\\(1\\)\",\n",
    "                    'holland_ayrault_2': \"/wiki/Gouvernement_Jean-Marc_Ayrault_\\(2\\)\",\n",
    "                    'holland_valls_1': \"/wiki/Gouvernement_Manuel_Valls_\\(1\\)\",\n",
    "                    'holland_valls_2': \"/wiki/Gouvernement_Manuel_Valls_\\(2\\)\" }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_lifespan_ministres(gouv_name):\n",
    "    (fonction_ministres_clean, wiki_ministres_clean) = get_my_gouv('https://fr.wikipedia.org' + dict_holland[gouv_name])\n",
    "    all_dates = []\n",
    "    all_birthdays = []\n",
    "    all_names = []\n",
    "    for ministre in wiki_ministres_clean:\n",
    "        page_ministre = \"https://fr.wikipedia.org\" + ministre\n",
    "        header = {'User-Agent': 'Mozilla/5.0'} #Needed to prevent 403 error on Wikipedia\n",
    "        page = request.Request(page_ministre, headers=header)\n",
    "        html = request.urlopen(page)\n",
    "        soup = BeautifulSoup(html, \"lxml\")\n",
    "        cells = soup.find('table', {'class': 'infobox_v2'})  # selection du tableau de droite\n",
    "        # -- Names\n",
    "        all_names.append(cells.find('td', {'class': 'entete defaut'}).getText())\n",
    "        # -- Birthdays\n",
    "        all_birthdays.append(cells.find('time', {'class': 'nowrap date-lien bday'})['datetime'])\n",
    "        # -- Lifespan\n",
    "        cells_gouv = cells.find('a', href=re.compile(dict_holland_char[gouv_name]))  # selection du gouvernement\n",
    "        l_dates = []\n",
    "        try: # recherche de(s) (la) date(s) immédiatement au dessus de 'gouvernement'\n",
    "            cells_date = cells_gouv.findPrevious('b').findAll('time')  \n",
    "            for c_dates in cells_date:\n",
    "                l_dates.append(c_dates['datetime'])\n",
    "        except AttributeError:\n",
    "            print(\"pas de date trouvée pour le ministre %s\" % (ministre))\n",
    "            l_dates.append(\"Non_trouvee\")\n",
    "        all_dates.append(l_dates)\n",
    "\n",
    "    # -- clean date\n",
    "    date_debut = [x[0] for x in all_dates]\n",
    "    date_fin = [x[1] if (len(x) == 2) else pd.datetime.today().date() for x in all_dates]\n",
    "    # -- Results table    \n",
    "    df_date = pd.DataFrame()\n",
    "    df_date['wiki'] = wiki_ministres_clean\n",
    "    df_date['Nom'] = all_names\n",
    "    df_date['birthday'] = pd.to_datetime(all_birthdays, errors='coerce')\n",
    "    df_date['date_debut'] = pd.to_datetime(date_debut, errors='coerce')\n",
    "    df_date['date_fin'] = pd.to_datetime(date_fin, errors='coerce')\n",
    "    df_date['duree_mendat'] = pd.TimedeltaIndex(df_date['date_fin'] - df_date['date_debut']).days\n",
    "    df_date['gouvernement'] = gouv_name\n",
    "    df_date = df_date.ix[:, ['gouvernement', 'wiki', 'Nom', 'birthday', 'date_debut', 'date_fin', 'duree_mendat']]\n",
    "    return df_date"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39 ministres trouvés\n",
      "pas de date trouvée pour le ministre /wiki/Juliette_M%C3%A9adel\n",
      "32 ministres trouvés\n",
      "38 ministres trouvés\n",
      "35 ministres trouvés\n"
     ]
    }
   ],
   "source": [
    "all_lifespan = pd.DataFrame()\n",
    "for keys, values in dict_holland_char.items():\n",
    "    df_date = get_lifespan_ministres(keys)\n",
    "    all_lifespan = pd.concat([df_date, all_lifespan])\n",
    "    all_lifespan.to_csv(path_file + 'ministres_lifespan.csv', sep=';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gouvernement</th>\n",
       "      <th>wiki</th>\n",
       "      <th>Nom</th>\n",
       "      <th>birthday</th>\n",
       "      <th>date_debut</th>\n",
       "      <th>date_fin</th>\n",
       "      <th>duree_mendat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>holland_ayrault_1</td>\n",
       "      <td>/wiki/Jean-Marc_Ayrault</td>\n",
       "      <td>Jean-Marc Ayrault</td>\n",
       "      <td>1950-01-25</td>\n",
       "      <td>2012-05-15</td>\n",
       "      <td>2014-03-31</td>\n",
       "      <td>685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>holland_ayrault_1</td>\n",
       "      <td>/wiki/Laurent_Fabius</td>\n",
       "      <td>Laurent Fabius</td>\n",
       "      <td>1946-08-20</td>\n",
       "      <td>2012-05-16</td>\n",
       "      <td>2016-02-11</td>\n",
       "      <td>1366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>holland_ayrault_1</td>\n",
       "      <td>/wiki/Vincent_Peillon</td>\n",
       "      <td>Vincent Peillon</td>\n",
       "      <td>1960-07-07</td>\n",
       "      <td>2012-05-16</td>\n",
       "      <td>2014-03-31</td>\n",
       "      <td>684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>holland_ayrault_1</td>\n",
       "      <td>/wiki/Christiane_Taubira</td>\n",
       "      <td>Christiane Taubira</td>\n",
       "      <td>1952-02-02</td>\n",
       "      <td>2012-05-16</td>\n",
       "      <td>2016-01-27</td>\n",
       "      <td>1351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>holland_ayrault_1</td>\n",
       "      <td>/wiki/Pierre_Moscovici</td>\n",
       "      <td>Pierre Moscovici</td>\n",
       "      <td>1957-09-16</td>\n",
       "      <td>2012-05-16</td>\n",
       "      <td>2014-03-31</td>\n",
       "      <td>684</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        gouvernement                      wiki                 Nom   birthday  \\\n",
       "0  holland_ayrault_1   /wiki/Jean-Marc_Ayrault   Jean-Marc Ayrault 1950-01-25   \n",
       "1  holland_ayrault_1      /wiki/Laurent_Fabius      Laurent Fabius 1946-08-20   \n",
       "2  holland_ayrault_1     /wiki/Vincent_Peillon     Vincent Peillon 1960-07-07   \n",
       "3  holland_ayrault_1  /wiki/Christiane_Taubira  Christiane Taubira 1952-02-02   \n",
       "4  holland_ayrault_1    /wiki/Pierre_Moscovici    Pierre Moscovici 1957-09-16   \n",
       "\n",
       "  date_debut   date_fin  duree_mendat  \n",
       "0 2012-05-15 2014-03-31           685  \n",
       "1 2012-05-16 2016-02-11          1366  \n",
       "2 2012-05-16 2014-03-31           684  \n",
       "3 2012-05-16 2016-01-27          1351  \n",
       "4 2012-05-16 2014-03-31           684  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_lifespan.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gouvernement</th>\n",
       "      <th>wiki</th>\n",
       "      <th>fonctions</th>\n",
       "      <th>Nom</th>\n",
       "      <th>Parti</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>holland_valls_2</td>\n",
       "      <td>/wiki/Manuel_Valls</td>\n",
       "      <td>Premier ministre</td>\n",
       "      <td>Manuel Valls</td>\n",
       "      <td>PS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      gouvernement                wiki           fonctions             Nom  \\\n",
       "0  holland_valls_2  /wiki/Manuel_Valls   Premier ministre    Manuel Valls    \n",
       "\n",
       "  Parti  \n",
       "0    PS  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_gouv_clean.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gouvernement</th>\n",
       "      <th>wiki</th>\n",
       "      <th>Nom</th>\n",
       "      <th>birthday</th>\n",
       "      <th>date_debut</th>\n",
       "      <th>date_fin</th>\n",
       "      <th>duree_mendat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>holland_ayrault_1</td>\n",
       "      <td>/wiki/Jean-Marc_Ayrault</td>\n",
       "      <td>Jean-Marc Ayrault</td>\n",
       "      <td>1950-01-25</td>\n",
       "      <td>2012-05-15</td>\n",
       "      <td>2014-03-31</td>\n",
       "      <td>685</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        gouvernement                     wiki                Nom   birthday  \\\n",
       "0  holland_ayrault_1  /wiki/Jean-Marc_Ayrault  Jean-Marc Ayrault 1950-01-25   \n",
       "\n",
       "  date_debut   date_fin  duree_mendat  \n",
       "0 2012-05-15 2014-03-31           685  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_lifespan.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "table_finale = pd.merge(all_lifespan,\n",
    "                        all_gouv_clean[['gouvernement', 'wiki', 'fonctions', 'Parti']],\n",
    "                        on=['gouvernement', 'wiki'])\n",
    "# -- Reorder columns\n",
    "table_finale = table_finale.ix[: ,['gouvernement', 'wiki', 'Nom', 'fonctions',\n",
    "                                   'birthday', 'date_debut', 'date_fin',\n",
    "                                   'duree_mendat', 'Parti']]\n",
    "# -- Clean string\n",
    "table_finale.fonctions = table_finale.fonctions.str.strip().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gouvernement</th>\n",
       "      <th>wiki</th>\n",
       "      <th>Nom</th>\n",
       "      <th>fonctions</th>\n",
       "      <th>birthday</th>\n",
       "      <th>date_debut</th>\n",
       "      <th>date_fin</th>\n",
       "      <th>duree_mendat</th>\n",
       "      <th>Parti</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>holland_ayrault_1</td>\n",
       "      <td>/wiki/Jean-Marc_Ayrault</td>\n",
       "      <td>Jean-Marc Ayrault</td>\n",
       "      <td>Premier ministre</td>\n",
       "      <td>1950-01-25</td>\n",
       "      <td>2012-05-15</td>\n",
       "      <td>2014-03-31</td>\n",
       "      <td>685</td>\n",
       "      <td>PS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>holland_ayrault_1</td>\n",
       "      <td>/wiki/Laurent_Fabius</td>\n",
       "      <td>Laurent Fabius</td>\n",
       "      <td>Ministre des Affaires étrangères</td>\n",
       "      <td>1946-08-20</td>\n",
       "      <td>2012-05-16</td>\n",
       "      <td>2016-02-11</td>\n",
       "      <td>1366</td>\n",
       "      <td>PS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        gouvernement                     wiki                Nom  \\\n",
       "0  holland_ayrault_1  /wiki/Jean-Marc_Ayrault  Jean-Marc Ayrault   \n",
       "1  holland_ayrault_1     /wiki/Laurent_Fabius     Laurent Fabius   \n",
       "\n",
       "                          fonctions   birthday date_debut   date_fin  \\\n",
       "0                  Premier ministre 1950-01-25 2012-05-15 2014-03-31   \n",
       "1  Ministre des Affaires étrangères 1946-08-20 2012-05-16 2016-02-11   \n",
       "\n",
       "   duree_mendat Parti  \n",
       "0           685    PS  \n",
       "1          1366    PS  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_finale.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "table_finale.to_csv(path_file + 'table_finale.csv', sep=';', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sandbox"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test fonction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35 ministres trouvés\n"
     ]
    }
   ],
   "source": [
    "df_date_holland_ayrault_1 = get_lifespan_ministres('holland_ayrault_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gouvernement</th>\n",
       "      <th>wiki</th>\n",
       "      <th>Nom</th>\n",
       "      <th>birthday</th>\n",
       "      <th>date_debut</th>\n",
       "      <th>date_fin</th>\n",
       "      <th>duree_mendat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>holland_ayrault_1</td>\n",
       "      <td>/wiki/Jean-Marc_Ayrault</td>\n",
       "      <td>Jean-Marc Ayrault</td>\n",
       "      <td>1950-01-25</td>\n",
       "      <td>2012-05-15</td>\n",
       "      <td>2014-03-31</td>\n",
       "      <td>685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>holland_ayrault_1</td>\n",
       "      <td>/wiki/Laurent_Fabius</td>\n",
       "      <td>Laurent Fabius</td>\n",
       "      <td>1946-08-20</td>\n",
       "      <td>2012-05-16</td>\n",
       "      <td>2016-02-11</td>\n",
       "      <td>1366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>holland_ayrault_1</td>\n",
       "      <td>/wiki/Vincent_Peillon</td>\n",
       "      <td>Vincent Peillon</td>\n",
       "      <td>1960-07-07</td>\n",
       "      <td>2012-05-16</td>\n",
       "      <td>2014-03-31</td>\n",
       "      <td>684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>holland_ayrault_1</td>\n",
       "      <td>/wiki/Christiane_Taubira</td>\n",
       "      <td>Christiane Taubira</td>\n",
       "      <td>1952-02-02</td>\n",
       "      <td>2012-05-16</td>\n",
       "      <td>2016-01-27</td>\n",
       "      <td>1351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>holland_ayrault_1</td>\n",
       "      <td>/wiki/Pierre_Moscovici</td>\n",
       "      <td>Pierre Moscovici</td>\n",
       "      <td>1957-09-16</td>\n",
       "      <td>2012-05-16</td>\n",
       "      <td>2014-03-31</td>\n",
       "      <td>684</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        gouvernement                      wiki                 Nom   birthday  \\\n",
       "0  holland_ayrault_1   /wiki/Jean-Marc_Ayrault   Jean-Marc Ayrault 1950-01-25   \n",
       "1  holland_ayrault_1      /wiki/Laurent_Fabius      Laurent Fabius 1946-08-20   \n",
       "2  holland_ayrault_1     /wiki/Vincent_Peillon     Vincent Peillon 1960-07-07   \n",
       "3  holland_ayrault_1  /wiki/Christiane_Taubira  Christiane Taubira 1952-02-02   \n",
       "4  holland_ayrault_1    /wiki/Pierre_Moscovici    Pierre Moscovici 1957-09-16   \n",
       "\n",
       "  date_debut   date_fin  duree_mendat  \n",
       "0 2012-05-15 2014-03-31           685  \n",
       "1 2012-05-16 2016-02-11          1366  \n",
       "2 2012-05-16 2014-03-31           684  \n",
       "3 2012-05-16 2016-01-27          1351  \n",
       "4 2012-05-16 2014-03-31           684  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_date_holland_ayrault_1.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test scrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for ministre in wiki_ministres_clean[:1]:\n",
    "    page_ministre = \"https://fr.wikipedia.org\" + ministre\n",
    "    header = {'User-Agent': 'Mozilla/5.0'} #Needed to prevent 403 error on Wikipedia\n",
    "    page = request.Request(page_ministre, headers=header)\n",
    "    html = request.urlopen(page)\n",
    "    soup = BeautifulSoup(html, \"lxml\")\n",
    "    cells = soup.find('table', {'class': 'infobox_v2'})  # selection du tableau de droite\n",
    "    cells_gouv = cells.find('a', href=re.compile(dict_holland_char['holland_ayrault_1']))  # selection du gouvernement\n",
    "    l_dates = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1950-01-25'"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# -- date_naissance\n",
    "cells.find('time', {'class': 'nowrap date-lien bday'})['datetime']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Jean-Marc Ayrault'"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# -- Nom\n",
    "cells.find('td', {'class': 'entete defaut'}).getText()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}