FM1p0 and FM2p0 Cross Validation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import importlib\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import classification_report,confusion_matrix, roc_auc_score,average_precision_score,f1_score\n",
    "from xgboost import XGBClassifier\n",
    "import shap\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "# from sklearn.metrics import plot_roc_curve\n",
    "from statistics import mean\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import re\n",
    "import pickle\n",
    "from matplotlib import pyplot\n",
    "import time\n",
    "import scispacy\n",
    "import spacy\n",
    "import nltk\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions to generate features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "#nltk.download('punkt')\n",
    "#nltk.download('stopwords')\n",
    "\n",
    "def feature_gen(dataframe,include_bigrams='FALSE'):\n",
    "    \n",
    "    meas_method_column_vals = []\n",
    "    entity_nlp = spacy.load('en_core_sci_sm')\n",
    "    chemical_disease_nlp = spacy.load('en_ner_bc5cdr_md')\n",
    "    chem_ent_ratios, seen_chem, chem_count = [], [], []\n",
    "    human_bigram = []    \n",
    "    for index, row in dataframe.iterrows():\n",
    "        abstract = row['abstract']\n",
    "    #   ------------ Chemical Names ----------------\n",
    "        entity_doc = entity_nlp(abstract)\n",
    "        chemical_disease_doc = chemical_disease_nlp(abstract)\n",
    "        chemical_ents = [ent.text for ent in chemical_disease_doc.ents if ent.label_ == 'CHEMICAL']\n",
    "        if len(entity_doc.ents) == 0:\n",
    "            chem_ent_ratios.append(0)\n",
    "        else:\n",
    "            chem_ent_ratios.append(len(chemical_ents) / len(entity_doc.ents))\n",
    "        seen_chem.append(list(set(chemical_ents)))\n",
    "        chem_count.append(len(chemical_ents))\n",
    "    #   ------------ Bigram Score --------------\n",
    "        if include_bigrams == 'TRUE':\n",
    "            tokens = nltk.word_tokenize(abstract)\n",
    "            bigrams = nltk.bigrams(tokens)\n",
    "            stopset = set(stopwords.words('english') + list(string.punctuation))\n",
    "            milk_bigrams = [(w1, w2) for w1, w2 in bigrams if \n",
    "                            (w1.lower() == 'milk' or w2.lower() == 'milk')\n",
    "                             and (w1.lower() not in stopset and w2.lower() not in stopset)]\n",
    "            human_bigrams = [(w1, w2) for w1, w2 in milk_bigrams if \n",
    "                             (w1.lower() == 'human' or w2.lower() == 'human')\n",
    "                             and (w1.lower() not in stopset and w2.lower() not in stopset)]\n",
    "            human_bigram.append(len(human_bigrams) / len(milk_bigrams) if len(milk_bigrams) != 0 else 0)\n",
    "        else:\n",
    "            human_bigram.append(0.0)\n",
    "\n",
    "    dataframe['chem_ent_ratio'] = chem_ent_ratios\n",
    "    dataframe['chemicals'] = seen_chem\n",
    "    dataframe['bigram_score'] = human_bigram\n",
    "    dataframe['chem_term_count'] = chem_count\n",
    "    \n",
    "    return dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "mpl.rcParams['figure.dpi']= 150\n",
    "\n",
    "def clean_plot(leg=True, grid=None, font=None):\n",
    "    ax = plt.gca()\n",
    "    ax.spines['right'].set_visible(False)\n",
    "    ax.spines['top'].set_visible(False)\n",
    "    ax.yaxis.set_ticks_position('left')\n",
    "    ax.xaxis.set_ticks_position('bottom')\n",
    "    \n",
    "    axis_color = 'lightgrey'\n",
    "    ax.spines['bottom'].set_color(axis_color)\n",
    "    ax.spines['left'].set_color(axis_color)\n",
    "    ax.tick_params(axis='both', color=axis_color)\n",
    "    \n",
    "    if leg:\n",
    "        ax.legend(frameon = False, loc='upper left', bbox_to_anchor=(1, 1))\n",
    "        \n",
    "    if grid is not None:\n",
    "        plt.grid(color='lightgrey', axis = grid, linestyle='-', linewidth=.5)\n",
    "        \n",
    "    if font is not None:\n",
    "        for item in ([ax.title, ax.xaxis.label, ax.yaxis.label] +\n",
    "            ax.get_xticklabels() + ax.get_yticklabels()):\n",
    "            \n",
    "            item.set_fontfamily(font['family'])\n",
    "            item.set_color(font['color'])\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\parth\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from src.filter import Filter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading datasets and generating features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    }
   ],
   "source": [
    "## Garlic and Cocoa\n",
    "\n",
    "gtrain = pd.read_csv(\"data/garlic_scoring.csv\", encoding='latin1')[['PMID', 'abstract', 'paper', 'mesh_terms', 'qual_terms', 'is_useful']]\n",
    "gtrain['food'] = 'garlic'\n",
    "ctrain = pd.read_csv(\"data/cocoa_scoring.csv\", encoding='latin1')[['PMID', 'abstract', 'paper', 'mesh_terms', 'qual_terms', 'is_useful']]\n",
    "ctrain['food'] = 'cocoa'\n",
    "gtrain['is_useful'] = gtrain['is_useful'].replace(2, 1, regex=True)\n",
    "ctrain['is_useful'] = ctrain['is_useful'].replace(2, 1, regex=True)\n",
    "gtrain = gtrain[gtrain['is_useful'].notnull()]\n",
    "ctrain = ctrain[ctrain['is_useful'].notnull()]\n",
    "\n",
    "## Basil\n",
    "\n",
    "btrain = pd.read_excel(\"data/basil_scoring.xls\")[['PMID', 'abstract', 'paper', 'mesh_terms', 'qual_terms', 'is_useful']]\n",
    "btrain['food'] = 'apple'\n",
    "\n",
    "for i in range(len(btrain)):\n",
    "    if btrain['is_useful'].loc[i] == 'x':\n",
    "        btrain['is_useful'].loc[i] = 0\n",
    "        \n",
    "btrain['is_useful'] = btrain['is_useful'].replace(2, 1, regex=True)\n",
    "btrain = btrain[btrain['is_useful'].notnull()]\n",
    "\n",
    "## Apple\n",
    "\n",
    "atrain = pd.read_excel(\"data/apple_scoring.xls\")[['PMID', 'abstract', 'paper', 'mesh_terms', 'qual_terms', 'is_useful']]\n",
    "atrain['food'] = 'apple'\n",
    "\n",
    "for i in range(len(atrain)):\n",
    "    if atrain['is_useful'].loc[i] == 'x':\n",
    "        atrain['is_useful'].loc[i] = 0\n",
    "\n",
    "atrain['is_useful'] = atrain['is_useful'].replace(2, 1, regex=True)\n",
    "atrain = atrain[atrain['is_useful'].notnull()]\n",
    "atrain = atrain[atrain['abstract'].notnull()]\n",
    "atrain = atrain[atrain['PMID'].notnull()]\n",
    "\n",
    "## Human Milk database\n",
    "\n",
    "mtrain_new = pd.read_csv(\"mBase_15Aug_abstract[chemical_gen].csv\")\n",
    "mtrain_new['food'] = 'milk'\n",
    "mtrain_new = mtrain_new[mtrain_new['abstract'].notnull()]\n",
    "mtrain_new = mtrain_new[mtrain_new['PMID'].notnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of the databases for garlic, cocoa, basil, apple, milk:  299 324 93 1653 229\n"
     ]
    }
   ],
   "source": [
    "print('Length of the databases for garlic, cocoa, basil, apple, milk: ', len(gtrain), len(ctrain), len(btrain), len(atrain), len(mtrain_new))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Useful and non-useful:  77 222\n"
     ]
    }
   ],
   "source": [
    "print('Useful and non-useful: ',len(gtrain[gtrain['is_useful'] == 1.0]),len(gtrain[gtrain['is_useful'] == 0.0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Useful and non-useful:  93 231\n"
     ]
    }
   ],
   "source": [
    "print('Useful and non-useful: ',len(ctrain[ctrain['is_useful'] == 1.0]),len(ctrain[ctrain['is_useful'] == 0.0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Useful and non-useful:  462 1191\n"
     ]
    }
   ],
   "source": [
    "print('Useful and non-useful: ',len(atrain[atrain['is_useful'] == 1.0]),len(atrain[atrain['is_useful'] == 0.0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Useful and non-useful:  57 36\n"
     ]
    }
   ],
   "source": [
    "print('Useful and non-useful: ',len(btrain[btrain['is_useful'] == 1.0]),len(btrain[btrain['is_useful'] == 0.0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def build_all_features(ftrain,include_bigrams='FALSE'):\n",
    "#     fmodel_data = Filter()\n",
    "\n",
    "#     fmodel_data.build_features(input_data = ftrain,is_traindata = True)\n",
    "\n",
    "#     ftrain = feature_gen(ftrain,include_bigrams)\n",
    "\n",
    "#     fmodel_data.data['chem_ent_ratio'] = ftrain['chem_ent_ratio'].values\n",
    "#     fmodel_data.data['chem_term_count'] = ftrain['chem_term_count'].values\n",
    "#     fmodel_data.data['bigram_score'] = ftrain['bigram_score'].values\n",
    "    \n",
    "#     return fmodel_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print('----Starting feature generation----')\n",
    "# gdata = build_all_features(gtrain,include_bigrams='FALSE')\n",
    "# print('----DONE----')\n",
    "# cdata = build_all_features(ctrain,include_bigrams='FALSE')\n",
    "# print('----DONE----')\n",
    "# bdata = build_all_features(btrain,include_bigrams='FALSE')\n",
    "# print('----DONE----')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print('----Starting feature generation----')\n",
    "# adata = build_all_features(atrain,include_bigrams='FALSE')\n",
    "# print('----DONE----')\n",
    "# mdata = build_all_features(mtrain_new,include_bigrams='TRUE')\n",
    "# print('----DONE----')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# gdata_features_class = gdata.data.copy()\n",
    "# cdata_features_class = cdata.data.copy()\n",
    "# bdata_features_class = bdata.data.copy()\n",
    "# adata_features_class = adata.data.copy()\n",
    "# mdata_features_class = mdata.data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# gdata_features_class.to_csv('data_with_feature/gdata_features_class.csv')\n",
    "# cdata_features_class.to_csv('data_with_feature/cdata_features_class.csv')\n",
    "# bdata_features_class.to_csv('data_with_feature/bdata_features_class.csv')\n",
    "# adata_features_class.to_csv('data_with_feature/adata_features_class.csv')\n",
    "# mdata_features_class.to_csv('data_with_feature/mdata_features_class.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdata_features_class = pd.read_csv('data_with_feature/gdata_features_class.csv')\n",
    "cdata_features_class = pd.read_csv('data_with_feature/cdata_features_class.csv')\n",
    "bdata_features_class = pd.read_csv('data_with_feature/bdata_features_class.csv')\n",
    "adata_features_class = pd.read_csv('data_with_feature/adata_features_class.csv')\n",
    "mdata_features_class = pd.read_csv('data_with_feature/mdata_features_class.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Doc2vec trained on FoodBase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "fmine = pd.read_csv('FoodBase_Abstracts_embeddings.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_list = []\n",
    "\n",
    "for index, row in fmine.iterrows():\n",
    "    local_list = []\n",
    "    for x in row['vectors'].replace('[','').replace(']','').replace('\\n',' ').replace('  ',' ').replace('   ',' ').split(' '):\n",
    "        if x != '':\n",
    "            local_list.append(float(x))\n",
    "    embedding_list.append(local_list)\n",
    "    \n",
    "fmine['embeddings'] = embedding_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "abstract_embedding_dict = dict()\n",
    "\n",
    "for index, row in fmine.iterrows():\n",
    "    abstract_embedding_dict[row['abstract']] = row['embeddings']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimension of Doc2Vec:  64\n"
     ]
    }
   ],
   "source": [
    "print('Dimension of Doc2Vec: ', len(embedding_list[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fmine_expanded = pd.concat([fmine, fmine['embeddings'].apply(pd.Series)], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "gdata_train_doc2vec = pd.merge(gtrain, fmine_expanded[fmine_expanded['food'] == 'garlic'], on=['abstract'])\n",
    "gdata_features_class_doc2vec = pd.merge(gdata_train_doc2vec, gdata_features_class, on=['PMID'])\n",
    "gdata_features_class_doc2vec = gdata_features_class_doc2vec.drop(columns=['PMID','abstract','paper','mesh_terms','qual_terms','is_useful_x','food_x','is_useful_y','food_y','vectors','embeddings'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "cdata_train_doc2vec = pd.merge(ctrain, fmine_expanded[fmine_expanded['food'] == 'cocoa'], on=['abstract'])\n",
    "cdata_features_class_doc2vec = pd.merge(cdata_train_doc2vec, cdata_features_class, on=['PMID'])\n",
    "cdata_features_class_doc2vec = cdata_features_class_doc2vec.drop(columns=['PMID','abstract','paper','mesh_terms','qual_terms','is_useful_x','food_x','is_useful_y','food_y','vectors','embeddings'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "bdata_train_doc2vec = pd.merge(btrain, fmine_expanded[fmine_expanded['food'] == 'basil'], on=['abstract'])\n",
    "bdata_features_class_doc2vec = pd.merge(bdata_train_doc2vec, bdata_features_class, on=['PMID'])\n",
    "bdata_features_class_doc2vec = bdata_features_class_doc2vec.drop(columns=['PMID','abstract','paper','mesh_terms','qual_terms','is_useful_x','food_x','is_useful_y','food_y','vectors','embeddings'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "adata_train_doc2vec = pd.merge(atrain, fmine_expanded[fmine_expanded['food'] == 'apple'], on=['abstract'])\n",
    "adata_features_class_doc2vec = pd.merge(adata_train_doc2vec, adata_features_class, on=['PMID'])\n",
    "adata_features_class_doc2vec = adata_features_class_doc2vec.drop(columns=['PMID','abstract','paper','mesh_terms','qual_terms','is_useful_x','food_x','is_useful_y','food_y','vectors','embeddings'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "mdata_train_doc2vec = pd.merge(mtrain_new, fmine_expanded[fmine_expanded['food'] == 'human milk'], on=['abstract'])\n",
    "mdata_features_class_doc2vec = pd.merge(mdata_train_doc2vec, mdata_features_class, on=['PMID'])\n",
    "mdata_features_class_doc2vec = mdata_features_class_doc2vec.drop(columns=['PMID','abstract','paper','mesh_terms','qual_terms','is_useful_x','food_x','is_useful_y','food_y','vectors','embeddings'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "mdata_features_class_doc2vec = mdata_features_class_doc2vec.drop(columns=['journal','mesh_UIds','qual_UIds','webpage','year','source','measmethod','chem_ent_ratio_x','chemicals','bigram_score_x','chem_term_count_x'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "mdata_features_class_doc2vec = mdata_features_class_doc2vec.rename(columns={'chem_ent_ratio_y':'chem_ent_ratio','chem_term_count_y':'chem_term_count','bigram_score_y':'bigram_score'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalize the features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_features(total_data):\n",
    "    total_data_features = total_data.drop(columns=['class'])\n",
    "    for col_feature in list(total_data_features.columns): \n",
    "        col_list = total_data_features[col_feature].tolist()\n",
    "        if col_list != [0.0] * len(col_list):\n",
    "            col_list_normalized = [(x-np.mean(col_list))/np.std(col_list) for x in col_list]\n",
    "        else:\n",
    "            col_list_normalized = col_list\n",
    "        total_data_features[col_feature] = col_list_normalized\n",
    "    # total_data_features_normalized = (total_data_features-total_data_features.mean())/total_data_features.std()\n",
    "    # total_data_features_normalized = (total_data_features-total_data_features.min())/(total_data_features.max()-total_data_features.min())\n",
    "    total_data_features['class'] = total_data['class'].tolist()\n",
    "    \n",
    "    return total_data_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "gdata_features_class_normalized = normalize_features(gdata_features_class)\n",
    "cdata_features_class_normalized = normalize_features(cdata_features_class)\n",
    "bdata_features_class_normalized = normalize_features(bdata_features_class)\n",
    "adata_features_class_normalized = normalize_features(adata_features_class)\n",
    "mdata_features_class_normalized = normalize_features(mdata_features_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "fm1p0_columns = ['chromatography', 'food_term_count', 'gen_term_count', 'sci_term_count', 'spectrometry', 'spectrophotometry']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performances on seen food "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def xgboost_model(x,y,kfold,n_splits=10,fm1p0_columns=fm1p0_columns):\n",
    "\n",
    "    cross_val_model_fm1 = RandomForestClassifier(max_depth=80,random_state=0)\n",
    "    cross_val_model_fm2 = RandomForestClassifier(max_depth=80,random_state=0)\n",
    "    \n",
    "    kfold = StratifiedKFold(n_splits)\n",
    "    \n",
    "    auc_fm1 = []\n",
    "    aup_fm1 = []\n",
    "    f1_score_list_fm1 = []\n",
    "    \n",
    "    auc_fm2 = []\n",
    "    aup_fm2 = []\n",
    "    f1_score_list_fm2 = []\n",
    "    \n",
    "    for i,(train,test) in enumerate(kfold.split(x,y)):\n",
    "        \n",
    "        x_old = x[fm1p0_columns]\n",
    "        \n",
    "        cross_val_model_fm2.fit(x.loc[train],y.loc[train])\n",
    "        cross_val_model_fm1.fit(x_old.loc[train],y.loc[train])\n",
    "        \n",
    "        #viz_fm1 = plot_roc_curve(cross_val_model_fm1, x_old.loc[test], y.loc[test],\n",
    "        #                 name='ROC fold {}'.format(i),\n",
    "        #                 alpha=0.3, lw=1)\n",
    "        #viz_fm2 = plot_roc_curve(cross_val_model_fm2, x.loc[test], y.loc[test],\n",
    "        #                 name='ROC fold {}'.format(i),\n",
    "        #                 alpha=0.3, lw=1)\n",
    "        \n",
    "        y_predicted_fm1 = cross_val_model_fm1.predict(x_old.loc[test])\n",
    "        y_predicted_fm2 = cross_val_model_fm2.predict(x.loc[test])\n",
    "        \n",
    "        auc_fm1.append(roc_auc_score(np.array(y.loc[test].tolist()),y_predicted_fm1))\n",
    "        aup_fm1.append(average_precision_score(np.array(y.loc[test].tolist()),y_predicted_fm1))\n",
    "        f1_score_list_fm1.append(f1_score(np.array(y.loc[test].tolist()),y_predicted_fm1))\n",
    "        \n",
    "        auc_fm2.append(roc_auc_score(np.array(y.loc[test].tolist()),y_predicted_fm2))\n",
    "        aup_fm2.append(average_precision_score(np.array(y.loc[test].tolist()),y_predicted_fm2))\n",
    "        f1_score_list_fm2.append(f1_score(np.array(y.loc[test].tolist()),y_predicted_fm2))\n",
    "        # plot_confusion_matrix(cross_val_model,x.loc[test],y.loc[test])\n",
    "        \n",
    "    print('------------------------------------------------------------------------------------------------------')\n",
    "    print('FoodMine1.0')\n",
    "    print('Average and SD of AUROC: ', mean(auc_fm1), np.std(auc_fm1))\n",
    "    print('Average and SD of AUPRC:', mean(aup_fm1), np.std(aup_fm1))\n",
    "    print('Average and SD of f1-Score:', mean(f1_score_list_fm1), np.std(f1_score_list_fm1))\n",
    "    print('All AUROC values: ', auc_fm1)\n",
    "    print('All AUPRC values: ', aup_fm1)\n",
    "    print('All f1-Score values: ', f1_score_list_fm1)\n",
    "    print('------------------------------------------------------------------------------------------------------')\n",
    "    print('FoodMine2.0')\n",
    "    print('Average and SD of AUROC: ', mean(auc_fm2), np.std(auc_fm2))\n",
    "    print('Average and SD of AUPRC:', mean(aup_fm2), np.std(aup_fm2))\n",
    "    print('Average and SD of f1-Score:', mean(f1_score_list_fm2), np.std(f1_score_list_fm2))\n",
    "    print('All AUROC values: ', auc_fm2)\n",
    "    print('All AUPRC values: ', aup_fm2)\n",
    "    print('All f1-Score values: ', f1_score_list_fm2)\n",
    "    print('------------------------------------------------------------------------------------------------------')\n",
    "\n",
    "    return mean(auc_fm2), mean(aup_fm2), mean(f1_score_list_fm2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def get_cross_validation_seen_food(fdata_features_class_normalized):\n",
    "\n",
    "    repeat = 1\n",
    "\n",
    "    for _ in tqdm(range(repeat)):\n",
    "        oversample = SMOTE()\n",
    "        y = fdata_features_class_normalized.copy()['class']\n",
    "        X = fdata_features_class_normalized.copy().drop('class', axis = 1)\n",
    "        X.columns = X.columns.astype(str)\n",
    "        X_smote, y_smote = oversample.fit_resample(X, y)\n",
    "        kfold = StratifiedKFold(n_splits=10)\n",
    "        auc_fm2, aup_fm2, f1_fm2 = xgboost_model(X_smote, y_smote,kfold,n_splits=10)\n",
    "        \n",
    "    return "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>gen_term_count</th>\n",
       "      <th>food_term_count</th>\n",
       "      <th>sci_term_count</th>\n",
       "      <th>class</th>\n",
       "      <th>spectrometry</th>\n",
       "      <th>chromatography</th>\n",
       "      <th>spectrophotometry</th>\n",
       "      <th>chem_ent_ratio</th>\n",
       "      <th>chem_term_count</th>\n",
       "      <th>bigram_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.318923</td>\n",
       "      <td>1.615156</td>\n",
       "      <td>0.801215</td>\n",
       "      <td>-0.258407</td>\n",
       "      <td>3.384802</td>\n",
       "      <td>-3.805800</td>\n",
       "      <td>1.640497</td>\n",
       "      <td>-2.008467</td>\n",
       "      <td>4.323183</td>\n",
       "      <td>1.473609</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.466219</td>\n",
       "      <td>1.549901</td>\n",
       "      <td>0.386801</td>\n",
       "      <td>0.742964</td>\n",
       "      <td>-0.081566</td>\n",
       "      <td>-1.721926</td>\n",
       "      <td>-0.858324</td>\n",
       "      <td>-3.293416</td>\n",
       "      <td>-0.251621</td>\n",
       "      <td>-0.822619</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.077940</td>\n",
       "      <td>-0.733739</td>\n",
       "      <td>-0.106453</td>\n",
       "      <td>-3.235562</td>\n",
       "      <td>-0.142488</td>\n",
       "      <td>-0.406461</td>\n",
       "      <td>0.403134</td>\n",
       "      <td>-2.334560</td>\n",
       "      <td>0.607535</td>\n",
       "      <td>1.598378</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.014768</td>\n",
       "      <td>0.478264</td>\n",
       "      <td>-0.868394</td>\n",
       "      <td>-0.492501</td>\n",
       "      <td>1.277909</td>\n",
       "      <td>-0.316566</td>\n",
       "      <td>0.148894</td>\n",
       "      <td>-2.738673</td>\n",
       "      <td>0.289150</td>\n",
       "      <td>0.453324</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.138469</td>\n",
       "      <td>1.089024</td>\n",
       "      <td>-0.205716</td>\n",
       "      <td>-1.830036</td>\n",
       "      <td>-0.178908</td>\n",
       "      <td>-0.019330</td>\n",
       "      <td>0.420647</td>\n",
       "      <td>-3.019905</td>\n",
       "      <td>0.205741</td>\n",
       "      <td>1.183853</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294</th>\n",
       "      <td>1.536668</td>\n",
       "      <td>2.807355</td>\n",
       "      <td>0.013995</td>\n",
       "      <td>-2.084901</td>\n",
       "      <td>2.366916</td>\n",
       "      <td>-3.757211</td>\n",
       "      <td>0.391783</td>\n",
       "      <td>-2.721904</td>\n",
       "      <td>3.457376</td>\n",
       "      <td>5.559508</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>1.476389</td>\n",
       "      <td>-2.733936</td>\n",
       "      <td>-0.561345</td>\n",
       "      <td>-2.081398</td>\n",
       "      <td>-0.446338</td>\n",
       "      <td>-2.195139</td>\n",
       "      <td>0.051300</td>\n",
       "      <td>-1.428699</td>\n",
       "      <td>-1.087287</td>\n",
       "      <td>1.599730</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296</th>\n",
       "      <td>0.194007</td>\n",
       "      <td>0.971212</td>\n",
       "      <td>-1.602840</td>\n",
       "      <td>-1.000298</td>\n",
       "      <td>2.354669</td>\n",
       "      <td>-1.833926</td>\n",
       "      <td>-0.000545</td>\n",
       "      <td>-0.132896</td>\n",
       "      <td>1.422767</td>\n",
       "      <td>1.188912</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297</th>\n",
       "      <td>0.324103</td>\n",
       "      <td>1.300528</td>\n",
       "      <td>-1.449213</td>\n",
       "      <td>-2.510239</td>\n",
       "      <td>1.168539</td>\n",
       "      <td>-3.529079</td>\n",
       "      <td>-1.929223</td>\n",
       "      <td>0.195159</td>\n",
       "      <td>-0.548822</td>\n",
       "      <td>4.082344</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>0.988856</td>\n",
       "      <td>-0.017443</td>\n",
       "      <td>-4.442276</td>\n",
       "      <td>-3.296131</td>\n",
       "      <td>2.492467</td>\n",
       "      <td>-3.962574</td>\n",
       "      <td>0.679276</td>\n",
       "      <td>-1.529342</td>\n",
       "      <td>-0.400329</td>\n",
       "      <td>0.335123</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>299 rows × 74 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0         1         2         3         4         5         6  \\\n",
       "0    2.318923  1.615156  0.801215 -0.258407  3.384802 -3.805800  1.640497   \n",
       "1    0.466219  1.549901  0.386801  0.742964 -0.081566 -1.721926 -0.858324   \n",
       "2   -0.077940 -0.733739 -0.106453 -3.235562 -0.142488 -0.406461  0.403134   \n",
       "3    2.014768  0.478264 -0.868394 -0.492501  1.277909 -0.316566  0.148894   \n",
       "4   -1.138469  1.089024 -0.205716 -1.830036 -0.178908 -0.019330  0.420647   \n",
       "..        ...       ...       ...       ...       ...       ...       ...   \n",
       "294  1.536668  2.807355  0.013995 -2.084901  2.366916 -3.757211  0.391783   \n",
       "295  1.476389 -2.733936 -0.561345 -2.081398 -0.446338 -2.195139  0.051300   \n",
       "296  0.194007  0.971212 -1.602840 -1.000298  2.354669 -1.833926 -0.000545   \n",
       "297  0.324103  1.300528 -1.449213 -2.510239  1.168539 -3.529079 -1.929223   \n",
       "298  0.988856 -0.017443 -4.442276 -3.296131  2.492467 -3.962574  0.679276   \n",
       "\n",
       "            7         8         9  ...  gen_term_count  food_term_count  \\\n",
       "0   -2.008467  4.323183  1.473609  ...             0.0              0.0   \n",
       "1   -3.293416 -0.251621 -0.822619  ...             2.0              3.0   \n",
       "2   -2.334560  0.607535  1.598378  ...             0.0              2.0   \n",
       "3   -2.738673  0.289150  0.453324  ...             0.0              2.0   \n",
       "4   -3.019905  0.205741  1.183853  ...             0.0              7.0   \n",
       "..        ...       ...       ...  ...             ...              ...   \n",
       "294 -2.721904  3.457376  5.559508  ...             0.0              1.0   \n",
       "295 -1.428699 -1.087287  1.599730  ...             0.0              0.0   \n",
       "296 -0.132896  1.422767  1.188912  ...             0.0              5.0   \n",
       "297  0.195159 -0.548822  4.082344  ...             0.0              4.0   \n",
       "298 -1.529342 -0.400329  0.335123  ...             0.0              1.0   \n",
       "\n",
       "     sci_term_count  class  spectrometry  chromatography  spectrophotometry  \\\n",
       "0               2.0    1.0           1.0             1.0                0.0   \n",
       "1               3.0    0.0           0.0             1.0                0.0   \n",
       "2               0.0    0.0           0.0             1.0                0.0   \n",
       "3               0.0    0.0           1.0             1.0                0.0   \n",
       "4               0.0    1.0           1.0             1.0                0.0   \n",
       "..              ...    ...           ...             ...                ...   \n",
       "294             2.0    0.0           0.0             1.0                0.0   \n",
       "295             0.0    0.0           0.0             1.0                0.0   \n",
       "296             0.0    0.0           0.0             0.0                0.0   \n",
       "297             1.0    1.0           0.0             1.0                0.0   \n",
       "298             0.0    0.0           0.0             0.0                0.0   \n",
       "\n",
       "     chem_ent_ratio  chem_term_count  bigram_score  \n",
       "0               0.0                0           0.0  \n",
       "1               0.0                0           0.0  \n",
       "2               0.0                0           0.0  \n",
       "3               0.0                0           0.0  \n",
       "4               0.0                0           0.0  \n",
       "..              ...              ...           ...  \n",
       "294             0.0                0           0.0  \n",
       "295             0.0                0           0.0  \n",
       "296             0.0                0           0.0  \n",
       "297             0.0                0           0.0  \n",
       "298             0.0                0           0.0  \n",
       "\n",
       "[299 rows x 74 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdata_features_class_doc2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:08<00:00,  8.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine1.0\n",
      "Average and SD of AUROC:  0.7761857707509882 0.12932230851301624\n",
      "Average and SD of AUPRC: 0.7372888826149695 0.11538431913800254\n",
      "Average and SD of f1-Score: 0.7258965694877088 0.19552879730470754\n",
      "All AUROC values:  [0.6739130434782609, 0.5375494071146245, 0.6146245059288538, 0.6857707509881422, 0.8181818181818181, 0.8636363636363636, 0.8863636363636365, 0.9318181818181819, 0.8863636363636365, 0.8636363636363635]\n",
      "All AUPRC values:  [0.681159420289855, 0.5320910973084886, 0.5893217893217892, 0.6313131313131313, 0.7704545454545454, 0.7857142857142857, 0.8359683794466404, 0.8942687747035573, 0.8495670995670996, 0.803030303030303]\n",
      "All f1-Score values:  [0.5161290322580645, 0.4324324324324324, 0.41379310344827586, 0.631578947368421, 0.8095238095238095, 0.88, 0.888888888888889, 0.9333333333333332, 0.8837209302325582, 0.8695652173913043]\n",
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine2.0\n",
      "Average and SD of AUROC:  0.8788537549407114 0.0904300836281626\n",
      "Average and SD of AUPRC: 0.8386038647342995 0.08987980422234669\n",
      "Average and SD of f1-Score: 0.8725986620378119 0.10207301249034782\n",
      "All AUROC values:  [0.8675889328063241, 0.6679841897233202, 0.7737154150197628, 0.865612648221344, 0.8636363636363635, 0.9545454545454545, 0.9318181818181819, 0.9545454545454545, 0.9545454545454545, 0.9545454545454545]\n",
      "All AUPRC values:  [0.8363008971704624, 0.6260869565217391, 0.7487012987012988, 0.8252525252525253, 0.803030303030303, 0.9166666666666666, 0.88, 0.9166666666666666, 0.9166666666666666, 0.9166666666666666]\n",
      "All f1-Score values:  [0.8636363636363636, 0.6511627906976744, 0.7222222222222223, 0.8571428571428572, 0.8695652173913043, 0.9565217391304348, 0.9361702127659575, 0.9565217391304348, 0.9565217391304348, 0.9565217391304348]\n",
      "------------------------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "## Garlic\n",
    "get_cross_validation_seen_food(gdata_features_class_doc2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:23<00:00, 23.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine1.0\n",
      "Average and SD of AUROC:  0.7083333333333334 0.18767976050063392\n",
      "Average and SD of AUPRC: 0.6816481419529523 0.15307435495359886\n",
      "Average and SD of f1-Score: 0.6222605158075792 0.29667529598067244\n",
      "All AUROC values:  [0.5235507246376812, 0.6032608695652174, 0.4565217391304347, 0.3913043478260869, 0.8913043478260869, 0.782608695652174, 0.9347826086956522, 0.891304347826087, 0.7826086956521738, 0.8260869565217391]\n",
      "All AUPRC values:  [0.5037927844588344, 0.5972644376899696, 0.4826086956521739, 0.4665551839464883, 0.8423913043478262, 0.7147826086956521, 0.9150197628458498, 0.8311036789297659, 0.7093397745571659, 0.7536231884057971]\n",
      "All f1-Score values:  [0.21428571428571427, 0.3870967741935483, 0.24242424242424243, 0.2222222222222222, 0.8936170212765957, 0.7916666666666667, 0.9333333333333332, 0.8979591836734695, 0.7999999999999999, 0.84]\n",
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine2.0\n",
      "Average and SD of AUROC:  0.8597826086956522 0.10198873135426001\n",
      "Average and SD of AUPRC: 0.8230979056147294 0.11100276069162467\n",
      "Average and SD of f1-Score: 0.8429803879638634 0.1326742694814446\n",
      "All AUROC values:  [0.7663043478260869, 0.8532608695652174, 0.6739130434782609, 0.7173913043478262, 0.891304347826087, 0.8913043478260869, 0.9782608695652174, 0.891304347826087, 0.9565217391304348, 0.9782608695652174]\n",
      "All AUPRC values:  [0.6933395004625347, 0.8381858902575587, 0.644927536231884, 0.686335403726708, 0.8311036789297659, 0.8423913043478262, 0.9782608695652174, 0.8214285714285714, 0.9366729678638942, 0.9583333333333334]\n",
      "All f1-Score values:  [0.7659574468085107, 0.8372093023255814, 0.5714285714285714, 0.6486486486486486, 0.8979591836734695, 0.8936170212765957, 0.9777777777777777, 0.9019607843137255, 0.9565217391304348, 0.9787234042553191]\n",
      "------------------------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "## Cocoa\n",
    "get_cross_validation_seen_food(cdata_features_class_doc2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:16<00:00, 16.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine1.0\n",
      "Average and SD of AUROC:  0.6 0.16799470891138873\n",
      "Average and SD of AUPRC: 0.6002561327561328 0.1537092327437355\n",
      "Average and SD of f1-Score: 0.5652225552225552 0.1814353492703115\n",
      "All AUROC values:  [0.41666666666666663, 0.8333333333333334, 0.41666666666666663, 0.41666666666666663, 0.5499999999999999, 0.6500000000000001, 0.75, 0.9, 0.5166666666666667, 0.55]\n",
      "All AUPRC values:  [0.4666666666666667, 0.7777777777777779, 0.4642857142857143, 0.4642857142857143, 0.5727272727272728, 0.6477272727272727, 0.7727272727272727, 0.8909090909090909, 0.4636363636363636, 0.4818181818181818]\n",
      "All f1-Score values:  [0.3636363636363636, 0.8333333333333334, 0.4615384615384615, 0.4615384615384615, 0.5454545454545454, 0.6, 0.6666666666666666, 0.888888888888889, 0.28571428571428575, 0.5454545454545454]\n",
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine2.0\n",
      "Average and SD of AUROC:  0.7666666666666667 0.1288840995278747\n",
      "Average and SD of AUPRC: 0.7401509139009139 0.1414315293829695\n",
      "Average and SD of f1-Score: 0.7801770451770451 0.11894852658705712\n",
      "All AUROC values:  [0.6666666666666667, 1.0, 0.6666666666666666, 0.5833333333333335, 0.7166666666666668, 0.6333333333333333, 0.8, 0.9, 0.9, 0.8]\n",
      "All AUPRC values:  [0.6041666666666667, 1.0, 0.625, 0.5462962962962963, 0.6861471861471862, 0.6262626262626263, 0.75, 0.8909090909090909, 0.8909090909090909, 0.7818181818181817]\n",
      "All f1-Score values:  [0.7142857142857143, 1.0, 0.6, 0.6666666666666667, 0.7692307692307692, 0.6666666666666666, 0.8571428571428571, 0.888888888888889, 0.888888888888889, 0.7499999999999999]\n",
      "------------------------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "## Basil\n",
    "get_cross_validation_seen_food(bdata_features_class_doc2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:58<00:00, 58.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine1.0\n",
      "Average and SD of AUROC:  0.5882142857142857 0.0657991931814764\n",
      "Average and SD of AUPRC: 0.5536636730710411 0.043121495309338946\n",
      "Average and SD of f1-Score: 0.607794907892713 0.0918336203025077\n",
      "All AUROC values:  [0.5438375350140056, 0.5273809523809524, 0.49579831932773116, 0.5252100840336135, 0.5966386554621849, 0.6386554621848739, 0.7016806722689075, 0.6806722689075629, 0.6092436974789917, 0.5630252100840336]\n",
      "All AUPRC values:  [0.5218491712915071, 0.5166007702839301, 0.4979187936856986, 0.5132803121248499, 0.5563146121758056, 0.5851057664445088, 0.632684652808492, 0.6150778782850719, 0.5632814101250256, 0.5345233634855215]\n",
      "All f1-Score values:  [0.5321888412017167, 0.5065502183406114, 0.4690265486725664, 0.5108225108225107, 0.6279069767441859, 0.6742424242424242, 0.7380073800738007, 0.7246376811594202, 0.6713780918727914, 0.6231884057971014]\n",
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine2.0\n",
      "Average and SD of AUROC:  0.867874649859944 0.08557089907020693\n",
      "Average and SD of AUPRC: 0.829113506614322 0.09080418298551551\n",
      "Average and SD of f1-Score: 0.8583729727910974 0.09867098721766222\n",
      "All AUROC values:  [0.7403011204481793, 0.7535714285714286, 0.7899159663865547, 0.7731092436974789, 0.9369747899159664, 0.9411764705882353, 0.9369747899159663, 0.9285714285714286, 0.9327731092436975, 0.9453781512605042]\n",
      "All AUPRC values:  [0.6853305241339336, 0.7094164280995375, 0.7525074545947411, 0.734093637454982, 0.8988261971455249, 0.918552036199095, 0.8932773109243697, 0.8850446428571428, 0.8946890756302521, 0.9193977591036414]\n",
      "All f1-Score values:  [0.7207207207207208, 0.7255813953488371, 0.7641509433962265, 0.7428571428571429, 0.9387755102040817, 0.9406779661016949, 0.9397590361445783, 0.9311740890688258, 0.9344262295081968, 0.9456066945606694]\n",
      "------------------------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "## Apple\n",
    "get_cross_validation_seen_food(adata_features_class_doc2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:16<00:00, 16.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine1.0\n",
      "Average and SD of AUROC:  0.5698717948717948 0.05589802399126064\n",
      "Average and SD of AUPRC: 0.540088597308277 0.032711519186533214\n",
      "Average and SD of f1-Score: 0.6732326415414651 0.03469393091285233\n",
      "All AUROC values:  [0.4615384615384615, 0.6153846153846154, 0.6538461538461539, 0.5384615384615384, 0.6153846153846154, 0.5, 0.5448717948717948, 0.5865384615384616, 0.608974358974359, 0.5737179487179487]\n",
      "All AUPRC values:  [0.4816053511705686, 0.5668016194331984, 0.5923076923076923, 0.5201465201465202, 0.5659340659340659, 0.5, 0.5434965034965034, 0.5674725274725274, 0.542962962962963, 0.5201587301587302]\n",
      "All f1-Score values:  [0.6111111111111112, 0.6875, 0.7272727272727274, 0.6470588235294118, 0.7058823529411765, 0.6285714285714286, 0.6857142857142856, 0.7058823529411765, 0.6666666666666667, 0.6666666666666667]\n",
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine2.0\n",
      "Average and SD of AUROC:  0.7945512820512821 0.04844835155003569\n",
      "Average and SD of AUPRC: 0.7404124261861819 0.052991973150938426\n",
      "Average and SD of f1-Score: 0.793109853109853 0.06197688015892626\n",
      "All AUROC values:  [0.8461538461538463, 0.8461538461538461, 0.8076923076923077, 0.8076923076923076, 0.6923076923076923, 0.7692307692307693, 0.8333333333333334, 0.8365384615384616, 0.7564102564102564, 0.75]\n",
      "All AUPRC values:  [0.8146853146853146, 0.7928994082840237, 0.7417582417582418, 0.7564102564102564, 0.6244343891402715, 0.6974358974358974, 0.7647058823529411, 0.7784615384615384, 0.6933333333333334, 0.74]\n",
      "All f1-Score values:  [0.8333333333333333, 0.8461538461538461, 0.8148148148148148, 0.8, 0.7333333333333334, 0.7857142857142856, 0.8666666666666666, 0.8571428571428571, 0.7272727272727272, 0.6666666666666666]\n",
      "------------------------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "## Milk\n",
    "get_cross_validation_seen_food(mdata_features_class_doc2vec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance on unseen food"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def xgboost_model_unseen_food(x_other,y_other,x_food,y_food,kfold,n_splits=10,fm1p0_columns=fm1p0_columns):\n",
    "\n",
    "    cross_val_model_fm1 = RandomForestClassifier(max_depth=80,random_state=0)\n",
    "    cross_val_model_fm2 = RandomForestClassifier(max_depth=80,random_state=0)\n",
    "    \n",
    "    kfold = StratifiedKFold(n_splits)\n",
    "    \n",
    "    auc_fm1 = []\n",
    "    aup_fm1 = []\n",
    "    f1_score_list_fm1 = []\n",
    "    \n",
    "    auc_fm2 = []\n",
    "    aup_fm2 = []\n",
    "    f1_score_list_fm2 = []\n",
    "    \n",
    "    for i,(test_food,train_food) in enumerate(kfold.split(x_food,y_food)): # test set is kept larger here \n",
    "        \n",
    "        x_other_old = x_other[fm1p0_columns]\n",
    "        x_food_old = x_food[fm1p0_columns]\n",
    "        \n",
    "        cross_val_model_fm2.fit(pd.concat([x_other,x_food.loc[test_food]]),pd.concat([y_other,y_food.loc[test_food]]))\n",
    "        cross_val_model_fm1.fit(pd.concat([x_other_old,x_food_old.loc[test_food]]),pd.concat([y_other,y_food.loc[test_food]]))\n",
    "        \n",
    "        #viz_fm1 = plot_roc_curve(cross_val_model_fm1, x_old.loc[test], y.loc[test],\n",
    "        #                 name='ROC fold {}'.format(i),\n",
    "        #                 alpha=0.3, lw=1)\n",
    "        #viz_fm2 = plot_roc_curve(cross_val_model_fm2, x.loc[test], y.loc[test],\n",
    "        #                 name='ROC fold {}'.format(i),\n",
    "        #                 alpha=0.3, lw=1)\n",
    "        \n",
    "        y_predicted_fm1 = cross_val_model_fm1.predict(x_food_old.loc[train_food])\n",
    "        y_predicted_fm2 = cross_val_model_fm2.predict(x_food.loc[train_food])\n",
    "        \n",
    "        auc_fm1.append(roc_auc_score(np.array(y_food.loc[train_food].tolist()),y_predicted_fm1))\n",
    "        aup_fm1.append(average_precision_score(np.array(y_food.loc[train_food].tolist()),y_predicted_fm1))\n",
    "        f1_score_list_fm1.append(f1_score(np.array(y_food.loc[train_food].tolist()),y_predicted_fm1))\n",
    "        \n",
    "        auc_fm2.append(roc_auc_score(np.array(y_food.loc[train_food].tolist()),y_predicted_fm2))\n",
    "        aup_fm2.append(average_precision_score(np.array(y_food.loc[train_food].tolist()),y_predicted_fm2))\n",
    "        f1_score_list_fm2.append(f1_score(np.array(y_food.loc[train_food].tolist()),y_predicted_fm2))\n",
    "        # plot_confusion_matrix(cross_val_model,x.loc[test],y.loc[test])\n",
    "        \n",
    "    print('------------------------------------------------------------------------------------------------------')\n",
    "    print('FoodMine1.0')\n",
    "    print('Average and SD of AUROC: ', mean(auc_fm1), np.std(auc_fm1))\n",
    "    print('Average and SD of AUPRC:', mean(aup_fm1), np.std(aup_fm1))\n",
    "    print('Average and SD of f1-Score:', mean(f1_score_list_fm1), np.std(f1_score_list_fm1))\n",
    "    print('All AUROC values: ', auc_fm1)\n",
    "    print('All AUPRC values: ', aup_fm1)\n",
    "    print('All f1-Score values: ', f1_score_list_fm1)\n",
    "    print('------------------------------------------------------------------------------------------------------')\n",
    "    print('FoodMine2.0')\n",
    "    print('Average and SD of AUROC: ', mean(auc_fm2), np.std(auc_fm2))\n",
    "    print('Average and SD of AUPRC:', mean(aup_fm2), np.std(aup_fm2))\n",
    "    print('Average and SD of f1-Score:', mean(f1_score_list_fm2), np.std(f1_score_list_fm2))\n",
    "    print('All AUROC values: ', auc_fm2)\n",
    "    print('All AUPRC values: ', aup_fm2)\n",
    "    print('All f1-Score values: ', f1_score_list_fm2)\n",
    "    print('------------------------------------------------------------------------------------------------------')\n",
    "\n",
    "    return mean(auc_fm2), mean(aup_fm2), mean(f1_score_list_fm2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cross_validation_unseen_food(other_data_features_class,fdata_features_class):\n",
    "\n",
    "    repeat = 1\n",
    "\n",
    "    for _ in tqdm(range(repeat)):\n",
    "        oversample = SMOTE()\n",
    "        y_other = other_data_features_class.copy()['class']\n",
    "        X_other = other_data_features_class.copy().drop('class', axis = 1)\n",
    "        X_other.columns = X_other.columns.astype(str)\n",
    "        X_other_smote, y_other_smote = oversample.fit_resample(X_other, y_other)\n",
    "        y_food = fdata_features_class.copy()['class']\n",
    "        X_food = fdata_features_class.copy().drop('class', axis = 1)\n",
    "        X_food.columns = X_food.columns.astype(str)\n",
    "        X_food_smote, y_food_smote = oversample.fit_resample(X_food, y_food)\n",
    "        kfold = StratifiedKFold(n_splits=10)\n",
    "        auc_fm2, aup_fm2, f1_fm2 = xgboost_model_unseen_food(X_other_smote, y_other_smote,X_food_smote, y_food_smote,kfold,n_splits=10)\n",
    "        \n",
    "    return "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [01:35<00:00, 95.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine1.0\n",
      "Average and SD of AUROC:  0.7518774703557313 0.16207307938147736\n",
      "Average and SD of AUPRC: 0.7137391618043792 0.1366497860151581\n",
      "Average and SD of f1-Score: 0.6768486906027685 0.2789390481771794\n",
      "All AUROC values:  [0.540513833992095, 0.540513833992095, 0.458498023715415, 0.7747035573122529, 0.8863636363636364, 0.7954545454545455, 0.8863636363636365, 0.8636363636363636, 0.8863636363636365, 0.8863636363636365]\n",
      "All AUPRC values:  [0.5358695652173913, 0.5358695652173913, 0.4747474747474747, 0.7345959595959596, 0.8245454545454546, 0.7245454545454546, 0.8495670995670996, 0.7857142857142857, 0.8359683794466404, 0.8359683794466404]\n",
      "All f1-Score values:  [0.3225806451612903, 0.3225806451612903, 0.14285714285714288, 0.7368421052631579, 0.8936170212765958, 0.8085106382978724, 0.8837209302325582, 0.88, 0.888888888888889, 0.888888888888889]\n",
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine2.0\n",
      "Average and SD of AUROC:  0.8769762845849802 0.13666207961404975\n",
      "Average and SD of AUPRC: 0.8494852503204906 0.14514411864740936\n",
      "Average and SD of f1-Score: 0.8774236265089924 0.1337926345545219\n",
      "All AUROC values:  [0.6452569169960475, 0.5988142292490118, 0.8428853754940712, 0.8873517786561266, 0.9772727272727273, 0.9772727272727273, 0.9090909090909091, 1.0, 0.9318181818181819, 1.0]\n",
      "All AUPRC values:  [0.6057971014492753, 0.5690821256038647, 0.8024986709197236, 0.8640085061137693, 0.9565217391304348, 0.9565217391304348, 0.8461538461538461, 1.0, 0.8942687747035573, 1.0]\n",
      "All f1-Score values:  [0.6363636363636365, 0.6250000000000001, 0.8292682926829269, 0.8780487804878049, 0.9777777777777777, 0.9777777777777777, 0.9166666666666666, 1.0, 0.9333333333333332, 1.0]\n",
      "------------------------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "## Garlic\n",
    "other_foods = pd.concat([cdata_features_class_doc2vec,bdata_features_class_doc2vec,adata_features_class_doc2vec,mdata_features_class_doc2vec])\n",
    "get_cross_validation_unseen_food(other_foods,gdata_features_class_doc2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 1/1 [01:42<00:00, 102.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine1.0\n",
      "Average and SD of AUROC:  0.7390398550724638 0.20570109725520008\n",
      "Average and SD of AUPRC: 0.7133440569263881 0.17814557620895305\n",
      "Average and SD of f1-Score: 0.6168039458009063 0.36519191887102104\n",
      "All AUROC values:  [0.5226449275362319, 0.4547101449275362, 0.4782608695652174, 0.5, 0.9130434782608696, 0.8913043478260869, 0.9347826086956522, 0.891304347826087, 0.8695652173913044, 0.9347826086956522]\n",
      "All AUPRC values:  [0.5047795251310515, 0.49586288416075647, 0.4927536231884058, 0.5, 0.8518518518518519, 0.8423913043478262, 0.8985507246376812, 0.8311036789297659, 0.8011272141706924, 0.9150197628458498]\n",
      "All f1-Score values:  [0.15384615384615383, 0.13333333333333333, 0.07692307692307691, 0.3428571428571428, 0.92, 0.8936170212765957, 0.9361702127659574, 0.8979591836734695, 0.8800000000000001, 0.9333333333333332]\n",
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine2.0\n",
      "Average and SD of AUROC:  0.8338768115942029 0.1256469552850396\n",
      "Average and SD of AUPRC: 0.7885256710886528 0.13393070246907285\n",
      "Average and SD of f1-Score: 0.8261715303459842 0.14272140936003294\n",
      "All AUROC values:  [0.5942028985507248, 0.8097826086956521, 0.7608695652173914, 0.6521739130434783, 0.891304347826087, 0.9130434782608696, 0.9782608695652174, 0.826086956521739, 0.9347826086956521, 0.9782608695652174]\n",
      "All AUPRC values:  [0.5470860314523589, 0.7705167173252279, 0.7086956521739131, 0.6074168797953965, 0.8311036789297659, 0.8518518518518519, 0.9583333333333334, 0.7473763118440779, 0.8846153846153846, 0.9782608695652174]\n",
      "All f1-Score values:  [0.5581395348837209, 0.7999999999999999, 0.7441860465116279, 0.6, 0.8979591836734695, 0.92, 0.9787234042553191, 0.8461538461538461, 0.9387755102040816, 0.9777777777777777]\n",
      "------------------------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "## Cocoa\n",
    "other_foods = pd.concat([gdata_features_class_doc2vec,bdata_features_class_doc2vec,adata_features_class_doc2vec,mdata_features_class_doc2vec])\n",
    "get_cross_validation_unseen_food(other_foods,cdata_features_class_doc2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [01:11<00:00, 71.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine1.0\n",
      "Average and SD of AUROC:  0.465 0.10366881026722659\n",
      "Average and SD of AUPRC: 0.5146363636363637 0.048086759032455975\n",
      "Average and SD of f1-Score: 0.23325396825396827 0.16915543547419964\n",
      "All AUROC values:  [0.5833333333333334, 0.5833333333333334, 0.5, 0.5833333333333334, 0.38333333333333336, 0.4, 0.4, 0.44999999999999996, 0.5166666666666667, 0.25]\n",
      "All AUPRC values:  [0.5833333333333334, 0.5555555555555556, 0.5, 0.5555555555555556, 0.51010101010101, 0.5454545454545454, 0.5454545454545454, 0.43272727272727274, 0.4636363636363636, 0.45454545454545453]\n",
      "All f1-Score values:  [0.2857142857142857, 0.4444444444444444, 0.25, 0.4444444444444444, 0.2222222222222222, 0.0, 0.0, 0.4000000000000001, 0.28571428571428575, 0.0]\n",
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine2.0\n",
      "Average and SD of AUROC:  0.5183333333333333 0.12898104942630567\n",
      "Average and SD of AUPRC: 0.5628030303030304 0.09032543066689859\n",
      "Average and SD of f1-Score: 0.23190476190476192 0.21093733150660945\n",
      "All AUROC values:  [0.6666666666666666, 0.33333333333333337, 0.6666666666666666, 0.33333333333333337, 0.5, 0.5833333333333334, 0.6666666666666666, 0.6, 0.4166666666666667, 0.4166666666666667]\n",
      "All AUPRC values:  [0.6666666666666667, 0.5, 0.6666666666666667, 0.45833333333333337, 0.5454545454545454, 0.6212121212121212, 0.696969696969697, 0.5636363636363637, 0.45454545454545453, 0.45454545454545453]\n",
      "All f1-Score values:  [0.5, 0.0, 0.5, 0.2, 0.0, 0.2857142857142857, 0.5, 0.33333333333333337, 0.0, 0.0]\n",
      "------------------------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "## Basil\n",
    "other_foods = pd.concat([gdata_features_class_doc2vec,cdata_features_class_doc2vec,adata_features_class_doc2vec,mdata_features_class_doc2vec])\n",
    "get_cross_validation_unseen_food(other_foods,bdata_features_class_doc2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:44<00:00, 44.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine1.0\n",
      "Average and SD of AUROC:  0.6007843137254902 0.06143304062099646\n",
      "Average and SD of AUPRC: 0.5640218358010893 0.041478061481557485\n",
      "Average and SD of f1-Score: 0.577256202334624 0.09944874448391819\n",
      "All AUROC values:  [0.5475140056022408, 0.5443627450980392, 0.5210084033613445, 0.5042016806722689, 0.6638655462184875, 0.6428571428571429, 0.6848739495798319, 0.6260504201680672, 0.634453781512605, 0.6386554621848739]\n",
      "All AUPRC values:  [0.5248516536814085, 0.5267382368054025, 0.511024211664864, 0.5021281239768635, 0.6063249727371864, 0.5935064935064935, 0.6257749001239841, 0.5773491214667685, 0.5854579119783506, 0.587062732069572]\n",
      "All f1-Score values:  [0.47058823529411764, 0.49302325581395345, 0.48181818181818187, 0.3979591836734694, 0.6799999999999999, 0.62882096069869, 0.6887966804979253, 0.6454183266932272, 0.6329113924050632, 0.653225806451613]\n",
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine2.0\n",
      "Average and SD of AUROC:  0.8808718487394959 0.07846100700963528\n",
      "Average and SD of AUPRC: 0.8434283188501709 0.08381904360027147\n",
      "Average and SD of f1-Score: 0.8736191752121701 0.08953466308713093\n",
      "All AUROC values:  [0.7318277310924369, 0.7911764705882353, 0.8109243697478993, 0.823529411764706, 0.957983193277311, 0.9495798319327732, 0.9411764705882354, 0.9411764705882353, 0.9201680672268907, 0.9411764705882354]\n",
      "All AUPRC values:  [0.6803061646519455, 0.7506276150627615, 0.7805078553160395, 0.7760388559093363, 0.928672268907563, 0.9235710813250919, 0.9029643353404355, 0.9058823529411766, 0.8768173936241164, 0.9088952654232425]\n",
      "All f1-Score values:  [0.7037037037037038, 0.7706422018348624, 0.7867298578199052, 0.8157894736842106, 0.959016393442623, 0.95, 0.943089430894309, 0.9426229508196722, 0.9224489795918367, 0.9421487603305786]\n",
      "------------------------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "## Apple\n",
    "other_foods = pd.concat([gdata_features_class_doc2vec,cdata_features_class_doc2vec,bdata_features_class_doc2vec,mdata_features_class_doc2vec])\n",
    "get_cross_validation_unseen_food(other_foods,adata_features_class_doc2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:48<00:00, 48.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine1.0\n",
      "Average and SD of AUROC:  0.48878205128205127 0.07430438424977137\n",
      "Average and SD of AUPRC: 0.5162673992673993 0.031023000480114728\n",
      "Average and SD of f1-Score: 0.09528822055137846 0.0633249372331306\n",
      "All AUROC values:  [0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5, 0.5, 0.5, 0.5384615384615384, 0.3846153846153846, 0.31089743589743585]\n",
      "All AUPRC values:  [0.5384615384615385, 0.5384615384615385, 0.5384615384615385, 0.5384615384615385, 0.5, 0.5, 0.52, 0.556923076923077, 0.48, 0.45190476190476186]\n",
      "All f1-Score values:  [0.14285714285714288, 0.14285714285714288, 0.14285714285714288, 0.14285714285714288, 0.0, 0.13333333333333336, 0.0, 0.14285714285714288, 0.0, 0.10526315789473685]\n",
      "------------------------------------------------------------------------------------------------------\n",
      "FoodMine2.0\n",
      "Average and SD of AUROC:  0.6608974358974359 0.0797658473348835\n",
      "Average and SD of AUPRC: 0.6323248418248418 0.08399215880648742\n",
      "Average and SD of f1-Score: 0.5920932766699357 0.08657969976546097\n",
      "All AUROC values:  [0.6923076923076923, 0.7307692307692307, 0.6923076923076924, 0.8076923076923077, 0.5, 0.576923076923077, 0.6891025641025642, 0.6506410256410255, 0.6346153846153846, 0.6346153846153846]\n",
      "All AUPRC values:  [0.6398601398601399, 0.7019230769230769, 0.6648351648351648, 0.8076923076923077, 0.5, 0.5461538461538462, 0.6756043956043956, 0.6405128205128205, 0.5733333333333333, 0.5733333333333333]\n",
      "All f1-Score values:  [0.6666666666666667, 0.6666666666666667, 0.6, 0.761904761904762, 0.4347826086956522, 0.5217391304347826, 0.6, 0.5263157894736842, 0.5714285714285715, 0.5714285714285715]\n",
      "------------------------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "## Milk\n",
    "other_foods = pd.concat([gdata_features_class_doc2vec,cdata_features_class_doc2vec,bdata_features_class_doc2vec,adata_features_class_doc2vec])\n",
    "get_cross_validation_unseen_food(other_foods,mdata_features_class_doc2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "foodmine_env",
   "language": "python",
   "name": "foodmine_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}