section5-prediction of-persuasive-effects.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import loader\n",
    "import importlib\n",
    "import pandas as pd\n",
    "importlib.reload(loader)\n",
    "import importlib\n",
    "import lexicons\n",
    "importlib.reload(lexicons)\n",
    "import text_miner\n",
    "importlib.reload(text_miner)\n",
    "import utility\n",
    "import machine_learning\n",
    "importlib.reload(machine_learning)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## IGNORE this section and load the json file containing the extracted features in section below under \"Load Data with **all** Features\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "load = loader.loader()\n",
    "corpus = load.corpus\n",
    "corpus = load.add_ideology_intensity()\n",
    "#articles_per_ideology_intensity = load.get_article_dfs_per_ideology(ideology='ideology_intensity', include_content=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The df contains all articles where duplicates were removed. Total is 979 articles.\n",
    "# It contains the majority effect on liberals and conservatives.\n",
    "# It contains the ADU counts (other, stats, testimony )\n",
    "df = pd.read_json('data/articles_with_majority_adus.json', orient='records')\n",
    "df.set_index('idx', inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "importlib.reload(lexicons)\n",
    "df = lexicons.count_nrc_emotions_and_sentiments(df, text_column= 'content')\n",
    "print(len(df.columns))\n",
    "df = lexicons.count_mpqa_arg(df, text_column= 'content')\n",
    "print(len(df.columns))\n",
    "df = lexicons.count_mpqa_subj_obj(df)\n",
    "print(len(df.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "liwc_df = loader.loader.load_discourse_liwc()\n",
    "df = df.join(liwc_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data with Style Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#utility.save_json(df.reset_index(), 'articles_with_adu_liwc_lexicons_conntent')\n",
    "df = pd.read_json('data/articles_with_adu_liwc_lexicons.json', orient='records')\n",
    "df.set_index('idx', inplace=True)\n",
    "#df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import text_miner\n",
    "def apply_add_lemma(row):\n",
    "    lemma = ( text_miner.preprocess(row['content']))['lemmas']\n",
    "    lemma_str = ' '.join(lemma)\n",
    "    row['lemma'] =lemma_str\n",
    "    return row\n",
    "\n",
    "print(df.columns)\n",
    "df = df.apply(apply_add_lemma, axis=1)\n",
    "print(df.columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df[df['split_label'] == 'train']\n",
    "#importlib.reload(text_miner)\n",
    "print('1 gram')\n",
    "df,  extracted_df1=text_miner.extract_n_grams_features(df, df_train, 'lemma', \n",
    "                             min_df=0.1, max_df=0.7, ngram_range=(1,1),\n",
    "                             count_type='tf-idf', idx= 'idx',\n",
    "                            cols_prefix='lemma1_')\n",
    "\n",
    "print('2 gram')\n",
    "\n",
    "df,  extracted_df2=text_miner.extract_n_grams_features(df, df_train, 'lemma', \n",
    "                             min_df=0.005, max_df=0.9, ngram_range=(2,2),\n",
    "                             count_type='tf-idf', idx= 'idx',\n",
    "                            cols_prefix='lemma2_')\n",
    "\n",
    "print('3 gram')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df,  extracted_df3=text_miner.extract_n_grams_features(df, df_train, 'lemma', \n",
    "                             min_df=0.005, max_df=0.9, ngram_range=(3,3),\n",
    "                             count_type='tf-idf', idx= 'idx',\n",
    "                            cols_prefix='lemma3_')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data with **all** Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#utility.save_json(df.reset_index(), 'articles_with_adu_liwc_lexicons_content')\n",
    "df = pd.read_json('data/articles_with_adu_liwc_lexicons_content.json', orient='records')\n",
    "df.set_index('idx', inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import acl20_editorial_training\n",
    "importlib.reload(machine_learning)\n",
    "importlib.reload(acl20_editorial_training)\n",
    "def validate_train_test(X_train_df, y_train, X_test_df, y_test, feature_types, validation_folds=5):\n",
    "    X_train, X_test = acl20_editorial_training.get_instances_with_features_sub(X_train_df, X_test_df, feature_types)\n",
    "\n",
    "    best_params = machine_learning.svc_param_gridsearch(X_train, y_train, nfolds_or_division=validation_folds)\n",
    "    \n",
    "\n",
    "    result =  machine_learning.train_test(X_train, y_train, X_test, y_test, params=best_params)\n",
    "    result['params'] = best_params\n",
    "    return result\n",
    "def run_experiments(df, filename=None, remove_outliers = True, \n",
    "                    normalize=True, save_models=False):\n",
    "    ideologies = ['liberal_majority', 'conservative_majority']\n",
    "   \n",
    "    \n",
    "    r = {}\n",
    "    \n",
    "    for ideology in ideologies:\n",
    "        ## FOR EACH IDEOLOGY\n",
    "        # For all combination\n",
    "        print(\"preprocessing data...\")\n",
    "        X_train_df, y_train, X_test_df, y_test = acl20_editorial_training.get_x_y(df, ideology, \n",
    "                                                                                  remove_outliers = True,\n",
    "                                                                                 normalizing_method=\"sqrt\")\n",
    "        print(\"END of preprocessing\")\n",
    "\n",
    "        results = []\n",
    "        print(ideology)\n",
    "        print('+++++++++++++++++++++++++++++++++++++++++++++')\n",
    "        all_feature_types_comb = acl20_editorial_training.get_all_feature_types_comb(df)\n",
    "        \n",
    "        for feature_types in all_feature_types_comb:\n",
    "            result = {}\n",
    "            \n",
    "            start_time = time.time()\n",
    "            result =validate_train_test(X_train_df, y_train, X_test_df, y_test, feature_types, validation_folds=5)\n",
    "            \n",
    "            result['features'] = str(feature_types)\n",
    "            result['ideology'] = ideology\n",
    "            results.append(result)\n",
    "            elapsed_time = time.time() - start_time\n",
    "            print(feature_types, ' ', 'macro-f1: ', result['macro'], 'time(s): ', round(elapsed_time,3))\n",
    "            print('-------------------------------------------')\n",
    "            \n",
    "        r[ideology] = pd.DataFrame.from_dict(results)\n",
    "\n",
    "            \n",
    "        if filename is not None:\n",
    "            r[ideology].sort_values(by=['macro'], ascending=False).to_csv('{}_{}.csv'.format(filename, ideology))\n",
    "    return r\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X_train_df, y_train, X_test_df, y_test = get_x_y(df, 'conservative_majority', remove_outliers = True, normalize=True)\n",
    "machine_learning.dummy_train_test(X_train_df, y_train, X_test_df, y_test, strategy='uniform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "importlib.reload(machine_learning)\n",
    "\n",
    "results= run_experiments(df, filename=\"models_no-outliers_sqrt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_stats(df, class_col):\n",
    "    for split, split_df in df.groupby(class_col):\n",
    "        print(split)\n",
    "        print(split_df['split_label'].value_counts()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_stats(df, 'liberal_majority')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_stats(df, 'conservative_majority')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(machine_learning)\n",
    "X_train, y_train, X_test, y_test = get_x_y(df, 'liberal_majority', remove_outliers = True, normalize=True)\n",
    "print()\n",
    "print(machine_learning.dummy_train_test(X_train.values, y_train, X_test.values, y_test, strategy='stratified'))\n",
    "print()\n",
    "print(machine_learning.dummy_train_test(X_train.values, y_train, X_test.values, y_test, strategy='most_frequent'))\n",
    "print()\n",
    "print(machine_learning.dummy_train_test(X_train.values, y_train, X_test.values, y_test, strategy='uniform'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold, KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kf = KFold(n_splits=5)\n",
    "X = [\"a\", \"b\", \"c\", \"d\"]\n",
    "kf = StratifiedKFold(n_splits=5, random_state=1, )\n",
    "for train, test in kf.split(X):\n",
    "    print(\"%s %s\" % (train, test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([[1, 2], [3, 4], [1, 2], [3, 4], [1, 2], [3, 4],\n",
    "             [1, 2], [3, 4], [1, 2], [3, 4], [1, 2], [3, 4]])\n",
    "y = np.array([0, 0, 1, 1,1,0, 0, 0, 1, 1,1,0])\n",
    "skf = StratifiedKFold(n_splits=5)\n",
    "skf.get_n_splits(X, y)\n",
    "\n",
    "for _, test_index in skf.split(X, y):\n",
    "    print( \"TEST:\", test_index)\n",
    "    X_test = X[test_index]\n",
    "    \n",
    "    y_test =y[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(acl20_editorial_training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import acl20_editorial_training\n",
    "def save_model(df, feature_types, ideology):\n",
    "    str_features = '-'.join([str(x) for x in feature_types])\n",
    "    pkl_filename = 'models/'+ideology+'/'+str_features+'.pkl'\n",
    "\n",
    "    X_train_df, y_train, X_test_df, y_test = acl20_editorial_training.get_x_y(df, ideology, \n",
    "                                                                              remove_outliers = True,\n",
    "                                                                             normalizing_method=\"sqrt\")\n",
    "    X_train, X_test = acl20_editorial_training.get_instances_with_features_sub(X_train_df, X_test_df, \n",
    "                                                                                       feature_types)\n",
    "    best_params = machine_learning.svc_param_gridsearch(X_train, y_train, nfolds_or_division=5)\n",
    "    print('saving file: ', pkl_filename)\n",
    "    machine_learning.train_save(X_train, y_train, pkl_filename, \n",
    "                                                         params=best_params)\n",
    "    \n",
    "\n",
    "# Liberal content\n",
    "save_model(df, [ 'lemma'], 'liberal_majority' )\n",
    "# Liberal style+content\n",
    "save_model(df,  ['liwc', 'mpqa_arg', 'mpqa_subjobg', 'lemma'], 'liberal_majority'  )\n",
    "# Liberal style\n",
    "save_model(df,['liwc', 'mpqa_subjobg'], 'liberal_majority'  )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Liberal content\n",
    "save_model(df, [ 'lemma'], 'conservative_majority' )\n",
    "# Liberal style+content\n",
    "save_model(df,  ['mpqa_arg', 'lemma'], 'conservative_majority'  )\n",
    "# Liberal style\n",
    "save_model(df,['nrc', 'adu'], 'conservative_majority'  )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Significancy between Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import acl20_editorial_training\n",
    "import scipy.stats as stats\n",
    "from sklearn.model_selection import StratifiedKFold, KFold\n",
    "from sklearn.metrics import classification_report, confusion_matrix , accuracy_score, f1_score\n",
    "\n",
    "import operator\n",
    "import pickle\n",
    "def dependent_pairs(dv1, dv2, alpha=0.05):\n",
    "\n",
    "    # Check if they are normally distriuted\n",
    "    #diff = list(map(operator.sub, dv1, dv2))\n",
    "    is_normal = stats.shapiro(diff)[1] > 0.05\n",
    "    \n",
    "    stat, p_val = stats.ttest_rel(dv1,dv2) if is_normal else stats.wilcoxon(dv1,dv2)\n",
    "\n",
    "    return stat, p_val, is_normal\n",
    "\n",
    "def run_experiments_with_test_repetition(df, all_feature_types_comb, ideology, score='macro'):\n",
    "    ideologies = ['liberal_majority', 'conservative_majority']\n",
    "    \n",
    "    result = {}\n",
    "    r=[]\n",
    "    ## FOR EACH IDEOLOGY\n",
    "    # For all combination#\n",
    "    \n",
    "    print(\"preprocessing data...\")\n",
    "    X_train_df, y_train, X_test_df, y_test = acl20_editorial_training.get_x_y(df, ideology, \n",
    "                                                                              remove_outliers = True,\n",
    "                                                                             normalizing_method=\"sqrt\")\n",
    "    print(\"END of preprocessing\")\n",
    "\n",
    "    \n",
    "    \n",
    "    print(ideology)\n",
    "    print('+++++++++++++++++++++++++++++++++++++++++++++')\n",
    "    #all_feature_types_comb = get_all_feature_types_comb(df)\n",
    "\n",
    "    for feature_types in all_feature_types_comb:\n",
    "        \n",
    "        print(feature_types)\n",
    "\n",
    "        start_time = time.time()\n",
    "        if feature_types[0] == 'dummy':\n",
    "            f = all_feature_types_comb[1]\n",
    "            X_train, X_test = acl20_editorial_training.get_instances_with_features_sub(X_train_df, X_test_df, \n",
    "                                                                                       f)\n",
    "\n",
    "        else:\n",
    "            X_train, X_test = acl20_editorial_training.get_instances_with_features_sub(X_train_df, X_test_df, \n",
    "                                                                                       feature_types)\n",
    "            \n",
    "\n",
    "        ## 1. svm search grid - Leave one out validation\n",
    "        ## get best param the test on test set 10 folds\n",
    "        #print(best_params)\n",
    "        elapsed_time = time.time() - start_time\n",
    "\n",
    "\n",
    "            \n",
    "        skf = StratifiedKFold(n_splits=5)\n",
    "        skf.get_n_splits(X_test, y_test)\n",
    "\n",
    "        \n",
    "        runs = []\n",
    "        for _, test_index in skf.split(X_test, y_test):\n",
    "            X_sub_test = X_test[test_index]\n",
    "\n",
    "            y_sub_test =y_test[test_index]\n",
    "            \n",
    "            if len(X_sub_test) != 0:\n",
    "                if feature_types[0] == 'dummy':\n",
    "                    macro =  machine_learning.dummy_train_test(X_train, y_train, X_sub_test, y_sub_test, \n",
    "                                                               strategy='uniform')[score]\n",
    "                else:\n",
    "                    str_features = '-'.join([str(x) for x in feature_types])\n",
    "                    pkl_filename = 'models/'+ideology+'/'+str_features+'.pkl'\n",
    "                    \n",
    "                    with open(pkl_filename, 'rb') as file:  \n",
    "                        #print('loading model: ' + pkl_filename)\n",
    "                        pickle_model = pickle.load(file)\n",
    "                        y_pred = pickle_model.predict(X_sub_test) \n",
    "                        macro = f1_score(y_pred=y_pred, y_true=y_sub_test, average=score)\n",
    "                    \n",
    "                    \n",
    "                runs.append(macro)\n",
    "                elapsed_time = time.time() - start_time\n",
    "   \n",
    "        result[str(feature_types)] = runs\n",
    "        r.append(runs)\n",
    "        #print(n, ' - ', feature_types, ' ', results)\n",
    "            \n",
    "    significancy = dependent_pairs(r[0], r[1]) \n",
    "                #r[ideology] = pd.DataFrame.from_dict(results)\n",
    "        #if filename is not None:\n",
    "        #    r[ideology].sort_values(by=['macro'], ascending=False).to_csv('{}_{}.csv'.format(filename, ideology))\n",
    "    return significancy, result\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "liberal_content_baseline[0][1] < 0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('liberal_content_baseline: ')\n",
    "print(liberal_content_baseline_micro )\n",
    "\n",
    "print('\\nliberal_content_contentstyle: ')\n",
    "print(liberal_content_contentstyle_micro)\n",
    "print('\\nliberal_contentstyle_baseline: ')\n",
    "print(liberal_contentstyle_baseline_micro )\n",
    "print('\\nliberal_style_baseline: ')\n",
    "print(liberal_style_baseline_micro)\n",
    "print('\\nliberal_style_content: ')\n",
    "print(liberal_style_content_micro)\n",
    "print('\\nliberal_style_stylecontent: ')\n",
    "print(liberal_style_stylecontent_micro )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('liberal_content_baseline: ')\n",
    "print(liberal_content_baseline_micro[0][1] < 0.05)\n",
    "\n",
    "print('\\nliberal_content_contentstyle: ')\n",
    "print(liberal_content_contentstyle_micro[0][1] < 0.05)\n",
    "print('\\nliberal_contentstyle_baseline: ')\n",
    "print(liberal_contentstyle_baseline_micro[0][1] < 0.05)\n",
    "print('\\nliberal_style_baseline: ')\n",
    "print(liberal_style_baseline_micro[0][1] < 0.05)\n",
    "print('\\nliberal_style_content: ')\n",
    "print(liberal_style_content_micro[0][1] < 0.05)\n",
    "print('\\nliberal_style_stylecontent: ')\n",
    "print(liberal_style_stylecontent_micro[0][1] < 0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('liberal_content_baseline: ')\n",
    "print(liberal_content_baseline[0][1] < 0.05)\n",
    "\n",
    "print('\\nliberal_content_contentstyle: ')\n",
    "print(liberal_content_contentstyle[0][1] < 0.05)\n",
    "print('\\nliberal_contentstyle_baseline: ')\n",
    "print(liberal_contentstyle_baseline[0][1] < 0.05)\n",
    "print('\\nliberal_style_baseline: ')\n",
    "print(liberal_style_baseline[0][1] < 0.05)\n",
    "print('\\nliberal_style_content: ')\n",
    "print(liberal_style_content[0][1] < 0.05)\n",
    "print('\\nliberal_style_stylecontent: ')\n",
    "print(liberal_style_stylecontent[0][1] < 0.05)\n",
    "\n",
    "print('liberal_content_baseline: ')\n",
    "print(liberal_content_baseline[0][1] )\n",
    "\n",
    "print('\\nliberal_content_contentstyle: ')\n",
    "print(liberal_content_contentstyle[0][1] )\n",
    "print('\\nliberal_contentstyle_baseline: ')\n",
    "print(liberal_contentstyle_baseline[0][1] )\n",
    "print('\\nliberal_style_baseline: ')\n",
    "print(liberal_style_baseline[0][1] )\n",
    "print('\\nliberal_style_content: ')\n",
    "print(liberal_style_content[0][1] )\n",
    "print('\\nliberal_style_stylecontent: ')\n",
    "print(liberal_style_stylecontent[0][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LIBERAL\n",
    "# style + content VS. Content\n",
    "print('Liberal')\n",
    "\n",
    "# (content) VS. dummy  NOT SIGNIFICANT\n",
    "liberal_content_baseline_micro = run_experiments_with_test_repetition(df, [['dummy'],['lemma']],\n",
    "                                     'liberal_majority', score='micro')\n",
    "    \n",
    "    \n",
    "liberal_content_contentstyle_micro = run_experiments_with_test_repetition(df, [['liwc', 'mpqa_arg', 'mpqa_subjobg', 'lemma'],\n",
    "                                                                        ['lemma']],\n",
    "                                     'liberal_majority', score='micro')\n",
    "\n",
    "\n",
    "liberal_contentstyle_baseline_micro = run_experiments_with_test_repetition(df, [['dummy'], ['liwc', 'mpqa_arg',\n",
    "                                                                                      'mpqa_subjobg', 'lemma']],\n",
    "                                     'liberal_majority', score='micro')\n",
    "\n",
    "liberal_style_baseline_micro = run_experiments_with_test_repetition(df, [['dummy'], ['liwc', 'mpqa_subjobg'] \n",
    "                                                                     ],\n",
    "                                     'liberal_majority', score='micro')\n",
    "\n",
    "liberal_style_content_micro = run_experiments_with_test_repetition(df, [['lemma'], ['liwc', 'mpqa_subjobg'] \n",
    "                                                                     ],\n",
    "                                     'liberal_majority', score='micro')\n",
    "\n",
    "liberal_style_stylecontent_micro = run_experiments_with_test_repetition(df, [['liwc', 'mpqa_arg', 'mpqa_subjobg', 'lemma'], ['liwc', 'mpqa_subjobg'] \n",
    "                                                                     ],\n",
    "                                     'liberal_majority', score='micro')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LIBERAL\n",
    "# style + content VS. Content\n",
    "print('Conservative') # (1.0, 0.0782481864330958), NOT SIGNI\n",
    "print(\"(style + content) VS. Content\")\n",
    "cons_contentstyle_content_strat = run_experiments_with_test_repetition(df, [['mpqa_arg', 'lemma'], ['lemma']],\n",
    "                                     'conservative_majority', score='macro')\n",
    "\n",
    "\n",
    "\n",
    "print(\"(style + content) VS. dummy\") # baseline SIGN better: 0.0, 0.043114446783075355\n",
    "cons_contentstyle_dummy_strat = run_experiments_with_test_repetition(df, [['dummy'], ['mpqa_arg', 'lemma']],\n",
    "                                     'conservative_majority', score='macro')\n",
    "\n",
    "\n",
    "print(\"content VS. dummy\") # baseline SIGN better: (0.0, 0.04216819709715596)\n",
    "cons_content_dummy_strat = run_experiments_with_test_repetition(df, [['dummy'], ['lemma']],\n",
    "                                     'conservative_majority', score='macro')\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "cons_content_style = ['mpqa_arg', 'lemma']\n",
    "cons_style = ['nrc', 'adu']\n",
    "conservative_content_baseline = run_experiments_with_test_repetition(df, [['dummy'],['lemma']],\n",
    "                                     'conservative_majority', score='macro')\n",
    "    \n",
    "    \n",
    "conservative_content_contentstyle = run_experiments_with_test_repetition(df, [cons_content_style,\n",
    "                                                                        ['lemma']],\n",
    "                                     'conservative_majority', score='macro')\n",
    "\n",
    "\n",
    "conservative_contentstyle_baseline = run_experiments_with_test_repetition(df, [['dummy'], cons_content_style],\n",
    "                                     'conservative_majority', score='macro')\n",
    "\n",
    "conservative_style_baseline = run_experiments_with_test_repetition(df, [['dummy'],  cons_style  ],\n",
    "                                     'conservative_majority', score='macro')\n",
    "\n",
    "conservative_style_content = run_experiments_with_test_repetition(df, [['lemma'],  cons_style ],\n",
    "                                     'conservative_majority', score='macro')\n",
    "\n",
    "conservative_style_stylecontent = run_experiments_with_test_repetition(df, [cons_style, cons_content_style \n",
    "                                                                     ],\n",
    "                                     'conservative_majority', score='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('conservative_content_baseline: ')\n",
    "print(conservative_content_baseline[0][1] )\n",
    "\n",
    "print('\\nconservative_content_contentstyle: ')\n",
    "print(conservative_content_contentstyle[0][1] )\n",
    "print('\\nconservative_contentstyle_baseline: ')\n",
    "print(conservative_contentstyle_baseline[0][1] )\n",
    "print('\\nconservative_style_baseline: ')\n",
    "print(conservative_style_baseline[0][1] )\n",
    "print('\\nconservative_style_content: ')\n",
    "print(conservative_style_content[0][1] )\n",
    "print('\\nconservative_style_stylecontent: ')\n",
    "print(conservative_style_stylecontent[0][1] )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "\n",
    "\n",
    "MACRO\n",
    "\n",
    "liberal - content VS. baseline: \n",
    "False\n",
    "liberal - contentstyle VS. baseline: \n",
    "True\n",
    "liberal - style VS. baseline: \n",
    "True\n",
    "\n",
    "\n",
    "\n",
    "liberal - style VS. content: \n",
    "False\n",
    "liberal - content VS. contentstyle: \n",
    "True\n",
    "liberal - style VS. stylecontent: \n",
    "False"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}