dataset_statistics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from statsmodels.stats.inter_rater import fleiss_kappa\n",
    "from statsmodels.stats import inter_rater as irr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "def print_question_statistics(annotations):\n",
    "    questions = ['cat1', 'cat2', 'cat3', 'compoundclaim', 'ironic']\n",
    "\n",
    "    disagreement_df = pd.DataFrame(columns=questions, index=['no', 'yes'])\n",
    "    disagreement_level_df = pd.DataFrame(columns=questions, index=['no', 'little', 'high'])\n",
    "    mean_answers_df = pd.DataFrame(columns=questions, index=[0, 0.25, 0.5, 0.75, 1])\n",
    "    consolidated_answers_df = pd.DataFrame(columns=questions, index=[0, 1])\n",
    "    fleiss_df = pd.DataFrame(columns=questions, index=['fleiss_kappa'])\n",
    "\n",
    "    all_no = len(annotations[(annotations['cat1_final_answer']==0.0) & (annotations['cat2_final_answer']==0.0) & (annotations['cat3_final_answer']==0.0)])\n",
    "    one_yes = len(annotations[(annotations['cat1_final_answer']==1.0) | (annotations['cat2_final_answer']==1.0) | (annotations['cat3_final_answer']==1.0)])\n",
    "    all_no_or_hd = len(annotations) - all_no - one_yes\n",
    "\n",
    "    for question in questions:\n",
    "        mean_answers = annotations[f'{question}_mean_answer']\n",
    "        disagreement_level = mean_answers.apply(lambda x: {0: 'no', 1: 'no', 0.25:'little', 0.75:'little', 0.5:'high'}[x])\n",
    "        consolidated_answer = annotations[annotations[f'{question}_final_answer']!=0.5][f'{question}_final_answer']\n",
    "\n",
    "        disagreement = disagreement_level != 'no'\n",
    "        answers = annotations[f'{question}_answers']\n",
    "\n",
    "        disagreement_df.loc['yes'][question] = f'{disagreement.mean() * 100:.2f}'\n",
    "        disagreement_df.loc['no'][question] = f'{100 - disagreement.mean() * 100:.2f}'\n",
    "\n",
    "        disagreement_level_vc = disagreement_level.value_counts(normalize=False)\n",
    "        disagreement_level_dict = {'no': 0, 'little':0, 'high':0}\n",
    "        disagreement_level_dict.update(disagreement_level_vc.to_dict())\n",
    "        disagreement_level_perc_vc = disagreement_level.value_counts(normalize=True)\n",
    "        disagreement_level_perc_dict = {'no':0.0, 'little':0.0, 'high':0.0}\n",
    "        disagreement_level_perc_dict.update(disagreement_level_perc_vc.to_dict())\n",
    "        disagreement_level_df.loc['no'][question] = f'{disagreement_level_dict[\"no\"]:.2f} ({disagreement_level_perc_dict[\"no\"]*100:.2f}%)'\n",
    "        disagreement_level_df.loc['little'][question] = f'{disagreement_level_dict[\"little\"]:.2f} ({disagreement_level_perc_dict[\"little\"]*100:.2f}%)'\n",
    "        disagreement_level_df.loc['high'][question] = f'{disagreement_level_dict[\"high\"]:.2f} ({disagreement_level_perc_dict[\"high\"]*100:.2f}%)'\n",
    "\n",
    "        mean_answers_vc= mean_answers.value_counts(normalize=False)\n",
    "        mean_answers_dict = {0: 0, 0.25:0, 0.5:0, 0.75:0, 1:0}\n",
    "        mean_answers_dict.update(mean_answers_vc.to_dict())\n",
    "        mean_answers_perc_vc= mean_answers.value_counts(normalize=True)\n",
    "        mean_answers_perc_dict = {0: 0.0, 0.25:0.0, 0.5:0.0, 0.75:0.0, 1:0.0}\n",
    "        mean_answers_perc_dict.update(mean_answers_perc_vc.to_dict())\n",
    "        mean_answers_df.loc[0][question] = f'{mean_answers_dict[0]:.2f} ({mean_answers_perc_dict[0]*100:.2f}%)'\n",
    "        mean_answers_df.loc[0.25][question] = f'{mean_answers_dict[0.25]:.2f} ({mean_answers_perc_dict[0.25]*100:.2f}%)'\n",
    "        mean_answers_df.loc[0.5][question] = f'{mean_answers_dict[0.5]:.2f} ({mean_answers_perc_dict[0.5]*100:.2f}%)'\n",
    "        mean_answers_df.loc[0.75][question] = f'{mean_answers_dict[0.75]:.2f} ({mean_answers_perc_dict[0.75]*100:.2f}%)'\n",
    "        mean_answers_df.loc[1][question] = f'{mean_answers_dict[1]:.2f} ({mean_answers_perc_dict[1]*100:.2f}%)'\n",
    "\n",
    "        consolidated_answer_vc = consolidated_answer.value_counts(normalize=False)\n",
    "        consolidated_answer_dict = {0: 0, 1:0}\n",
    "        consolidated_answer_dict.update(consolidated_answer_vc.to_dict())\n",
    "        consolidated_answer_perc_vc = consolidated_answer.value_counts(normalize=True)\n",
    "        consolidated_answer_perc_dict = {0:0.0, 1:0.0}\n",
    "        consolidated_answer_perc_dict.update(consolidated_answer_perc_vc.to_dict())\n",
    "        consolidated_answers_df.loc[0][question] = f'{consolidated_answer_dict[0]:.2f} ({consolidated_answer_perc_dict[0]*100:.2f}%)'\n",
    "        consolidated_answers_df.loc[1][question] = f'{consolidated_answer_dict[1]:.2f} ({consolidated_answer_perc_dict[1]*100:.2f}%)'\n",
    "\n",
    "        answers = answers.apply(lambda x: [int(a) for a in x.split(\" \")])\n",
    "        answers = np.array(answers.tolist())\n",
    "        agg = irr.aggregate_raters(answers)  # returns a tuple (data, categories)\n",
    "        fleiss_df.loc['fleiss_kappa'][question] = f'{fleiss_kappa(agg[0], method=\"fleiss\"):.2f}'\n",
    "\n",
    "    print('number of tweets:')\n",
    "    print(len(annotations))\n",
    "    print()\n",
    "\n",
    "    print('number of negative tweets (not related to science):')\n",
    "    print(all_no)\n",
    "    print()\n",
    "\n",
    "    print('number of tweets without any positive label (only negative or high disagreement labels):')\n",
    "    print(all_no_or_hd)\n",
    "    print()\n",
    "\n",
    "    print('number of positive tweets (related to science):')\n",
    "    print(one_yes)\n",
    "    print()\n",
    "\n",
    "    print('disagreement (no/ yes) statistics')\n",
    "    print(disagreement_df)\n",
    "    print()\n",
    "\n",
    "    print('disagreement (no/ little/ high) statistics')\n",
    "    print(disagreement_level_df)\n",
    "    print()\n",
    "\n",
    "    print('mean answers statistics')\n",
    "    print(mean_answers_df)\n",
    "    print()\n",
    "\n",
    "    print('consolidated answers statistics')\n",
    "    print(consolidated_answers_df)\n",
    "    print()\n",
    "\n",
    "    print('fleiss kappa statistics')\n",
    "    print(fleiss_df)\n",
    "    print()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "annotations = pd.read_csv('annotations/annotations.tsv', sep='\\t')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of tweets:\n",
      "1046\n",
      "\n",
      "number of negative tweets (not related to science):\n",
      "751\n",
      "\n",
      "number of tweets without any positive label (only negative or high disagreement labels):\n",
      "84\n",
      "\n",
      "number of positive tweets (related to science):\n",
      "211\n",
      "\n",
      "disagreement (no/ yes) statistics\n",
      "      cat1   cat2   cat3 compoundclaim ironic\n",
      "no   72.37  86.23  82.03         86.33  95.22\n",
      "yes  27.63  13.77  17.97         13.67   4.78\n",
      "\n",
      "disagreement (no/ little/ high) statistics\n",
      "                   cat1             cat2             cat3    compoundclaim  \\\n",
      "no      757.00 (72.37%)  902.00 (86.23%)  858.00 (82.03%)  903.00 (86.33%)   \n",
      "little  216.00 (20.65%)    97.00 (9.27%)  142.00 (13.58%)  115.00 (10.99%)   \n",
      "high      73.00 (6.98%)    47.00 (4.49%)    46.00 (4.40%)    28.00 (2.68%)   \n",
      "\n",
      "                 ironic  \n",
      "no      996.00 (95.22%)  \n",
      "little    40.00 (3.82%)  \n",
      "high      10.00 (0.96%)  \n",
      "\n",
      "mean answers statistics\n",
      "                 cat1             cat2             cat3    compoundclaim  \\\n",
      "0.00  653.00 (62.43%)  871.00 (83.27%)  793.00 (75.81%)  900.00 (86.04%)   \n",
      "0.25  170.00 (16.25%)    56.00 (5.35%)    83.00 (7.93%)   103.00 (9.85%)   \n",
      "0.50    73.00 (6.98%)    47.00 (4.49%)    46.00 (4.40%)    28.00 (2.68%)   \n",
      "0.75    46.00 (4.40%)    41.00 (3.92%)    59.00 (5.64%)    12.00 (1.15%)   \n",
      "1.00   104.00 (9.94%)    31.00 (2.96%)    65.00 (6.21%)     3.00 (0.29%)   \n",
      "\n",
      "               ironic  \n",
      "0.00  996.00 (95.22%)  \n",
      "0.25    39.00 (3.73%)  \n",
      "0.50    10.00 (0.96%)  \n",
      "0.75     1.00 (0.10%)  \n",
      "1.00     0.00 (0.00%)  \n",
      "\n",
      "consolidated answers statistics\n",
      "              cat1             cat2             cat3     compoundclaim  \\\n",
      "0  823.00 (84.58%)  927.00 (92.79%)  876.00 (87.60%)  1003.00 (98.53%)   \n",
      "1  150.00 (15.42%)    72.00 (7.21%)  124.00 (12.40%)     15.00 (1.47%)   \n",
      "\n",
      "             ironic  \n",
      "0  1035.00 (99.90%)  \n",
      "1      1.00 (0.10%)  \n",
      "\n",
      "fleiss kappa statistics\n",
      "              cat1  cat2  cat3 compoundclaim ironic\n",
      "fleiss_kappa  0.55  0.56  0.61          0.23   0.13\n",
      "\n"
     ]
    }
   ],
   "source": [
    "round1_annotations = annotations[annotations['round'] == 1]\n",
    "print_question_statistics(round1_annotations)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of tweets:\n",
      "215\n",
      "\n",
      "number of negative tweets (not related to science):\n",
      "24\n",
      "\n",
      "number of tweets without any positive label (only negative or high disagreement labels):\n",
      "0\n",
      "\n",
      "number of positive tweets (related to science):\n",
      "191\n",
      "\n",
      "disagreement (no/ yes) statistics\n",
      "      cat1   cat2   cat3 compoundclaim ironic\n",
      "no   63.72  60.47  62.33         72.09  96.74\n",
      "yes  36.28  39.53  37.67         27.91   3.26\n",
      "\n",
      "disagreement (no/ little/ high) statistics\n",
      "                   cat1             cat2             cat3    compoundclaim  \\\n",
      "no      137.00 (63.72%)  130.00 (60.47%)  134.00 (62.33%)  155.00 (72.09%)   \n",
      "little   58.00 (26.98%)   55.00 (25.58%)   56.00 (26.05%)   42.00 (19.53%)   \n",
      "high      20.00 (9.30%)   30.00 (13.95%)   25.00 (11.63%)    18.00 (8.37%)   \n",
      "\n",
      "                 ironic  \n",
      "no      208.00 (96.74%)  \n",
      "little     5.00 (2.33%)  \n",
      "high       2.00 (0.93%)  \n",
      "\n",
      "mean answers statistics\n",
      "                cat1            cat2            cat3    compoundclaim  \\\n",
      "0.00  46.00 (21.40%)  70.00 (32.56%)  55.00 (25.58%)  154.00 (71.63%)   \n",
      "0.25  28.00 (13.02%)   20.00 (9.30%)   19.00 (8.84%)   28.00 (13.02%)   \n",
      "0.50   20.00 (9.30%)  30.00 (13.95%)  25.00 (11.63%)    18.00 (8.37%)   \n",
      "0.75  30.00 (13.95%)  35.00 (16.28%)  37.00 (17.21%)    14.00 (6.51%)   \n",
      "1.00  91.00 (42.33%)  60.00 (27.91%)  79.00 (36.74%)     1.00 (0.47%)   \n",
      "\n",
      "               ironic  \n",
      "0.00  208.00 (96.74%)  \n",
      "0.25     4.00 (1.86%)  \n",
      "0.50     2.00 (0.93%)  \n",
      "0.75     1.00 (0.47%)  \n",
      "1.00     0.00 (0.00%)  \n",
      "\n",
      "consolidated answers statistics\n",
      "              cat1             cat2             cat3    compoundclaim  \\\n",
      "0   82.00 (38.14%)   97.00 (45.12%)   80.00 (37.21%)  183.00 (85.12%)   \n",
      "1  133.00 (61.86%)  118.00 (54.88%)  135.00 (62.79%)   32.00 (14.88%)   \n",
      "\n",
      "            ironic  \n",
      "0  212.00 (98.60%)  \n",
      "1     3.00 (1.40%)  \n",
      "\n",
      "fleiss kappa statistics\n",
      "              cat1  cat2  cat3 compoundclaim ironic\n",
      "fleiss_kappa  0.59  0.56  0.57          0.31   0.29\n",
      "\n"
     ]
    }
   ],
   "source": [
    "round2_annotations = annotations[annotations['round'] == 2]\n",
    "print_question_statistics(round2_annotations)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of tweets:\n",
      "1261\n",
      "\n",
      "number of negative tweets (not related to science):\n",
      "775\n",
      "\n",
      "number of tweets without any positive label (only negative or high disagreement labels):\n",
      "84\n",
      "\n",
      "number of positive tweets (related to science):\n",
      "402\n",
      "\n",
      "disagreement (no/ yes) statistics\n",
      "      cat1   cat2   cat3 compoundclaim ironic\n",
      "no   70.90  81.84  78.67         83.90  95.48\n",
      "yes  29.10  18.16  21.33         16.10   4.52\n",
      "\n",
      "disagreement (no/ little/ high) statistics\n",
      "                   cat1              cat2             cat3     compoundclaim  \\\n",
      "no      894.00 (70.90%)  1032.00 (81.84%)  992.00 (78.67%)  1058.00 (83.90%)   \n",
      "little  274.00 (21.73%)   152.00 (12.05%)  198.00 (15.70%)   157.00 (12.45%)   \n",
      "high      93.00 (7.38%)     77.00 (6.11%)    71.00 (5.63%)     46.00 (3.65%)   \n",
      "\n",
      "                  ironic  \n",
      "no      1204.00 (95.48%)  \n",
      "little     45.00 (3.57%)  \n",
      "high       12.00 (0.95%)  \n",
      "\n",
      "mean answers statistics\n",
      "                 cat1             cat2             cat3     compoundclaim  \\\n",
      "0.00  699.00 (55.43%)  941.00 (74.62%)  848.00 (67.25%)  1054.00 (83.58%)   \n",
      "0.25  198.00 (15.70%)    76.00 (6.03%)   102.00 (8.09%)   131.00 (10.39%)   \n",
      "0.50    93.00 (7.38%)    77.00 (6.11%)    71.00 (5.63%)     46.00 (3.65%)   \n",
      "0.75    76.00 (6.03%)    76.00 (6.03%)    96.00 (7.61%)     26.00 (2.06%)   \n",
      "1.00  195.00 (15.46%)    91.00 (7.22%)  144.00 (11.42%)      4.00 (0.32%)   \n",
      "\n",
      "                ironic  \n",
      "0.00  1204.00 (95.48%)  \n",
      "0.25     43.00 (3.41%)  \n",
      "0.50     12.00 (0.95%)  \n",
      "0.75      2.00 (0.16%)  \n",
      "1.00      0.00 (0.00%)  \n",
      "\n",
      "consolidated answers statistics\n",
      "              cat1              cat2             cat3     compoundclaim  \\\n",
      "0  905.00 (76.18%)  1024.00 (84.35%)  956.00 (78.68%)  1186.00 (96.19%)   \n",
      "1  283.00 (23.82%)   190.00 (15.65%)  259.00 (21.32%)     47.00 (3.81%)   \n",
      "\n",
      "             ironic  \n",
      "0  1247.00 (99.68%)  \n",
      "1      4.00 (0.32%)  \n",
      "\n",
      "fleiss kappa statistics\n",
      "              cat1  cat2  cat3 compoundclaim ironic\n",
      "fleiss_kappa  0.61  0.63  0.66          0.26   0.15\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_question_statistics(annotations)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}