plot.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import and Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import csv\n",
    "import json\n",
    "import math\n",
    "import openai\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "from scipy import stats\n",
    "from datetime import datetime\n",
    "from collections import defaultdict, Counter\n",
    "from matplotlib.ticker import FuncFormatter\n",
    "from matplotlib.colors import ListedColormap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_brackets(text, brackets='[]'):\n",
    "    assert len(brackets) == 2\n",
    "    pattern = re.escape(brackets[0]) + r'(.*?)' + re.escape(brackets[1])\n",
    "    matches = re.findall(pattern, text)\n",
    "    return matches\n",
    "\n",
    "def extract_amout(\n",
    "    message, \n",
    "    prefix='',\n",
    "    print_except=True,\n",
    "    type=float,\n",
    "    brackets='[]'\n",
    "):\n",
    "    try:\n",
    "        matches = extract_brackets(message, brackets=brackets)\n",
    "        matches = [s[len(prefix):] \\\n",
    "            if s.startswith(prefix) \\\n",
    "            else s for s in matches]\n",
    "        invalid = False\n",
    "        if len(matches) == 0:\n",
    "            invalid = True\n",
    "        for i in range(len(matches)):\n",
    "            if matches[i] != matches[0]:\n",
    "                invalid = True\n",
    "        if invalid:\n",
    "            raise ValueError('Invalid answer: %s' % message)\n",
    "        return type(matches[0])\n",
    "    except Exception as e: \n",
    "        if print_except: print(e)\n",
    "        return None\n",
    "\n",
    "def extract_choices(recrods):\n",
    "    choices = [extract_amout(\n",
    "        messages[-1]['content'], \n",
    "        prefix='$', \n",
    "        print_except=True,\n",
    "        type=float) for messages in records['messages']\n",
    "    ]\n",
    "    choices = [x for x in choices if x is not None]\n",
    "    # print(choices)\n",
    "    return choices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def choices_to_df(choices, hue):\n",
    "    df = pd.DataFrame(choices, columns=['choices'])\n",
    "    df['hue'] = hue\n",
    "    df['hue'] = df['hue'].astype(str)\n",
    "    return df\n",
    "\n",
    "def plot(\n",
    "        df, \n",
    "        title='',\n",
    "        x='choices',\n",
    "        hue='hue',\n",
    "        binrange=None, \n",
    "        binwidth=None,\n",
    "        stat='count',\n",
    "        multiple='dodge'\n",
    "    ):\n",
    "    if binrange is None:\n",
    "        binrange = (df[x].min(), df[x].max())\n",
    "    df = df.dropna(axis=0, subset=[x]).reset_index()\n",
    "    plt.figure(figsize=(8, 6))\n",
    "    ax = sns.histplot(\n",
    "        data=df, \n",
    "        x=x,\n",
    "        hue=hue, \n",
    "        kde=True,\n",
    "        binrange=binrange, \n",
    "        binwidth=binwidth,\n",
    "        stat=stat,\n",
    "        multiple=multiple,\n",
    "        shrink=.8,\n",
    "    )\n",
    "    ax.set_title(title)\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_facet(\n",
    "    df_list,\n",
    "    x='choices',\n",
    "    hue='hue',\n",
    "    palette=None,\n",
    "    binrange=None,\n",
    "    bins=10,\n",
    "    # binwidth=10,\n",
    "    stat='count',\n",
    "    x_label='',\n",
    "    sharex=True,\n",
    "    sharey=False,\n",
    "    subplot=sns.histplot,\n",
    "    xticks_locs=None,\n",
    "    # kde=False,\n",
    "    **kwargs\n",
    "):\n",
    "    data = pd.concat(df_list)\n",
    "    if binrange is None:\n",
    "        binrange = (data[x].min(), data[x].max())\n",
    "    g = sns.FacetGrid(\n",
    "        data, row=hue, hue=hue, \n",
    "        palette=palette,\n",
    "        aspect=2, height=2, \n",
    "        sharex=sharex, sharey=sharey,\n",
    "        despine=True,\n",
    "    )\n",
    "    g.map_dataframe(\n",
    "        subplot, \n",
    "        x=x, \n",
    "        # kde=kde, \n",
    "        binrange=binrange, \n",
    "        bins=bins,\n",
    "        stat=stat,\n",
    "        **kwargs\n",
    "    )\n",
    "    # g.add_legend(title='hue')\n",
    "    g.set_axis_labels(x_label, stat.title())\n",
    "    g.set_titles(row_template=\"{row_name}\")\n",
    "    for ax in g.axes.flat:\n",
    "        ax.yaxis.set_major_formatter(\n",
    "            FuncFormatter(lambda y, pos: '{:.2f}'.format(y))\n",
    "        )\n",
    "    \n",
    "    binwidth = (binrange[1] - binrange[0]) / bins\n",
    "    if xticks_locs is None:\n",
    "        locs = np.linspace(binrange[0], binrange[1], bins//2+1)\n",
    "        locs = [loc + binwidth for loc in locs]\n",
    "    else: \n",
    "        locs = xticks_locs\n",
    "    labels = [str(int(loc)) for loc in locs]\n",
    "    locs = [loc + 0.5*binwidth for loc in locs]\n",
    "    plt.xticks(locs, labels)\n",
    "    \n",
    "    g.set(xlim=binrange)\n",
    "    return g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set(rc={'figure.figsize':(5,4)})\n",
    "sns.set_style(\"ticks\")\n",
    "\n",
    "default_palette = sns.color_palette(None)\n",
    "blue = default_palette[0]\n",
    "orange = default_palette[1]\n",
    "green = default_palette[2]\n",
    "red = default_palette[3]\n",
    "purple = default_palette[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_occupations(\n",
    "    df_baseline,\n",
    "    choices_all,\n",
    "    binrange=(0,100),\n",
    "    binwidth=5,\n",
    "    x_label='$ to give',\n",
    "    stat='density',\n",
    "    model='ChatGPT-4',\n",
    "):\n",
    "    print('baseline: ', len(df_baseline))\n",
    "    df_list = []\n",
    "    for occupation in choices_all:\n",
    "        choices = choices_all[occupation]\n",
    "        print(occupation, ':', len(choices))\n",
    "        df = choices_to_df(choices, hue='{} ({})'.format(model, occupation))\n",
    "        df_list.append(df)\n",
    "    g = plot_facet(\n",
    "        df_list=[df_baseline]+df_list,\n",
    "        binrange=binrange,\n",
    "        binwidth=binwidth,\n",
    "        x_label=x_label,\n",
    "        stat=stat,\n",
    "    )\n",
    "    return g"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig. 1: Big Five\n",
    "\n",
    "### Human Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigfive_human_data = pd.read_csv('data/bigfive_data.csv', delimiter='\\t')\n",
    "bigfive_human_data['hue'] = 'Human'\n",
    "bigfive_human_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = sns.histplot(\n",
    "    bigfive_human_data['age'],\n",
    "    bins=20,\n",
    "    binrange=(13, 80),\n",
    ")\n",
    "ax.set_xlabel('Age')\n",
    "ax.set_ylabel('# of subjects')\n",
    "plt.savefig('figures/demo-bigfive-age.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### independent 30 instances\n",
    "records_gpt4 = json.load(open('records/bigfive_gpt4_2023_06_26-01_37_11_PM.json', 'r'))\n",
    "records_turbo = json.load(open('records/bigfive_turbo_2023_06_26-02_06_26_AM.json', 'r'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigfive_model_data = {}\n",
    "bigfive_model_data['gpt4'] = pd.DataFrame(records_gpt4['choices'])\n",
    "bigfive_model_data['turbo'] = pd.DataFrame(records_turbo['choices'])\n",
    "bigfive_model_data['gpt4']['hue'] = 'ChatGPT-4'\n",
    "bigfive_model_data['turbo']['hue'] = 'ChatGPT-3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = {}\n",
    "with open('data/bigfive.tsv', 'r') as tsvfile:\n",
    "    reader = csv.reader(tsvfile, delimiter='\\t')\n",
    "    for row in reader:\n",
    "        questions[row[0]] = row[1]\n",
    "\n",
    "keyed = {}\n",
    "indices = defaultdict(int)\n",
    "dimensions = 'EACNO'\n",
    "with open('data/bigfive_IPIP.tsv', 'r') as tsvfile:\n",
    "    reader = csv.reader(tsvfile, delimiter='\\t')\n",
    "    for row in reader:\n",
    "        d = dimensions[int(row[-1][1])-1]\n",
    "        v = row[-1][2]\n",
    "        indices[d] += 1\n",
    "        k = '%s%i' % (d, indices[d])\n",
    "        keyed[k] = v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = ['gpt4', 'turbo']\n",
    "beg_pos = list(range(7, 100, 10))[:5]\n",
    "# d_scores_model = defaultdict(dict)\n",
    "\n",
    "dimensions = 'ENACO'\n",
    "for i, d in enumerate(dimensions):\n",
    "    \n",
    "    ### human scores\n",
    "    d_score = 0\n",
    "    for j in range(10):\n",
    "        k = '%s%i' % (d, j+1)\n",
    "        v = keyed[k]\n",
    "        score = bigfive_human_data.iloc[:, beg_pos[i]+j]\n",
    "        if v == '-': score = 6 - score\n",
    "        d_score += score\n",
    "    bigfive_human_data[d] = d_score\n",
    "\n",
    "    ### model scores\n",
    "    for model in models:\n",
    "        d_score = 0\n",
    "        records = eval('records_%s' % model)\n",
    "        for j in range(10):\n",
    "            k = '%s%i' % (d, j+1)\n",
    "            v = keyed[k]\n",
    "            score = bigfive_model_data[model].iloc[:, i*10+j]\n",
    "            # score = np.mean(records['choices'][k])\n",
    "            if v == '-': score = 6 - score\n",
    "            d_score += score\n",
    "        # d_scores_model[model][d] = d_score\n",
    "        bigfive_model_data[model][d] = d_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([\n",
    "    bigfive_human_data, \n",
    "    bigfive_model_data['gpt4'], \n",
    "    bigfive_model_data['turbo']\n",
    "], ignore_index=True)\n",
    "data[['E', 'hue']]\n",
    "data['N'] = 60 - data['N']\n",
    "data[[*dimensions, 'hue']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import percentileofscore\n",
    "\n",
    "for d in dimensions:\n",
    "    median_gpt4 = np.median(data[data['hue'] == 'ChatGPT-4'][d])\n",
    "    median_turbo = np.median(data[data['hue'] == 'ChatGPT-3'][d])\n",
    "    print(d, median_gpt4, median_turbo)\n",
    "    human_data = data[data['hue'] == 'Human'][d]\n",
    "    print('gpt4', percentileofscore(human_data, median_gpt4))\n",
    "    print('turbo', percentileofscore(human_data, median_turbo))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from math import pi\n",
    " \n",
    "sns.set(rc={'figure.figsize':(10,8)})\n",
    "sns.set(font_scale=1.5)\n",
    "sns.set_style('whitegrid')\n",
    "\n",
    "categories=['Extraversion', 'Neuroticism', 'Agreeableness', 'Conscientiousness', 'Openness']\n",
    "categories=[*dimensions]\n",
    "categories = [''] * len(dimensions)\n",
    "N = len(categories)\n",
    "value_type = 'median'\n",
    "angles = [n / float(N) * 2 * pi for n in range(N)]\n",
    "angles += angles[:1]\n",
    "ax = plt.subplot(111, polar=True)\n",
    "ax.set_theta_offset(pi / 2)\n",
    "ax.set_theta_direction(-1)\n",
    "\n",
    "hues = ['Human', 'ChatGPT-4', 'ChatGPT-3']\n",
    "hue2line = {\n",
    "    'Human': '--',\n",
    "    'ChatGPT-4': '-',\n",
    "    'ChatGPT-3': ':',\n",
    "}\n",
    "for hue in hues:\n",
    "    d_scores = data[data['hue'] == hue][[*dimensions]].agg(['median', 'mean', 'std'])\n",
    "    values = np.array(d_scores.loc[value_type])\n",
    "    errors = np.array(d_scores.loc['std']) * 2\n",
    "    print(values, errors)\n",
    "    values = np.concatenate((values, [values[0]]))  # Close the plot\n",
    "    errors = np.concatenate((errors, [errors[0]]))  # Close the plot\n",
    "\n",
    "    ax.fill_between(angles, values - errors, values + errors, alpha=0.2)\n",
    "    ax.plot(angles, values, linestyle=hue2line[hue], linewidth=3, label=hue)\n",
    " \n",
    "plt.xticks(angles[:-1], categories)\n",
    "# ax.set_rlabel_position(0)\n",
    "plt.ylim(0,50)\n",
    "plt.legend(loc='lower right', bbox_to_anchor=(1.4, 0.2))\n",
    "# plt.savefig('figures/cmp-bigfive.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig. 2: Turing Test\n",
    "\n",
    "Results may vary due to randomness in the simulation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simulate_Turing(samples_0, samples_1, n_bin=10, lim_a=0, lim_b=100, n_draw=100000):\n",
    "    hist_0 = np.histogram(samples_0, bins=n_bin, range=(lim_a, lim_b))[0] / len(samples_0)\n",
    "    n_wins = 0\n",
    "    n_ties = 0\n",
    "    for _ in tqdm(range(n_draw)):\n",
    "        try:\n",
    "            sample_0 = np.random.choice(samples_0)\n",
    "            sample_1 = np.random.choice(samples_1)\n",
    "            idx_0 = min(math.floor((sample_0 - lim_a) / (lim_b - lim_a) * n_bin), n_bin-1)\n",
    "            idx_1 = min(math.floor((sample_1 - lim_a) / (lim_b - lim_a) * n_bin), n_bin-1)\n",
    "            if hist_0[idx_1] > hist_0[idx_0]:\n",
    "                n_wins += 1\n",
    "            elif hist_0[idx_1] == hist_0[idx_0]:\n",
    "                n_ties += 1\n",
    "        except:\n",
    "            continue\n",
    "    return n_wins / n_draw, n_ties / n_draw, (n_draw - n_wins - n_ties) / n_draw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('human', simulate_Turing(df_dictator_human['choices'], df_dictator_human['choices']))\n",
    "print('gpt4', simulate_Turing(df_dictator_human['choices'], df_dictator_gpt4['choices']))\n",
    "print('turbo', simulate_Turing(df_dictator_human['choices'], df_dictator_turbo['choices']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('human', simulate_Turing(df_ultimatum_1_human['choices'], df_ultimatum_1_human['choices']))\n",
    "print('gpt4', simulate_Turing(df_ultimatum_1_human['choices'], df_ultimatum_1_gpt4['choices']))\n",
    "print('turbo', simulate_Turing(df_ultimatum_1_human['choices'], df_ultimatum_1_turbo['choices']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('human', simulate_Turing(df_ultimatum_2_human['choices'], df_ultimatum_2_human['choices']))\n",
    "print('gpt4', simulate_Turing(df_ultimatum_2_human['choices'], df_ultimatum_2_gpt4['choices']))\n",
    "print('turbo', simulate_Turing(df_ultimatum_2_human['choices'], df_ultimatum_2_turbo['choices']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('human', simulate_Turing(df_trust_1_human['choices'], df_trust_1_human['choices']))\n",
    "print('gpt4', simulate_Turing(df_trust_1_human['choices'], df_trust_1_gpt4['choices']))\n",
    "print('turbo', simulate_Turing(df_trust_1_human['choices'], df_trust_1_turbo['choices']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('human', simulate_Turing(df_trust_3_human['choices'], df_trust_3_human['choices'], lim_b=150))\n",
    "print('gpt4', simulate_Turing(df_trust_3_human['choices'], df_trust_3_gpt4['choices'], lim_b=150))\n",
    "print('turbo', simulate_Turing(df_trust_3_human['choices'], df_trust_3_turbo['choices'], lim_b=150))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('human', simulate_Turing(df_PG_human['choices'], df_PG_human['choices'], lim_b=20))\n",
    "print('gpt4', simulate_Turing(df_PG_human['choices'], df_PG_gpt4['choices'], lim_b=20))\n",
    "print('gpt4', simulate_Turing(df_PG_turbo['choices'], df_PG_turbo['choices'], lim_b=20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('human', simulate_Turing(prefix_to_choices_human[''], prefix_to_choices_human['']))\n",
    "print('gpt4', simulate_Turing(prefix_to_choices_human[''],prefix_to_choices_model['ChatGPT-4']['']))\n",
    "print('turbo', simulate_Turing(prefix_to_choices_human[''], prefix_to_choices_model['ChatGPT-3']['']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('human', r_coo_human * r_def_human, r_coo_human * r_coo_human + r_def_human * r_def_human, r_def_human * r_coo_human)\n",
    "print('gpt4', r_coo_human * r_def_gpt4, r_coo_human * r_coo_gpt4 + r_def_human * r_def_gpt4, r_def_human * r_coo_gpt4)\n",
    "print('turbo', r_coo_human * r_def_turbo, r_coo_human * r_coo_turbo + r_def_human * r_def_turbo, r_def_human * r_coo_turbo)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig. 3: Distributions"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dictator Game\n",
    "\n",
    "#### Human Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "binrange = (0, 100)\n",
    "moves = []\n",
    "with open('data/dictator.csv', 'r') as f:\n",
    "    reader = csv.reader(f)\n",
    "    header = next(reader)\n",
    "    col2idx = {col: idx for idx, col in enumerate(header)}\n",
    "    for row in reader:\n",
    "        record = {col: row[idx] for col, idx in col2idx.items()}\n",
    "\n",
    "        if record['Role'] != 'first': continue\n",
    "        if int(record['Round']) > 1: continue\n",
    "        if int(record['Total']) != 100: continue\n",
    "        if record['move'] == 'None': continue\n",
    "        if record['gameType'] != 'dictator': continue\n",
    "\n",
    "        move = float(record['move'])\n",
    "        if move < binrange[0] or \\\n",
    "            move > binrange[1]: continue\n",
    "        \n",
    "        moves.append(move)\n",
    "\n",
    "df_dictator_human = choices_to_df(moves, 'Human')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "choices = [50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0]\n",
    "df_dictator_gpt4 = choices_to_df(choices, hue=str('ChatGPT-4'))\n",
    "\n",
    "choices = [25, 35, 70, 30, 20, 25, 40, 80, 30, 30, 40, 30, 30, 30, 30, 30, 40, 40, 30, 30, 40, 30, 60, 20, 40, 25, 30, 30, 30]\n",
    "df_dictator_turbo = choices_to_df(choices, hue=str('ChatGPT-3'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(df_dictator_gpt4.dropna()))\n",
    "print(len(df_dictator_turbo.dropna()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = json.load(open('records/dictator_wo_ex_2023_03_13-11_24_07_PM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "print(', '.join([str(x) for x in choices]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for df_dictator_gpt4\n",
    "print('df_dictator_gpt4:')\n",
    "print('min:', df_dictator_gpt4['choices'].min())\n",
    "print('max:', df_dictator_gpt4['choices'].max())\n",
    "print('mean:', df_dictator_gpt4['choices'].mean())\n",
    "print('std:', df_dictator_gpt4['choices'].std())\n",
    "\n",
    "# for df_dictator_turbo\n",
    "print('df_dictator_turbo:')\n",
    "print('min:', df_dictator_turbo['choices'].min())\n",
    "print('max:', df_dictator_turbo['choices'].max())\n",
    "print('mean:', df_dictator_turbo['choices'].mean())\n",
    "print('std:', df_dictator_turbo['choices'].std())\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_facet(\n",
    "    df_list=[\n",
    "        df_dictator_human, \n",
    "        df_dictator_gpt4, \n",
    "        df_dictator_turbo\n",
    "    ],\n",
    "    binrange=(0, 100),\n",
    "    stat='density',\n",
    "    x_label='Split offered ($)',\n",
    "    kde=False,\n",
    ")\n",
    "# plt.savefig('figures/cmp-dictator.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ultimatum Game\n",
    "\n",
    "#### Human Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/ultimatum_strategy.csv')\n",
    "df = df[df['gameType'] == 'ultimatum_strategy']\n",
    "df = df[df['Role'] == 'player']\n",
    "df = df[df['Round'] == 1]\n",
    "df = df[df['Total'] == 100]\n",
    "df = df[df['move'] != 'None']\n",
    "df['propose'] = df['move'].apply(lambda x: eval(x)[0])\n",
    "df['accept'] = df['move'].apply(lambda x: eval(x)[1])\n",
    "df = df[(df['propose'] >= 0) & (df['propose'] <= 100)]\n",
    "df = df[(df['accept'] >= 0) & (df['accept'] <= 100)]\n",
    "# df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ultimatum_1_human = choices_to_df(list(df['propose']), 'Human')\n",
    "df_ultimatum_2_human = choices_to_df(list(df['accept']), 'Human')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "choices = [50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0]\n",
    "df_ultimatum_1_gpt4 = choices_to_df(choices, hue=str('ChatGPT-4'))\n",
    "\n",
    "choices = [40, 40, 40, 30, 70, 70, 50, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 30, 30, 35, 50, 40, 70, 40, 60, 60, 70, 40, 50]\n",
    "df_ultimatum_1_turbo = choices_to_df(choices, hue=str('ChatGPT-3'))\n",
    "\n",
    "choices = [50.0, 50.0, 50.0, 1.0, 1.0, 1.0, 50.0, 25.0, 50.0, 1.0, 1.0, 20.0, 50.0, 50.0, 50.0, 20.0, 50.0, 1.0, 1.0, 1.0, 50.0, 50.0, 50.0, 1.0, 1.0, 1.0, 20.0, 1.0] + [0, 1]\n",
    "df_ultimatum_2_gpt4 = choices_to_df(choices, hue=str('ChatGPT-4'))\n",
    "\n",
    "choices = [None, 50, 50, 50, 50, 30, None, None, 30, 33.33, 40, None, 50, 40, None, 1, 30, None, 10, 50, 30, 10, 30, None, 30, None, 10, 30, 30, 30]\n",
    "df_ultimatum_2_turbo = choices_to_df(choices, hue=str('ChatGPT-3'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(df_ultimatum_1_gpt4.dropna()))\n",
    "print(len(df_ultimatum_1_turbo.dropna()))\n",
    "\n",
    "print(len(df_ultimatum_2_gpt4.dropna()))\n",
    "print(len(df_ultimatum_2_turbo.dropna()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# min max mean std\n",
    "# for df_ultimatum_1_gpt4\n",
    "print('df_ultimatum_1_gpt4:')\n",
    "print('min:', df_ultimatum_1_gpt4['choices'].min())\n",
    "print('max:', df_ultimatum_1_gpt4['choices'].max())\n",
    "print('mean:', df_ultimatum_1_gpt4['choices'].mean())\n",
    "print('std:', df_ultimatum_1_gpt4['choices'].std())\n",
    "\n",
    "# for df_ultimatum_1_turbo\n",
    "print('df_ultimatum_1_turbo:')\n",
    "print('min:', df_ultimatum_1_turbo['choices'].min())\n",
    "print('max:', df_ultimatum_1_turbo['choices'].max())\n",
    "print('mean:', df_ultimatum_1_turbo['choices'].mean())\n",
    "print('std:', df_ultimatum_1_turbo['choices'].std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "choices = [50.0, 50.0, 10.0, 40.0, 20.0, 50.0, 1.0, 1.0, 50.0, 1.0, 50.0, 50.0, 20.0, 10.0, 50.0, 20.0, 1.0, 1.0, 50.0, 1.0, 20.0, 1.0, 50.0, 50.0, 20.0, 20.0, 50.0, 20.0, 1.0, 50.0]\n",
    "df_ultimatum_2_gpt4_female = choices_to_df(choices, hue='ChatGPT-4 Female')\n",
    "\n",
    "choices = [1.0, 1.0, 1.0, 20.0, 1.0, 1.0, 50.0, 1.0, 1.0, 50.0, 50.0, 50.0, 20.0, 20.0, 1.0, 50.0, 1.0, 1.0, 1.0, 50.0, 20.0, 1.0, 50.0, 20.0, 20.0, 10.0, 50.0, 1.0, 1.0, 1.0]\n",
    "df_ultimatum_2_gpt4_male = choices_to_df(choices, hue='ChatGPT-4 Male')\n",
    "\n",
    "choices = [40.0, 1.0, 1.0, 20.0, 1.0, 20.0, 50.0, 50.0, 1.0, 1.0, 1.0, 50.0, 1.0, 20.0, 50.0, 10.0, 50.0, 1.0, 1.0, 20.0, 1.0, 50.0, 20.0, 20.0, 20.0, 1.0, 1.0, 1.0, 1.0, 40.0]\n",
    "df_ultimatum_2_gpt4_US = choices_to_df(choices, hue='ChatGPT-4 US')\n",
    "\n",
    "choices = [1.0, 1.0, 20.0, 50.0, 1.0, 1.0, 1.0, 1.0, 20.0, 20.0, 50.0, 20.0, 20.0, 50.0, 20.0, 1.0, 40.0, 50.0, 1.0, 1.0, 1.0, 20.0, 1.0, 1.0, 50.0, 50.0, 1.0, 1.0, 1.0, 1.0]\n",
    "df_ultimatum_2_gpt4_Poland = choices_to_df(choices, hue='ChatGPT-4 Poland')\n",
    "\n",
    "choices = [50.0, 1.0, 20.0, 50.0, 50.0, 50.0, 50.0, 1.0, 1.0, 50.0, 1.0, 50.0, 1.0, 50.0, 1.0, 20.0, 1.0, 1.0, 20.0, 50.0, 0.0, 20.0, 1.0, 1.0, 1.0, 1.0, 20.0, 20.0, 50.0, 20.0]\n",
    "df_ultimatum_2_gpt4_China = choices_to_df(choices, hue='ChatGPT-4 China')\n",
    "\n",
    "choices = [1.0, 1.0, 1.0, 50.0, 1.0, 1.0, 50.0, 40.0, 1.0, 1.0, 1.0, 1.0, 20.0, 1.0, 1.0, 50.0, 1.0, 50.0, 1.0, 20.0, 1.0, 20.0, 1.0, 50.0, 1.0, 50.0, 20.0, 1.0, 1.0, 50.0]\n",
    "df_ultimatum_2_gpt4_UK = choices_to_df(choices, hue='ChatGPT-4 UK')\n",
    "\n",
    "choices = [50.0, 1.0, 20.0, 50.0, 50.0, 50.0, 50.0, 10.0, 1.0, 40.0, 50.0, 20.0, 1.0, 1.0, 1.0, 50.0, 50.0, 20.0, 20.0, 1.0, 1.0, 50.0, 20.0, 50.0, 50.0, 20.0, 1.0, 20.0, 50.0, 1]\n",
    "df_ultimatum_2_gpt4_Columbia = choices_to_df(choices, hue='ChatGPT-4 Columbia')\n",
    "\n",
    "choices = [50.0, 1.0, 50.0, 20.0, 20.0, 20.0, 50.0, 20.0, 20.0, 1.0, 1.0, 1.0, 1.0, 20.0, 1.0, 50.0, 1.0, 20.0, 20.0, 50.0, 1.0, 50.0, 1.0, 40.0, 1.0, 20.0, 1.0, 20.0, 1.0, 1.0]\n",
    "df_ultimatum_2_gpt4_under = choices_to_df(choices, hue='ChatGPT-4 Undergrad')\n",
    "\n",
    "choices = [1.0, 20.0, 1.0, 40.0, 50.0, 1.0, 1.0, 1.0, 25.0, 20.0, 50.0, 20.0, 50.0, 50.0, 1.0, 50.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 50.0, 20.0, 1.0, 1.0, 1.0, 50.0, 20.0, 20.0]\n",
    "df_ultimatum_2_gpt4_grad = choices_to_df(choices, hue='ChatGPT-4 Graduate')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_facet(\n",
    "    df_list=[\n",
    "        df_ultimatum_2_gpt4,\n",
    "        df_ultimatum_2_gpt4_female,\n",
    "        df_ultimatum_2_gpt4_male,\n",
    "        df_ultimatum_2_gpt4_US,\n",
    "        df_ultimatum_2_gpt4_Poland,\n",
    "        df_ultimatum_2_gpt4_China,\n",
    "        df_ultimatum_2_gpt4_UK,\n",
    "        df_ultimatum_2_gpt4_Columbia,\n",
    "        df_ultimatum_2_gpt4_under,\n",
    "        df_ultimatum_2_gpt4_grad,\n",
    "    ],\n",
    "    binrange=(0, 100),\n",
    "    # binwidth=10,\n",
    "    stat='density',\n",
    "    x_label='Minimum proposal to accept ($)',\n",
    ")\n",
    "# plt.savefig('figures/cmp-ultimatum-respond-demo.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_facet(\n",
    "    df_list=[\n",
    "        df_ultimatum_1_human,\n",
    "        df_ultimatum_1_gpt4,\n",
    "        df_ultimatum_1_turbo,\n",
    "    ],\n",
    "    binrange=(0, 100),\n",
    "    # binwidth=10,\n",
    "    stat='density',\n",
    "    x_label='Proposal to give ($)',\n",
    ")\n",
    "# plt.savefig('figures/cmp-ultimatum-proposal.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_facet(\n",
    "    df_list=[\n",
    "        df_ultimatum_2_human,\n",
    "        df_ultimatum_2_gpt4,\n",
    "        df_ultimatum_2_turbo,\n",
    "    ],\n",
    "    binrange=(0, 100),\n",
    "    # binwidth=10,\n",
    "    stat='density',\n",
    "    x_label='Minimum proposal to accept ($)',\n",
    ")\n",
    "# plt.savefig('figures/cmp-ultimatum-respond.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "choices = [50.0, 50.0, 50.0, 1.0, 1.0, 1.0, 50.0, 25.0, 50.0, 1.0, 1.0, 20.0, 50.0, 50.0, 50.0, 20.0, 50.0, 1.0, 1.0, 1.0, 50.0, 50.0, 50.0, 1.0, 1.0, 1.0, 20.0, 1.0] + [0, 1]\n",
    "df_ultimatum_2_gpt4 = choices_to_df(choices, hue=str('ChatGPT-4 (0314, 04.05)'))\n",
    "\n",
    "records = json.load(open('records/ultimatum_2_gpt4_2023_12_29-10_42_42_PM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "df_ultimatum_2_gpt4_1229 = choices_to_df(choices, hue=str('ChatGPT-4 (0314, 12.29)'))\n",
    "\n",
    "records = json.load(open('records/ultimatum_2_gpt4_1106_2023_12_29-11_15_35_PM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "df_ultimatum_2_gpt4_1106_1229 = choices_to_df(choices, hue=str('ChatGPT-4 (1106, 12.29)'))\n",
    "\n",
    "plot_facet(\n",
    "    df_list=[\n",
    "        df_ultimatum_2_gpt4,\n",
    "        df_ultimatum_2_gpt4_1229,\n",
    "        df_ultimatum_2_gpt4_1106_1229,\n",
    "    ],\n",
    "    binrange=(0, 100),\n",
    "    # binwidth=10,\n",
    "    stat='density',\n",
    "    x_label='Minimum proposal to accept ($)',\n",
    ")\n",
    "plt.savefig('figures/repro-ultimatum-respond.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "choices = [None, 50, 50, 50, 50, 30, None, None, 30, 33.33, 40, None, 50, 40, None, 1, 30, None, 10, 50, 30, 10, 30, None, 30, None, 10, 30, 30, 30]\n",
    "df_ultimatum_2_turbo = choices_to_df(choices, hue=str('ChatGPT-3 (0301, 03.13)'))\n",
    "\n",
    "records = json.load(open('records/ultimatum_2_turbo_2023_12_29-10_43_41_PM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "df_ultimatum_2_turbo_1229 = choices_to_df(choices, hue=str('ChatGPT-3 (0301, 12.29)'))\n",
    "\n",
    "records = json.load(open('records/ultimatum_2_turbo_1106_2023_12_29-11_57_30_PM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "df_ultimatum_2_turbo_1106_1229 = choices_to_df(choices, hue=str('ChatGPT-3 (1106, 12.29)'))\n",
    "\n",
    "plot_facet(\n",
    "    df_list=[\n",
    "        df_ultimatum_2_turbo,\n",
    "        df_ultimatum_2_turbo_1229,\n",
    "        df_ultimatum_2_turbo_1106_1229,\n",
    "    ],\n",
    "    binrange=(0, 100),\n",
    "    # binwidth=10,\n",
    "    stat='density',\n",
    "    x_label='Minimum proposal to accept ($)',\n",
    ")\n",
    "# plt.savefig('figures/cmp-ultimatum-respond.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trust Game\n",
    "\n",
    "#### Human Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "binrange = (0, 100)\n",
    "moves_1 = []\n",
    "moves_2 = defaultdict(list)\n",
    "with open('data/trust_investment.csv', 'r') as f:\n",
    "    reader = csv.reader(f)\n",
    "    header = next(reader)\n",
    "    col2idx = {col: idx for idx, col in enumerate(header)}\n",
    "    for row in reader:\n",
    "        record = {col: row[idx] for col, idx in col2idx.items()}\n",
    "\n",
    "        # if record['Role'] != 'first': continue\n",
    "        if int(record['Round']) > 1: continue\n",
    "        # if int(record['Total']) != 100: continue\n",
    "        if record['move'] == 'None': continue\n",
    "        if record['gameType'] != 'trust_investment': continue\n",
    "\n",
    "        if record['Role'] == 'first':\n",
    "            move = float(record['move'])\n",
    "            if move < binrange[0] or \\\n",
    "                move > binrange[1]: continue\n",
    "            moves_1.append(move)\n",
    "        elif record['Role'] == 'second':\n",
    "            inv, ret = eval(record['roundResult'])\n",
    "            if ret < 0 or \\\n",
    "                ret > inv * 3: continue\n",
    "            moves_2[inv].append(ret)\n",
    "        else: continue\n",
    "\n",
    "df_trust_1_human = choices_to_df(moves_1, 'Human')\n",
    "df_trust_2_human = choices_to_df(moves_2[10], 'Human')\n",
    "df_trust_3_human = choices_to_df(moves_2[50], 'Human')\n",
    "df_trust_4_human = choices_to_df(moves_2[100], 'Human')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "choices = [50.0, 50.0, 40.0, 30.0, 50.0, 50.0, 40.0, 50.0, 50.0, 50.0, 50.0, 50.0, 30.0, 30.0, 50.0, 50.0, 50.0, 40.0, 40.0, 50.0, 50.0, 50.0, 50.0, 40.0, 50.0, 50.0, 50.0, 50.0] \n",
    "df_trust_1_gpt4 = choices_to_df(choices, hue=str('ChatGPT-4'))\n",
    "\n",
    "choices = [50.0, 50.0, 30.0, 30.0, 30.0, 60.0, 50.0, 40.0, 20.0, 20.0, 50.0, 40.0, 30.0, 20.0, 30.0, 20.0, 30.0, 60.0, 50.0, 30.0, 50.0, 20.0, 20.0, 30.0, 50.0, 30.0, 30.0, 50.0, 40.0] + [30]\n",
    "df_trust_1_turbo = choices_to_df(choices, hue=str('ChatGPT-3'))\n",
    "\n",
    "choices = [20.0, 20.0, 20.0, 20.0, 15.0, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 15.0, 15.0, 20.0, 15.0, 15.0, 15.0, 15.0, 15.0, 20.0, 20.0, 15.0]\n",
    "df_trust_2_gpt4 = choices_to_df(choices, hue=str('ChatGPT-4'))\n",
    "\n",
    "choices = [20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 15.0, 25.0, 30.0, 30.0, 20.0, 25.0, 30.0, 20.0, 20.0, 18.0] + [20, 20, 20, 25, 25, 25, 30]\n",
    "df_trust_2_turbo = choices_to_df(choices, hue=str('ChatGPT-3'))\n",
    "\n",
    "choices = [100.0, 75.0, 75.0, 75.0, 75.0, 75.0, 100.0, 75.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 75.0, 100.0, 75.0, 75.0, 75.0, 100.0, 100.0, 100.0, 75.0, 100.0, 100.0, 100.0, 100.0, 75.0, 100.0, 75.0]\n",
    "df_trust_3_gpt4 = choices_to_df(choices, hue=str('ChatGPT-4'))\n",
    "\n",
    "choices = [150.0, 100.0, 150.0, 150.0, 50.0, 150.0, 100.0, 150.0, 100.0, 100.0, 100.0, 150.0] + [100, 100, 100, 100, 100, 100, 100, 100]\n",
    "df_trust_3_turbo = choices_to_df(choices, hue=str('ChatGPT-3'))\n",
    "\n",
    "choices = [200.0, 200.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 200.0, 200.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0, 150.0]\n",
    "df_trust_4_gpt4 = choices_to_df(choices, hue=str('ChatGPT-4'))\n",
    "\n",
    "choices = [225.0, 225.0, 300.0, 300.0, 220.0, 300.0, 250.0] + [200, 200, 250, 200, 200]\n",
    "df_trust_4_turbo = choices_to_df(choices, hue=str('ChatGPT-3'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(df_trust_1_gpt4.dropna()))\n",
    "print(len(df_trust_1_turbo.dropna()))\n",
    "\n",
    "print(len(df_trust_2_gpt4.dropna()))\n",
    "print(len(df_trust_2_turbo.dropna()))\n",
    "\n",
    "print(len(df_trust_3_gpt4.dropna()))\n",
    "print(len(df_trust_3_turbo.dropna()))\n",
    "\n",
    "print(len(df_trust_4_gpt4.dropna()))\n",
    "print(len(df_trust_4_turbo.dropna()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = json.load(open('records/trust_2_gpt4_2023_04_07-11_46_45_PM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "print(', '.join([str(c) for c in choices]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = json.load(open('records/trust_4_gpt4_2023_04_08-12_24_56_AM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "print(', '.join([str(c) for c in choices]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# min max mean std of df_trust_1_turbo\n",
    "print('df_trust_1_turbo:')\n",
    "print('min:', df_trust_1_turbo['choices'].min())\n",
    "print('max:', df_trust_1_turbo['choices'].max())\n",
    "print('mean:', df_trust_1_turbo['choices'].mean())\n",
    "print('std:', df_trust_1_turbo['choices'].std())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# min max mean std of df_trust_1_gpt4\n",
    "print('df_trust_1_gpt4:')\n",
    "print('min:', df_trust_1_gpt4['choices'].min())\n",
    "print('max:', df_trust_1_gpt4['choices'].max())\n",
    "print('mean:', df_trust_1_gpt4['choices'].mean())\n",
    "print('std:', df_trust_1_gpt4['choices'].std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# min max mean std of df_trust_4_turbo\n",
    "print('df_trust_4_turbo:')\n",
    "print('min:', df_trust_4_turbo['choices'].min())\n",
    "print('max:', df_trust_4_turbo['choices'].max())\n",
    "print('mean:', df_trust_4_turbo['choices'].mean())\n",
    "print('std:', df_trust_4_turbo['choices'].std())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_facet(\n",
    "    df_list=[\n",
    "        df_trust_1_human,\n",
    "        df_trust_1_gpt4,\n",
    "        df_trust_1_turbo,\n",
    "    ],\n",
    "    binrange=(0, 100),\n",
    "    # binwidth=10,\n",
    "    stat='density',\n",
    "    x_label='Investment ($)',\n",
    ")\n",
    "# plt.savefig('figures/cmp-trust-invest.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_facet(\n",
    "    df_list=[\n",
    "        df_trust_3_human,\n",
    "        df_trust_3_gpt4,\n",
    "        df_trust_3_turbo,\n",
    "    ],\n",
    "    binrange=(0, 150),\n",
    "    bins=15,\n",
    "    # binwidth=10,\n",
    "    stat='density',\n",
    "    x_label='Return to investor ($)',\n",
    "    xticks_locs=[0, 25, 50, 75, 100, 125, 150],\n",
    ")\n",
    "# plt.savefig('figures/cmp-trust-return.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Public Goods\n",
    "\n",
    "#### Human Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/public_goods_linear_water.csv')\n",
    "df = df[df['Role'] == 'contributor']\n",
    "df = df[df['Round'] <= 3]\n",
    "df = df[df['Total'] == 20]\n",
    "df = df[df['groupSize'] == 4]\n",
    "df = df[df['move'] != None]\n",
    "df = df[(df['move'] >= 0) & (df['move'] <= 20)]\n",
    "df = df[df['gameType'] == 'public_goods_linear_water']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "round_1 = df[df['Round'] == 1]['move']\n",
    "round_2 = df[df['Round'] == 2]['move']\n",
    "round_3 = df[df['Round'] == 3]['move']\n",
    "print(len(round_1), len(round_2), len(round_3))\n",
    "df_PG_human = pd.DataFrame({\n",
    "    'choices': list(round_1)\n",
    "})\n",
    "df_PG_human['hue'] = 'Human'\n",
    "df_PG_human"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Payoffs"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_names = [\n",
    "    # 'records/PG_basic_turbo_2023_05_09-02_49_09_AM.json',\n",
    "    # 'records/PG_basic_turbo_loss_2023_05_09-03_59_49_AM.json'\n",
    "    'records/PG_basic_gpt4_2023_05_09-11_15_42_PM.json',\n",
    "    'records/PG_basic_gpt4_loss_2023_05_09-10_44_38_PM.json',\n",
    "]\n",
    "\n",
    "choices = []\n",
    "for file_name in file_names:\n",
    "    with open(file_name, 'r') as f:\n",
    "        choices += json.load(f)['choices']\n",
    "choices_baseline = choices\n",
    "\n",
    "choices = [tuple(x)[0] for x in choices]\n",
    "# df_PG_turbo = choices_to_df(choices, hue=str('ChatGPT-3'))\n",
    "# df_PG_turbo.head()\n",
    "df_PG_gpt4 = choices_to_df(choices, hue=str('ChatGPT-4'))\n",
    "df_PG_gpt4.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(df_PG_gpt4.dropna()))\n",
    "print(len(df_PG_turbo.dropna()))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = plot_facet(\n",
    "    df_list=[\n",
    "        df_PG_human, \n",
    "        df_PG_gpt4, \n",
    "        df_PG_turbo\n",
    "    ],\n",
    "    binrange=(0, 20),\n",
    "    # binwidth=2,\n",
    "    stat='density',\n",
    "    x_label='Contribution ($)',\n",
    ")\n",
    "# g.set(ylim=(0, 0.5))\n",
    "# plt.savefig('figures/cmp-PG.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bomb Risk Game\n",
    "\n",
    "1 safe, 0 bomb\n",
    "\n",
    "#### Human Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/bomb_risk.csv')\n",
    "df = df[df['Role'] == 'player']\n",
    "df = df[df['gameType'] == 'bomb_risk']\n",
    "df.sort_values(by=['UserID', 'Round'])\n",
    "\n",
    "prefix_to_choices_human = defaultdict(list)\n",
    "prefix_to_IPW = defaultdict(list)\n",
    "prev_user = None\n",
    "prev_move = None\n",
    "prefix = ''\n",
    "bad_user = False\n",
    "for _, row in df.iterrows():\n",
    "    if bad_user: continue\n",
    "    if row['UserID'] != prev_user:\n",
    "        prev_user = row['UserID']\n",
    "        prefix = ''\n",
    "        bad_user = False\n",
    "\n",
    "    move = row['move']\n",
    "    if move < 0 or move > 100:\n",
    "        bad_users = True\n",
    "        continue\n",
    "    prefix_to_choices_human[prefix].append(move)\n",
    "\n",
    "    if len(prefix) == 0:\n",
    "        prefix_to_IPW[prefix].append(1)\n",
    "    elif prefix[-1] == '1':\n",
    "        prev_move = min(prev_move, 98)\n",
    "        prefix_to_IPW[prefix].append(1./(100 - prev_move))\n",
    "    elif prefix[-1] == '0':\n",
    "        prev_move = max(prev_move, 1)\n",
    "        prefix_to_IPW[prefix].append(1./(prev_move))\n",
    "    else: assert False\n",
    "    \n",
    "    prev_move = move\n",
    "\n",
    "    prefix += '1' if row['roundResult'] == 'SAFE' else '0'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix_to_choices_model = defaultdict(lambda : defaultdict(list))\n",
    "for model in ['ChatGPT-4', 'ChatGPT-3']:\n",
    "    if model == 'ChatGPT-4':\n",
    "        file_names = [\n",
    "            'bomb_gpt4_2023_05_15-12_13_51_AM.json'\n",
    "        ]\n",
    "    elif model == 'ChatGPT-3':\n",
    "        file_names = [\n",
    "            'bomb_turbo_2023_05_14-10_45_50_PM.json'\n",
    "        ]\n",
    "\n",
    "    choices = []\n",
    "    scenarios = []\n",
    "    for file_name in file_names:\n",
    "        with open(os.path.join('records', file_name), 'r') as f:\n",
    "            records = json.load(f)\n",
    "            choices += records['choices']\n",
    "            scenarios += records['scenarios']\n",
    "\n",
    "    assert len(scenarios) == len(choices)\n",
    "    print('loaded %i valid records' % len(scenarios))\n",
    "\n",
    "    prefix_to_choice = defaultdict(list)\n",
    "    prefix_to_result = defaultdict(list)\n",
    "    prefix_to_pattern = defaultdict(Counter)\n",
    "    wrong_sum = 0\n",
    "    for scenarios_tmp, choices_tmp in zip(scenarios, choices):\n",
    "\n",
    "        result = 0\n",
    "        for i, scenario in enumerate(scenarios_tmp):\n",
    "            prefix = tuple(scenarios_tmp[:i])\n",
    "            prefix = ''.join([str(x) for x in prefix])\n",
    "            choice = choices_tmp[i]\n",
    "            \n",
    "            prefix_to_choice[prefix].append(choice)\n",
    "            prefix_to_pattern[prefix][tuple(choices_tmp[:-1])] += 1\n",
    "\n",
    "            prefix = tuple(scenarios_tmp[:i+1])\n",
    "            if scenario == 1:\n",
    "                result += choice\n",
    "            prefix_to_result[prefix].append(result)\n",
    "\n",
    "    print('# of wrong sum:', wrong_sum)\n",
    "    print('# of correct sum:', len(scenarios) - wrong_sum)\n",
    "\n",
    "    prefix_to_choices_model[model] = prefix_to_choice"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix = ''\n",
    "df_bomb_human = choices_to_df(prefix_to_choices_human[prefix], hue='Human')\n",
    "df_bomb_human['weight'] = prefix_to_IPW[prefix]\n",
    "df_bomb_models = pd.concat([choices_to_df(\n",
    "        prefix_to_choices_model[model][prefix], hue=model\n",
    "    ) for model in prefix_to_choices_model]\n",
    ")\n",
    "df_bomb_models['weight'] = 1\n",
    "g = plot_facet(\n",
    "    df_list=[\n",
    "        df_bomb_human,\n",
    "        df_bomb_models,\n",
    "    ],\n",
    "    binrange=(0, 100),\n",
    "    # binwidth=10,\n",
    "    stat='density',\n",
    "    # x_label='# of boxes to open ({})'.format(prefix),\n",
    "    x_label='# of boxes opened',\n",
    "    weights='weight',\n",
    ")\n",
    "# g.set(ylim=(0, 0.1))\n",
    "# plt.savefig(f'figures/cmp-bomb-{prefix}.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prisoner Dilemma\n",
    "\n",
    "#### Human Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/push_pull.csv')\n",
    "df = df[df['gameType'] == 'push_pull']\n",
    "df = df[df['Role'] == 'player']\n",
    "df = df[(df['move'] == 0) | (df['move'] == 1)]\n",
    "# df = df[df['Round'] <= 2]\n",
    "df = df[df['groupSize'] == 2]\n",
    "\n",
    "counter = -1\n",
    "playIDs = []\n",
    "otherMoves = []\n",
    "for i, row in df.iterrows():\n",
    "    if row['Round'] == 1:\n",
    "        counter += 1\n",
    "    playIDs.append(counter)\n",
    "    roundResult = eval(row['roundResult'])\n",
    "    roundResult.remove(row['move'])\n",
    "    otherMoves.append(roundResult[0])\n",
    "df['playID'] = playIDs\n",
    "df['otherMove'] = otherMoves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### first round\n",
    "\n",
    "df[df['Round'] == 1]['move'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_coo_human = 36269\n",
    "n_def_human = 44114\n",
    "r_coo_human = n_coo_human / (n_coo_human + n_def_human)\n",
    "r_def_human = n_def_human / (n_coo_human + n_def_human)\n",
    "print(r_coo_human, r_def_human)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### two rounds\n",
    "\n",
    "playIDs_push = df[(\n",
    "    df['Round'] == 1) & (\n",
    "    df['otherMove'] == 0)]['playID'].unique()\n",
    "playIDs_pull = df[(\n",
    "    df['Round'] == 1) & (\n",
    "    df['otherMove'] == 1)]['playID'].unique()\n",
    "\n",
    "df_push = df[df['playID'].isin(playIDs_push)]\n",
    "df_pull = df[df['playID'].isin(playIDs_pull)]\n",
    "\n",
    "counter = Counter()\n",
    "for playID in tqdm(playIDs_pull):\n",
    "    df_tmp = df[df['playID'] == playID]\n",
    "    try:\n",
    "        move_1 = df_tmp[df_tmp['Round'] == 1]['move'].values[0]\n",
    "        move_2 = df_tmp[df_tmp['Round'] == 2]['move'].values[0]\n",
    "        counter[(move_1, move_2)] += 1\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "playIDs_five = set(df[(\n",
    "    df['Round'] == 1) & (\n",
    "    df['otherMove'] == 1)]['playID'].unique())\n",
    "playIDs_five = set(df[(\n",
    "    df['Round'] == 2) & (\n",
    "    df['otherMove'] == 1)]['playID'].unique()).intersection(playIDs_five)\n",
    "playIDs_five = set(df[(\n",
    "    df['Round'] == 3) & (\n",
    "    df['otherMove'] == 0)]['playID'].unique()).intersection(playIDs_five)\n",
    "playIDs_five = set(df[(\n",
    "    df['Round'] == 4) & (\n",
    "    df['otherMove'] == 0)]['playID'].unique()).intersection(playIDs_five)\n",
    "playIDs_five = set(df[(\n",
    "    df['Round'] == 5)]['playID'].unique()).intersection(playIDs_five)\n",
    "\n",
    "df_five = df[df['playID'].isin(playIDs_five)]\n",
    "print(len(playIDs_five))\n",
    "\n",
    "for i in range(1, 6):\n",
    "    n_push = len(df_five[(\n",
    "        df_five['Round'] == i) & (\n",
    "        df_five['move'] == 0)])\n",
    "    n_pull = len(df_five[(\n",
    "        df_five['Round'] == i) & (\n",
    "        df_five['move'] == 1)])\n",
    "    print('Round %i: %.2f push' % (i, n_push / (n_pull + n_push)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(counter)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# records = json.load(open('records/PD_gpt4_two_rounds_push_2023_05_10-10_04_33_PM.json', 'r'))\n",
    "records = json.load(open('records/PD_turbo_two_rounds_push_2023_05_08-06_03_40_PM.json', 'r'))\n",
    "# records = json.load(open('records/PD_gpt4_two_rounds_pull_2023_05_08-08_57_08_PM.json', 'r'))\n",
    "# records = json.load(open('records/PD_turbo_two_rounds_pull_2023_05_08-09_23_13_PM.json', 'r'))\n",
    "# records = json.load(open('records/PD_gpt4_five_rounds_pull_2023_05_11-05_17_36_PM.json', 'r'))\n",
    "# records = json.load(open('records/PD_turbo_five_rounds_pull_2023_05_11-08_03_28_PM.json', 'r'))\n",
    "# print(records['choices'])\n",
    "for i in range(len(records['choices'][0])-1):\n",
    "    choices = [tuple(x[i:i+2]) for x in records['choices']]\n",
    "    print(sorted(Counter(choices).items(), key=lambda item: item[0], reverse=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_coo_gpt4 = 29 + 0 + 0 + 26\n",
    "n_def_gpt4 = 0 + 1 + 1 + 3\n",
    "n_coo_turbo = 21 + 3 + 7 + 15\n",
    "n_def_turbo = 3 + 3 + 4 + 4\n",
    "r_coo_gpt4 = n_coo_gpt4 / (n_coo_gpt4 + n_def_gpt4)\n",
    "r_def_gpt4 = n_def_gpt4 / (n_coo_gpt4 + n_def_gpt4)\n",
    "r_coo_turbo = n_coo_turbo / (n_coo_turbo + n_def_turbo)\n",
    "r_def_turbo = n_def_turbo / (n_coo_turbo + n_def_turbo)\n",
    "print(r_coo_gpt4, r_def_gpt4)\n",
    "print(r_coo_turbo, r_def_turbo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_names = [\n",
    "    'records/PD_gpt4_two_rounds_push_2023_05_10-10_04_33_PM.json',\n",
    "    'records/PD_gpt4_two_rounds_pull_2023_05_08-08_57_08_PM.json'\n",
    "]\n",
    "\n",
    "choices = []\n",
    "for file_name in file_names:\n",
    "    with open(file_name, 'r') as f:\n",
    "        choices += json.load(f)['choices']\n",
    "choices_baseline = choices\n",
    "\n",
    "choices = [tuple(x)[0] for x in choices]\n",
    "df_PG_gpt4 = choices_to_df(choices, hue=str('ChatGPT-4'))\n",
    "df_PG_gpt4.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_names = [\n",
    "    'records/PD_turbo_two_rounds_push_2023_05_08-06_03_40_PM.json',\n",
    "    'records/PD_turbo_two_rounds_pull_2023_05_08-09_23_13_PM.json',\n",
    "]\n",
    "\n",
    "choices = []\n",
    "for file_name in file_names:\n",
    "    with open(file_name, 'r') as f:\n",
    "        choices += json.load(f)['choices']\n",
    "choices_baseline = choices\n",
    "\n",
    "choices = [tuple(x)[0] for x in choices]\n",
    "df_PG_turbo = choices_to_df(choices, hue=str('ChatGPT-3'))\n",
    "df_PG_turbo.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing libraries\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(16, 8), subplot_kw=dict(aspect=\"equal\"))\n",
    "  \n",
    "# declaring data\n",
    "# data = [46.87, 100-46.87] # human\n",
    "# data = [96.67+86.67, 200-96.67-86.67] # gpt4\n",
    "data = [80+73.33, 200-80-73.33] # turbo\n",
    "keys = ['Cooperate', 'Defect']\n",
    "  \n",
    "# define Seaborn color palette to use\n",
    "palette_color = [green, red]\n",
    "  \n",
    "# plotting data on chart\n",
    "wedges, texts, autotexts = ax.pie(\n",
    "    data, \n",
    "    # labels=keys, \n",
    "    # labeldistance=0.2,\n",
    "    # pctdistance=0.5,\n",
    "    colors=palette_color, \n",
    "    textprops=dict(color=\"w\"),\n",
    "    # hatch=['+', 'o'], \n",
    "    autopct='%.2f%%',\n",
    "    startangle=90\n",
    ")\n",
    "\n",
    "# ax.legend(\n",
    "#     wedges, keys,\n",
    "#     # title=\"Ingredients\",\n",
    "#     loc=\"upper center\",\n",
    "#     fontsize=18,\n",
    "#     bbox_to_anchor=(0.5, 1.1, 0, 0)\n",
    "# )\n",
    "  \n",
    "# displaying chart\n",
    "plt.setp(texts, size=24, weight=\"bold\")\n",
    "plt.setp(autotexts, size=24, weight=\"bold\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig. 6: Payoff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = 1.\n",
    "def CES(b, S, P, r=r):\n",
    "    S = np.array(S)\n",
    "    P = np.array(P)\n",
    "    b = np.array(b)\n",
    "    return (\n",
    "        b     * S**r + \\\n",
    "        (1.-b) * P**r\n",
    "    )**(1./r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Games\n",
    "\n",
    "#### Dictator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expected_payoff(propose):\n",
    "    payoff_player = 100 - propose\n",
    "    payoff_partner = propose\n",
    "    payoff_total = 100\n",
    "    return payoff_player, payoff_total, payoff_partner\n",
    "\n",
    "payoff_player, payoff_total, payoff_partner = zip(*[\n",
    "    expected_payoff(p) for p in range(0, 101)\n",
    "])\n",
    "\n",
    "def player_average(player, choices):\n",
    "    # remove None and NaN in choices\n",
    "    choices = [c for c in choices if c == c]\n",
    "    print('player', 'partner', 'combined')\n",
    "    payoff_player_mean = np.mean([payoff_player[int(c)] for c in choices])\n",
    "    payoff_total_mean = np.mean([payoff_total[int(c)] for c in choices])\n",
    "    payoff_partner_mean = np.mean([payoff_partner[int(c)] for c in choices])\n",
    "    print(player, 'mean', payoff_player_mean, payoff_partner_mean, payoff_total_mean)\n",
    "    payoff_player_std = np.std([payoff_player[int(c)] for c in choices])\n",
    "    payoff_total_std = np.std([payoff_total[int(c)] for c in choices])\n",
    "    payoff_partner_std = np.std([payoff_partner[int(c)] for c in choices])\n",
    "    print(player, 'std', payoff_player_std, payoff_partner_std, payoff_total_std)\n",
    "    payoff_player_samples = [payoff_player[int(c)] for c in choices]\n",
    "    payoff_partner_samples = [payoff_partner[int(c)] for c in choices]\n",
    "    payoff_player_se = np.std([payoff_player[int(c)] for c in choices]) / np.sqrt(len(choices))\n",
    "    payoff_total_se = np.std([payoff_total[int(c)] for c in choices]) / np.sqrt(len(choices))\n",
    "    payoff_partner_se = np.std([payoff_partner[int(c)] for c in choices]) / np.sqrt(len(choices))\n",
    "    print(player, 'se', payoff_player_se, payoff_partner_se, payoff_total_se)\n",
    "    return payoff_player_samples, payoff_partner_samples\n",
    "payoff_player_samples_dictator_human, payoff_partner_samples_dictator_human = player_average('human', df_dictator_human['choices'])\n",
    "payoff_player_samples_dictator_gpt4, payoff_partner_samples_dictator_gpt4 = player_average('gpt4', df_dictator_gpt4['choices'])\n",
    "payoff_player_samples_dictator_turbo, payoff_partner_samples_dictator_turbo = player_average('turbo', df_dictator_turbo['choices'])\n",
    "\n",
    "plt.plot(payoff_player, label='Selfish payoff (b=1)', color='red')\n",
    "plt.plot(payoff_partner, label='Selfless payoff (b=0)', color='green')\n",
    "plt.plot(CES(.5, payoff_player, payoff_partner, r=1.), label='Utility (b=1/2, r=1)', color='blue')\n",
    "plt.plot(CES(.5, payoff_player, payoff_partner, r=.5), label='Utility (b=1/2, r=1/2)', color='gold')\n",
    "plt.xlabel('Propose to give ($)')\n",
    "plt.ylabel('Expected payoff ($)')\n",
    "plt.legend(loc='lower center')\n",
    "# plt.savefig('figures/payoff-dictator.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "payoff_player_dictator, payoff_partner_dictator = payoff_player, payoff_partner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ultimatum\n",
    "\n",
    "Before running the code, run the code to load human data first (Fig. 3 - Ultimatum Game - Human Data)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expected_payoff(propose):\n",
    "    accept_ratio = 1. * len(df[df['accept'] <= propose]) / len(df)\n",
    "    payoff_player = (100 - propose) * accept_ratio\n",
    "    payoff_total = 100 * accept_ratio\n",
    "    payoff_partner = propose * accept_ratio\n",
    "    return payoff_player, payoff_total, payoff_partner\n",
    "\n",
    "payoff_player, payoff_total, payoff_partner = zip(*[\n",
    "    expected_payoff(p) for p in range(0, 101)\n",
    "])\n",
    "\n",
    "print(payoff_player.index(max(payoff_player)), max(payoff_player))\n",
    "# print(payoff_player.index(max(payoff_total)), max(payoff_total))\n",
    "# print(payoff_player.index(max(payoff_partner)), max(payoff_partner))\n",
    "\n",
    "payoff_player_samples_ultimatum_1_human, payoff_partner_samples_ultimatum_1_human = player_average('human', df_ultimatum_1_human['choices'])\n",
    "payoff_player_samples_ultimatum_1_gpt4, payoff_partner_samples_ultimatum_1_gpt4 = player_average('gpt4', df_ultimatum_1_gpt4['choices'])\n",
    "payoff_player_samples_ultimatum_1_turbo, payoff_partner_samples_ultimatum_1_turbo = player_average('turbo', df_ultimatum_1_turbo['choices'])\n",
    "\n",
    "plt.plot(payoff_player, label='Selfish payoff (b=1)', color='red')\n",
    "plt.plot(payoff_partner, label='Selfless payoff (b=0)', color='green')\n",
    "plt.plot(CES(.5, payoff_player, payoff_partner, r=1.), label='Utility (b=1/2, r=1)', color='blue')\n",
    "plt.plot(CES(.5, payoff_player, payoff_partner, r=.5), label='Utility (b=1/2, r=1/2)', color='gold')\n",
    "plt.xlabel('Propose to give ($)')\n",
    "plt.ylabel('Expected payoff ($)')\n",
    "plt.legend(loc='upper left')\n",
    "# plt.savefig('figures/payoff-ultimatum-1.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "payoff_player_ultimatum_1, payoff_partner_ultimatum_1 = payoff_player, payoff_partner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expected_payoff(accept):\n",
    "    df_accepted = df[df['propose'] >= accept]\n",
    "    propose_mean = df_accepted.mean(numeric_only=True)['propose']\n",
    "    accept_ratio = 1. * len(df_accepted) / len(df)\n",
    "    payoff_player =  propose_mean * accept_ratio\n",
    "    payoff_total = 100 * accept_ratio\n",
    "    payoff_partner = (100 - propose_mean) * accept_ratio\n",
    "    return payoff_player, payoff_total, payoff_partner\n",
    "\n",
    "payoff_player, payoff_total, payoff_partner = zip(*[\n",
    "    expected_payoff(p) for p in range(0, 101)\n",
    "])\n",
    "\n",
    "print(payoff_player.index(max(payoff_player)), max(payoff_player))\n",
    "print(payoff_total.index(max(payoff_total)), max(payoff_total))\n",
    "print(payoff_partner.index(max(payoff_partner)), max(payoff_partner))\n",
    "\n",
    "payoff_player_samples_ultimatum_2_human, payoff_partner_samples_ultimatum_2_human = player_average('human', df_ultimatum_2_human['choices'])\n",
    "payoff_player_samples_ultimatum_2_gpt4, payoff_partner_samples_ultimatum_2_gpt4 = player_average('gpt4', df_ultimatum_2_gpt4['choices'])\n",
    "payoff_player_samples_ultimatum_2_turbo, payoff_partner_samples_ultimatum_2_turbo = player_average('turbo', df_ultimatum_2_turbo['choices'])\n",
    "\n",
    "plt.plot(payoff_player, label='Selfish payoff (b=1)', color='red')\n",
    "plt.plot(payoff_partner, label='Selfless payoff (b=0)', color='green')\n",
    "plt.plot(CES(.5, payoff_player, payoff_partner, r=1.), label='Utility (b=1/2, r=1)', color='blue')\n",
    "plt.plot(CES(.5, payoff_player, payoff_partner, r=.5), label='Utility (b=1/2, r=1/2)', color='gold')\n",
    "plt.xlabel('Respond to accept ($)')\n",
    "plt.ylabel('Expected payoff ($)')\n",
    "plt.legend(loc='upper right')\n",
    "# plt.savefig('figures/payoff-ultimatum-2.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "payoff_player_ultimatum_2, payoff_partner_ultimatum_2 = payoff_player, payoff_partner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trust"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expected_payoff(invest):\n",
    "    # returns = moves_2[invest]\n",
    "    returns = []\n",
    "    binwidth = 0\n",
    "    for inv in range(\n",
    "        max(0, invest - binwidth), \n",
    "        min(100, invest + binwidth)+1\n",
    "    ):\n",
    "        if inv > 0:\n",
    "            ratios = [ret / (inv*3) for ret in moves_2[inv]]\n",
    "        else:\n",
    "            ratios = [0] * len(moves_2[inv])\n",
    "        values = [r * invest * 3 for r in ratios]\n",
    "        returns.extend(values)\n",
    "    return_mean = sum(returns) / len(returns)\n",
    "    payoff_player = return_mean - invest + 100\n",
    "    payoff_total = invest * 3 - invest + 100\n",
    "    payoff_partner = invest * 3 - return_mean\n",
    "    return payoff_player, payoff_total, payoff_partner\n",
    "\n",
    "payoff_player, payoff_total, payoff_partner = zip(*[\n",
    "    expected_payoff(p) for p in range(0, 101)\n",
    "])\n",
    "\n",
    "print(payoff_player.index(max(payoff_player)), max(payoff_player))\n",
    "print(payoff_total.index(max(payoff_total)), max(payoff_total))\n",
    "print(payoff_partner.index(max(payoff_partner)), max(payoff_partner))\n",
    "\n",
    "payoff_player_samples_trust_1_human, payoff_partner_samples_trust_1_human = player_average('human', df_trust_1_human['choices'])\n",
    "payoff_player_samples_trust_1_gpt4, payoff_partner_samples_trust_1_gpt4 = player_average('gpt4', df_trust_1_gpt4['choices'])\n",
    "payoff_player_samples_trust_1_turbo, payoff_partner_samples_trust_1_turbo = player_average('turbo', df_trust_1_turbo['choices'])\n",
    "\n",
    "plt.plot(payoff_player, label='Selfish payoff (b=1)', color='red')\n",
    "plt.plot(payoff_partner, label='Selfless payoff (b=0)', color='green')\n",
    "plt.plot(CES(.5, payoff_player, payoff_partner, r=1.), label='Utility (b=1/2, r=1)', color='blue')\n",
    "plt.plot(CES(.5, payoff_player, payoff_partner, r=.5), label='Utility (b=1/2, r=1/2)', color='gold')\n",
    "plt.xlabel('$ to invest')\n",
    "plt.ylabel('Expected payoff ($)')\n",
    "plt.legend(loc='upper left')\n",
    "# plt.savefig('figures/payoff-trust-1.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "payoff_player_trust_1, payoff_partner_trust_1 = payoff_player, payoff_partner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expected_payoff(ret):\n",
    "    payoff_player = 150 - ret\n",
    "    payoff_total = 100 + 100\n",
    "    payoff_partner = ret - 50 + 100\n",
    "    return payoff_player, payoff_total, payoff_partner\n",
    "\n",
    "# x = np.linspace(0, 1, 101)\n",
    "x = list(range(0, 151))\n",
    "payoff_player, payoff_total, payoff_partner = zip(*[\n",
    "    expected_payoff(p) for p in x\n",
    "])\n",
    "\n",
    "print(payoff_player.index(max(payoff_player)), max(payoff_player))\n",
    "print(payoff_total.index(max(payoff_total)), max(payoff_total))\n",
    "print(payoff_partner.index(max(payoff_partner)), max(payoff_partner))\n",
    "\n",
    "payoff_player_samples_trust_3_human, payoff_partner_samples_trust_3_human = player_average('human', df_trust_3_human['choices'])\n",
    "payoff_player_samples_trust_3_gpt4, payoff_partner_samples_trust_3_gpt4 = player_average('gpt4', df_trust_3_gpt4['choices'])\n",
    "payoff_player_samples_trust_3_turbo, payoff_partner_samples_trust_3_turbo = player_average('turbo', df_trust_3_turbo['choices'])\n",
    "\n",
    "plt.plot(payoff_player, label='Selfish payoff (b=1)', color='red')\n",
    "plt.plot(payoff_partner, label='Selfless payoff (b=0)', color='green')\n",
    "plt.plot(CES(.5, payoff_player, payoff_partner, r=1.), label='Utility (b=1/2, r=1)', color='blue')\n",
    "plt.plot(CES(.5, payoff_player, payoff_partner, r=.5), label='Utility (b=1/2, r=1/2)', color='gold')\n",
    "plt.xlabel('$ to return')\n",
    "plt.ylabel('Expected payoff ($)')\n",
    "plt.legend(loc='upper left')\n",
    "# plt.savefig('figures/payoff-trust-3.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "payoff_player_trust_3, payoff_partner_trust_3 = payoff_player, payoff_partner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Public Goods\n",
    "\n",
    "Before running the code, run the code to load human data first (Fig. 3 - Public Goods - Human Data)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "other_contri = df[df['Round'] == 1]['move'].mean()\n",
    "def expected_payoff(contri):\n",
    "    total_contri = contri + other_contri * 3\n",
    "    payoff_player = 0.5 * total_contri - contri\n",
    "    payoff_total = 0.5 * total_contri * 4 - total_contri\n",
    "    payoff_partner = 0.5 * total_contri - other_contri\n",
    "    return payoff_player, payoff_total, payoff_partner\n",
    "\n",
    "payoff_player, payoff_total, payoff_partner = zip(*[\n",
    "    expected_payoff(p) for p in range(0, 21)\n",
    "])\n",
    "\n",
    "print(payoff_player.index(max(payoff_player)), max(payoff_player))\n",
    "print(payoff_total.index(max(payoff_total)), max(payoff_total))\n",
    "print(payoff_partner.index(max(payoff_partner)), max(payoff_partner))\n",
    "\n",
    "payoff_player_samples_PG_human, payoff_partner_samples_PG_human = player_average('human', df_PG_human['choices'])\n",
    "payoff_player_samples_PG_gpt4, payoff_partner_samples_PG_gpt4 = player_average('gpt4', df_PG_gpt4['choices'])\n",
    "payoff_player_samples_PG_turbo, payoff_partner_samples_PG_turbo = player_average('turbo', df_PG_turbo['choices'])\n",
    "\n",
    "plt.plot(payoff_player, label='Selfish payoff (b=1)', color='red')\n",
    "plt.plot(payoff_partner, label='Selfless payoff (b=0)', color='green')\n",
    "plt.plot(CES(.5, payoff_player, payoff_partner, r=1.), label='Utility (b=1/2, r=1)', color='blue')\n",
    "plt.plot(CES(.5, payoff_player, payoff_partner, r=.5), label='Utility (b=1/2, r=1/2)', color='gold')\n",
    "plt.xlabel('$ to contribute')\n",
    "plt.ylabel('Expected payoff ($)')\n",
    "plt.legend(loc='lower center')\n",
    "# plt.savefig('figures/payoff-PG.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "payoff_player_PG, payoff_partner_PG = payoff_player, payoff_partner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prisoner Dilemma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_coo = r_coo_human\n",
    "r_def = r_def_human\n",
    "\n",
    "def expected_payoff(coo_prob):\n",
    "    payoff_player = \\\n",
    "        coo_prob * r_coo * 400 + \\\n",
    "        coo_prob * r_def * 000 + \\\n",
    "        (1 - coo_prob) * r_coo * 700 + \\\n",
    "        (1 - coo_prob) * r_def * 300\n",
    "    payoff_partner = \\\n",
    "        coo_prob * r_coo * 400 + \\\n",
    "        coo_prob * r_def * 700 + \\\n",
    "        (1 - coo_prob) * r_coo * 000 + \\\n",
    "        (1 - coo_prob) * r_def * 300\n",
    "    payoff_combined = payoff_player + payoff_partner\n",
    "    return payoff_player, payoff_combined, payoff_partner\n",
    "\n",
    "payoff_player, payoff_total, payoff_partner = zip(*[\n",
    "    expected_payoff(p) for p in np.linspace(0, 1, 2)\n",
    "])\n",
    "\n",
    "payoff_player_PD = payoff_player\n",
    "payoff_partner_PD = payoff_partner\n",
    "payoff_player_samples_PD_human, payoff_partner_samples_PD_human = player_average('human', [1]*n_coo_human + [0]*n_def_human)\n",
    "payoff_player_samples_PD_gpt4, payoff_partner_samples_PD_gpt4 = player_average('gpt4', [1]*n_coo_gpt4 + [0]*n_def_gpt4)\n",
    "payoff_player_samples_PD_turbo, payoff_partner_samples_PD_turbo = player_average('turbo', [1]*n_coo_turbo + [0]*n_def_turbo)\n",
    "\n",
    "plt.plot(payoff_player, label='Player payoff', color='red')\n",
    "plt.plot(payoff_total, label='Total payoff', color='blue')\n",
    "plt.plot(payoff_partner, label='Partner payoff', color='green')\n",
    "plt.xlabel('Prabability to cooperate (%)')\n",
    "plt.ylabel('Expected payoff ($)')\n",
    "plt.legend(loc='upper right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MSE of Objective Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# r = 1.\n",
    "r = .5\n",
    "SE_dict = {}\n",
    "# games = ['dictator', 'ultimatum_1', 'ultimatum_2', 'trust_1', 'trust_3', 'PG', 'PD']\n",
    "games = ['PD']\n",
    "for model in ['human', 'gpt4', 'turbo']:\n",
    "    SE_dict[model] = {}\n",
    "    for i, b in enumerate(np.linspace(0, 1, 11)):\n",
    "        SE_dict[model][i] = {}\n",
    "        for game in games:\n",
    "            SE_dict[model][i][game] = []\n",
    "            \n",
    "            payoff_curve = CES(b, eval('payoff_player_' + game), eval('payoff_partner_' + game), r=r)\n",
    "            theoretical_max = max(payoff_curve)\n",
    "            theoretical_min = min(payoff_curve)\n",
    "\n",
    "            if game != 'PD':\n",
    "                payoff_player_samples = eval('payoff_player_samples_' + game + '_' + model)\n",
    "                payoff_partner_samples = eval('payoff_partner_samples_' + game + '_' + model)\n",
    "            else:\n",
    "                r_coo = r_coo_human\n",
    "                r_def = r_def_human\n",
    "                if model == 'human':\n",
    "                    n_coo = n_coo_human\n",
    "                    n_def = n_def_human\n",
    "                elif model == 'gpt4':\n",
    "                    n_coo = n_coo_gpt4\n",
    "                    n_def = n_def_gpt4\n",
    "                elif model == 'turbo':\n",
    "                    n_coo = n_coo_turbo\n",
    "                    n_def = n_def_turbo\n",
    "                payoff_player_samples = [r_coo * 400 + r_def * 0] * n_coo + [r_coo * 700 + r_def * 300] * n_def\n",
    "                payoff_partner_samples = [r_coo * 400 + r_def * 700] * n_coo + [r_coo * 0 + r_def * 300] * n_def\n",
    "\n",
    "            actual_val = (b * np.array(payoff_player_samples)**r + (1-b) * np.array(payoff_partner_samples)**r)**(1/r)\n",
    "            SE = np.square(1 - np.array(actual_val) / theoretical_max)\n",
    "            SE_dict[model][i][game] += SE.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SE_df = pd.DataFrame(columns=['model', 'b', 'SE', 'game'])\n",
    "for model in SE_dict:\n",
    "    for i, b in enumerate(np.linspace(0, 1, 11)):\n",
    "        for game in SE_dict[model][i]:\n",
    "            ls = SE_dict[model][i][game]\n",
    "            SE_df = pd.concat([\n",
    "                SE_df,\n",
    "                pd.DataFrame({\n",
    "                    'model': [model] * len(ls),\n",
    "                    'b': [b] * len(ls),\n",
    "                    'SE': ls,\n",
    "                    'game': game,\n",
    "                })\n",
    "            ], ignore_index=True)\n",
    "SE_df = SE_df.dropna()\n",
    "SE_df.loc[SE_df['model'] == 'human', 'model'] = 'Human'\n",
    "SE_df.loc[SE_df['model'] == 'gpt4', 'model'] = 'ChatGPT-4'\n",
    "SE_df.loc[SE_df['model'] == 'turbo', 'model'] = 'ChatGPT-3'\n",
    "SE_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plt.figure(figsize=(8, 4))\n",
    "ax = sns.lineplot(\n",
    "    data=SE_df,\n",
    "    x='b',\n",
    "    y='SE',\n",
    "    hue='model',\n",
    "    palette=[blue, orange, green],\n",
    "    linewidth=2,\n",
    "    legend=True,\n",
    "    # errorbar='se'\n",
    "    errorbar=('ci', 95),\n",
    ")\n",
    "\n",
    "# use SE_df to annotate SE values of all models and all b\n",
    "if True:\n",
    "    for model in ['Human', 'ChatGPT-4', 'ChatGPT-3']: # SE_dict:\n",
    "        for i, b in enumerate(np.linspace(0, 1, 11)):\n",
    "            plt.annotate(\n",
    "                text='{:.2f}'.format(np.mean(SE_df[(SE_df['model'] == model) & (SE_df['b'] == b)]['SE'])),\n",
    "                xy=(b, np.mean(SE_df[(SE_df['model'] == model) & (SE_df['b'] == b)]['SE'])),\n",
    "                xytext=(b, np.mean(SE_df[(SE_df['model'] == model) & (SE_df['b'] == b)]['SE']) + 0.01),\n",
    "                ha='center',\n",
    "                va='bottom',\n",
    "                fontsize=8,\n",
    "                color={\n",
    "                    'Human': blue,\n",
    "                    'ChatGPT-4': orange,\n",
    "                    'ChatGPT-3': green,\n",
    "                }[model],\n",
    "            )\n",
    "\n",
    "ax.legend(title='')\n",
    "# plt.ylim(0, 0.4)\n",
    "plt.xticks(np.arange(0, 1.1, 0.1))\n",
    "plt.xlabel('b')\n",
    "plt.ylabel('Optimization squared error')\n",
    "# plt.savefig('figures/payoff_b_SE_CI.pdf', format='pdf', bbox_inches='tight')\n",
    "# plt.savefig('figures/payoff_b_SE_CI_annotated.pdf', format='pdf', bbox_inches='tight')\n",
    "# plt.savefig('figures/payoff_b_SE_CI_{}.pdf'.format(games[0]), format='pdf', bbox_inches='tight')\n",
    "\n",
    "# plt.savefig('figures/payoff_b_SE_CI_CES.pdf', format='pdf', bbox_inches='tight')\n",
    "# plt.savefig('figures/payoff_b_SE_CI_CES_annotated.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.savefig('figures/payoff_b_SE_CI_CES_{}.pdf'.format(games[0]), format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.stats as stats\n",
    "import statsmodels.api as sm\n",
    "\n",
    "df_plot = pd.DataFrame(columns=['model', 'b', 'MSE', 'clustered_SE', 'corrected_CI'])\n",
    "for model in SE_dict:\n",
    "    for i, b in enumerate(np.linspace(0, 1, 11)):\n",
    "        df = pd.DataFrame(columns=['model', 'b', 'SE', 'game'])\n",
    "        for game in SE_dict[model][i]:\n",
    "            ls = SE_dict[model][i][game]\n",
    "            df = pd.concat([\n",
    "                df,\n",
    "                pd.DataFrame({\n",
    "                    'model': [model] * len(ls),\n",
    "                    'b': [b] * len(ls),\n",
    "                    'SE': ls,\n",
    "                    'game': game,\n",
    "                })\n",
    "            ], ignore_index=True)\n",
    "        \n",
    "        Y = df['SE'].values\n",
    "        X = np.ones(len(Y))\n",
    "        clusters = df['game'].values\n",
    "\n",
    "        ols_model = sm.OLS(Y, X)\n",
    "        ols_results = ols_model.fit()\n",
    "        clustered_results = ols_results.get_robustcov_results(cov_type='cluster', groups=clusters)\n",
    "        MSE = clustered_results.predict(1)[0]\n",
    "        clustered_SE = clustered_results.bse[0]\n",
    "        corrected_CI = stats.norm.interval(0.95, loc=MSE, scale=clustered_SE)\n",
    "\n",
    "        df_plot = pd.concat([\n",
    "            df_plot,\n",
    "            pd.DataFrame({\n",
    "                'model': [model],\n",
    "                'b': [b],\n",
    "                'MSE': [MSE],\n",
    "                'clustered_SE': [clustered_SE],\n",
    "                'corrected_CI': [corrected_CI],\n",
    "            })\n",
    "        ], ignore_index=True)\n",
    "\n",
    "df_plot.loc[df_plot['model'] == 'human', 'model'] = 'Human'\n",
    "df_plot.loc[df_plot['model'] == 'gpt4', 'model'] = 'ChatGPT-4'\n",
    "df_plot.loc[df_plot['model'] == 'turbo', 'model'] = 'ChatGPT-3'\n",
    "df_plot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multinomial Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.optimize import minimize, minimize_scalar\n",
    "from scipy.misc import derivative\n",
    "from scipy.stats import norm\n",
    "\n",
    "\n",
    "def estimate_beta(\n",
    "    S, P, k, \n",
    "    use_scalar_minimization=True,\n",
    "    normalize=False,\n",
    "    r=1.\n",
    "):\n",
    "\n",
    "    S = np.array(S).reshape(1, -1)  # Reshape S to be a column vector\n",
    "    P = np.array(P).reshape(1, -1)  # Reshape P to be a column vector\n",
    "\n",
    "    if normalize:\n",
    "        # standard normalize S and P\n",
    "        U_mean = np.mean([S, P])\n",
    "        U_std = np.std([S, P])\n",
    "        S = (S - U_mean) / U_std\n",
    "        P = (P - U_mean) / U_std\n",
    "\n",
    "    def multinomial_log_likelihood(beta, S, P, k):\n",
    "        \"\"\"\n",
    "        Custom likelihood function for multinomial logistic regression with a single beta.\n",
    "        \"\"\"\n",
    "        # transform k to be a 1D array of integers\n",
    "        k = np.array(k).flatten().astype(int)\n",
    "\n",
    "        # Calculate the logistic scores for each category and observation\n",
    "        # logistic_scores = (beta * S**r + (1-beta) * P**r)**(1/r)\n",
    "        logistic_scores = CES(beta, S, P, r=r)\n",
    "\n",
    "        # Convert scores to probabilities using softmax\n",
    "        exp_scores = np.exp(logistic_scores)\n",
    "        probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)\n",
    "        # print(beta, probs)\n",
    "        # assert False\n",
    "\n",
    "        # Calculate the log-likelihood of the observed classes\n",
    "        log_likelihood = np.sum(np.log(probs[:, k]))\n",
    "        return -log_likelihood  # Return negative log-likelihood for minimization\n",
    "\n",
    "    if not use_scalar_minimization:\n",
    "\n",
    "        # Initial guess for beta\n",
    "        beta_initial = 1\n",
    "\n",
    "        # Minimize the negative log-likelihood\n",
    "        result = minimize(\n",
    "            fun=multinomial_log_likelihood,\n",
    "            x0=beta_initial,\n",
    "            args=(S, P, k),\n",
    "            method='BFGS'\n",
    "            # method='L-BFGS-B',  # 'L-BFGS-B' supports bounds\n",
    "            # bounds=[(0, 1)]     # Bounds for beta\n",
    "        )\n",
    "        \n",
    "        beta_hat = result.x[0]  # Estimated beta\n",
    "        print(f\"Estimated beta: {beta_hat}\")\n",
    "\n",
    "        # Calculate andard error using the Hessian inverse provided by the 'BFGS' method\n",
    "        std_err = np.sqrt(np.diag(result.hess_inv))\n",
    "        print(f\"Standard error of beta: {std_err}\")\n",
    "\n",
    "        # Confidence interval can be calculated as beta_hat +/- z * std_err\n",
    "        # For a 95% confidence interval, z is approximately 1.96\n",
    "        z = 1.96\n",
    "        conf_int = beta_hat - z * std_err[0], beta_hat + z * std_err[0]\n",
    "        print(f\"95% costnfidence interval for beta: {conf_int}\")\n",
    "\n",
    "    else:\n",
    "\n",
    "        result = minimize_scalar(\n",
    "            fun=multinomial_log_likelihood,\n",
    "            bounds=(0, 1),\n",
    "            args=(S, P, k),\n",
    "            method='bounded'\n",
    "        )\n",
    "        beta_hat = result.x  # Estimated beta\n",
    "\n",
    "        param_estimate = result.x\n",
    "        print(f\"Parameter Estimate: {param_estimate}\")\n",
    "\n",
    "        # Define a function to calculate the numerical second derivative\n",
    "        def numerical_second_derivative(f, x, h=1e-5):\n",
    "            return (f(x - h) - 2 * f(x) + f(x + h)) / h**2\n",
    "\n",
    "        # Calculate the numerical second derivative at the minimum\n",
    "        objective_function = lambda beta: multinomial_log_likelihood(beta, S, P, k)\n",
    "        second_derivative = numerical_second_derivative(objective_function, param_estimate)\n",
    "\n",
    "        # Proceed only if the second derivative is positive and thus indicates a minimum\n",
    "        if second_derivative <= 0:\n",
    "            print(\"The curvature at the minimum is non-positive, which suggests a flat minimum or numerical issues.\")\n",
    "            print('second derivative:', second_derivative)\n",
    "        second_derivative = np.abs(second_derivative)\n",
    "\n",
    "        # Estimate the variance (inverse of the curvature at the minimum for MLE)\n",
    "        variance_estimate = 1 / second_derivative\n",
    "        standard_error = np.sqrt(variance_estimate)\n",
    "\n",
    "        # Calculate the 95% confidence interval for the parameter estimate\n",
    "        ci_lower, ci_upper = norm.interval(0.95, loc=param_estimate, scale=standard_error)\n",
    "\n",
    "        # print(f\"Variance Estimate: {variance_estimate}\")\n",
    "        print(f\"Standard Error: {standard_error}\")\n",
    "        print(f\"95% Confidence Interval: ({ci_lower:.3f}, {ci_upper:.3f})\")\n",
    "\n",
    "    \n",
    "\n",
    "# Dummy data for demonstration purposes\n",
    "S = np.random.rand(3)  # S_k for each category\n",
    "P = np.random.rand(3)  # P_k for each category\n",
    "k = np.random.randint(0, 3, 100)  # Observed category for each observation\n",
    "\n",
    "estimate_beta(S, P, k)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dictator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Human:')\n",
    "estimate_beta(payoff_player_dictator, payoff_partner_dictator, df_dictator_human['choices'])\n",
    "print('ChatGPT-4:')\n",
    "estimate_beta(payoff_player_dictator, payoff_partner_dictator, df_dictator_gpt4['choices'])\n",
    "print('ChatGPT-3:')\n",
    "estimate_beta(payoff_player_dictator, payoff_partner_dictator, df_dictator_turbo['choices'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ultimatum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Ultimatum 1')\n",
    "print('Human:')\n",
    "estimate_beta(payoff_player_ultimatum_1, payoff_partner_ultimatum_1, df_ultimatum_1_human['choices'])\n",
    "print('ChatGPT-4:')\n",
    "estimate_beta(payoff_player_ultimatum_1, payoff_partner_ultimatum_1, df_ultimatum_1_gpt4['choices'])\n",
    "print('ChatGPT-3:')\n",
    "estimate_beta(payoff_player_ultimatum_1, payoff_partner_ultimatum_1, df_ultimatum_1_turbo['choices'])\n",
    "\n",
    "print('Ultimatum 2')\n",
    "print('Human:')\n",
    "estimate_beta(payoff_player_ultimatum_2, payoff_partner_ultimatum_2, df_ultimatum_2_human['choices'])\n",
    "print('ChatGPT-4:')\n",
    "estimate_beta(payoff_player_ultimatum_2, payoff_partner_ultimatum_2, df_ultimatum_2_gpt4['choices'])\n",
    "print('ChatGPT-3:')\n",
    "estimate_beta(payoff_player_ultimatum_2, payoff_partner_ultimatum_2, df_ultimatum_2_turbo['choices'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trust"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Trust 1')\n",
    "print('Human:')\n",
    "estimate_beta(payoff_player_trust_1, payoff_partner_trust_1, df_trust_1_human['choices'])\n",
    "print('ChatGPT-4:')\n",
    "estimate_beta(payoff_player_trust_1, payoff_partner_trust_1, df_trust_1_gpt4['choices'])\n",
    "print('ChatGPT-3:')\n",
    "estimate_beta(payoff_player_trust_1, payoff_partner_trust_1, df_trust_1_turbo['choices'])\n",
    "\n",
    "print('Trust 3')\n",
    "print('Human:')\n",
    "estimate_beta(payoff_player_trust_3, payoff_partner_trust_3, df_trust_3_human['choices'])\n",
    "print('ChatGPT-4:')\n",
    "estimate_beta(payoff_player_trust_3, payoff_partner_trust_3, df_trust_3_gpt4['choices'])\n",
    "print('ChatGPT-3:')\n",
    "estimate_beta(payoff_player_trust_3, payoff_partner_trust_3, df_trust_3_turbo['choices'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Public Goods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Human:')\n",
    "estimate_beta(payoff_player_PG, payoff_partner_PG, df_PG_human['choices'])\n",
    "print('ChatGPT-4:')\n",
    "estimate_beta(payoff_player_PG, payoff_partner_PG, df_PG_gpt4['choices'])\n",
    "print('ChatGPT-3:')\n",
    "estimate_beta(payoff_player_PG, payoff_partner_PG, df_PG_turbo['choices'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prisoner's Dilemma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_coo = r_coo_human\n",
    "r_def = r_def_human\n",
    "S = [r_coo * 400 + r_def * 000, r_coo * 700 + r_def * 300]\n",
    "P = [r_coo * 400 + r_def * 700, r_coo * 000 + r_def * 300]\n",
    "k = {}\n",
    "for model in ['Human', 'ChatGPT-4', 'ChatGPT-3']:    \n",
    "    if model == 'Human':\n",
    "        n_coo = n_coo_human\n",
    "        n_def = n_def_human\n",
    "    elif model == 'ChatGPT-4':\n",
    "        n_coo = n_coo_gpt4\n",
    "        n_def = n_def_gpt4\n",
    "    elif model == 'ChatGPT-3':\n",
    "        n_coo = n_coo_turbo\n",
    "        n_def = n_def_turbo\n",
    "    k[model] = [0] * n_coo + [1] * n_def\n",
    "    print(model)\n",
    "    # estimate_beta(S, P, k[model])\n",
    "    estimate_beta(S, P, k[model], r=.5)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig. S6: Framing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_steer(\n",
    "    df_list, \n",
    "    hues, \n",
    "    xlim,\n",
    "    xlabel,\n",
    "    fig_size=(5,4),\n",
    "    palette=[\n",
    "        orange, 'tab:olive', 'tab:purple', \n",
    "        'tab:cyan', 'tab:brown', 'tab:pink'\n",
    "    ],\n",
    "    multiple='dodge',\n",
    "    shrink=0.6,\n",
    "    **kargs\n",
    "):\n",
    "    # palette=[orange, '#10F30C', '#0C84F3', '#EF0CF3']\n",
    "    lines = ['-', '--', '-.', ':', '--', '-.', ':', '--', '-.', ':']\n",
    "    \n",
    "    for i, df in enumerate(df_list):\n",
    "        df = pd.DataFrame(df, columns=['choices'])\n",
    "        df['hue'] = hues[i]\n",
    "        df_list[i] = df\n",
    "    df = pd.concat(df_list, ignore_index=True)\n",
    "    ax = sns.histplot(\n",
    "        data=df,\n",
    "        x='choices',\n",
    "        hue='hue',\n",
    "        legend=True,\n",
    "        common_norm=False,\n",
    "        palette=palette,\n",
    "        multiple=multiple,\n",
    "        stat='density',\n",
    "        shrink=shrink,\n",
    "        **kargs\n",
    "    )\n",
    "\n",
    "    legend = plt.gca().get_legend()\n",
    "    legend.set_title('')\n",
    "\n",
    "    sns.set(rc={'figure.figsize':fig_size})\n",
    "    sns.set_style(\"ticks\")\n",
    "\n",
    "    plt.xlim(*xlim)\n",
    "    plt.xlabel(xlabel)\n",
    "    sns.despine()\n",
    "    # plt.legend()\n",
    "    # plt.show()\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explanation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "choices = [50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0]\n",
    "df_dictator_gpt4_witnessed = choices_to_df(choices, hue=str('ChatGPT-4 (witnessed)'))\n",
    "\n",
    "choices = [40.0, 60.0, 60.0, 50.0, 20.0, 20.0, 30.0, 30.0, 50.0, 70.0, 30.0, 30.0, 30.0, 50.0, 30.0] + [30, 20, 40, 40, 40, 70, 40, 40, 30, 70, 60, 40, 50, 40]\n",
    "df_dictator_turbo_witnessed = choices_to_df(choices, hue=str('ChatGPT-3 (witnessed)'))\n",
    "\n",
    "choices = [50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0]\n",
    "df_dictator_gpt4_explained = choices_to_df(choices, hue=str('ChatGPT-4 (with explanation)'))\n",
    "\n",
    "choices = [40, 60, 50, 70, 50, 50, 30, 50, 70, 50, 40, 30, 80, 70, 40, 30, 30, 75, 30, 50, 30, 50, 50, 30, 40, 40, 50, 30, 50, 20]\n",
    "df_dictator_turbo_explained = choices_to_df(choices, hue=str('ChatGPT-3 (with explanation)'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = json.load(open('records/dictator_witnessed_2023_03_31-10_01_37_PM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "print(', '.join([str(c) for c in choices if c is not None]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_dictator_turbo_witnessed['choices'].min())\n",
    "print(df_dictator_turbo_witnessed['choices'].max())\n",
    "print(df_dictator_turbo_witnessed['choices'].mean())\n",
    "print(df_dictator_turbo_witnessed['choices'].std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [\n",
    "    df_dictator_turbo, \n",
    "    df_dictator_turbo_witnessed,\n",
    "    df_dictator_turbo_explained,\n",
    "]\n",
    "hues = ['ChatGPT-3', 'Witnessed', 'Explained']\n",
    "plot_facet(\n",
    "    df_list=df_list,\n",
    "    binrange=(0, 100),\n",
    "    stat='density',\n",
    "    x_label='Split offered ($)',\n",
    "    palette=[green, 'tab:olive', 'tab:purple'],\n",
    "    kde=False,\n",
    ")\n",
    "# plt.savefig('figures/steer-dictator-witness-hist.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Paired with Other Players (Ultimatum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "choices = [50.0, 50.0, 50.0, 50.0, 1.0, 50.0, 1.0, 1.0, 50.0, 50.0, 20.0, 20.0, 20.0, 20.0, 20.0, 50.0, 20.0, 50.0, 20.0, 50.0, 50.0, 20.0, 50.0, 20.0, 40.0, 50.0, 50.0, 50.0, 1.0, 20.0]\n",
    "df_ultimatum_2_gpt4_paired_female = choices_to_df(choices, hue=str('ChatGPT-4 (paired with female)'))\n",
    "\n",
    "choices = [20.0, 50.0, 50.0, 50.0, 50.0, 50.0, 0.0, 20.0, 20.0, 20.0, 50.0, 20.0, 20.0, 50.0, 50.0, 20.0, 40.0, 1.0, 50.0, 1.0, 50.0, 40.0, 50.0, 1.0, 20.0, 20.0, 1.0, 50.0, 20.0]\n",
    "df_ultimatum_2_gpt4_paired_male = choices_to_df(choices, hue=str('ChatGPT-4 (paired with male)'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [\n",
    "    df_ultimatum_2_gpt4,\n",
    "    df_ultimatum_2_gpt4_paired_male,\n",
    "    df_ultimatum_2_gpt4_paired_female,\n",
    "]\n",
    "hues = ['ChatGPT-4', 'With male proposer', 'With female proposer']\n",
    "plot_facet(\n",
    "    df_list=df_list,\n",
    "    binrange=(0, 100),\n",
    "    stat='density',\n",
    "    x_label='Minimum proposal to accept ($)',\n",
    "    palette=[orange, 'tab:olive', 'tab:purple'],\n",
    "    kde=False,\n",
    ")\n",
    "# plt.savefig('figures/steer-ultimatum-gender-hist.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Occupation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "records_all, choices_all = json.load(open('records/ultimatum_2_gpt4_occupations_described_2023_04_10-10_56_28_AM.json', 'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot facet\n",
    "df_list = [\n",
    "    # df_ultimatum_2_gpt4,\n",
    "    choices_to_df(choices_all['mathematician'], hue='Mathematician'),\n",
    "    choices_to_df(choices_all['public relations specialist'], hue='Public relations specialist'),\n",
    "    # choices_to_df(choices_all['journalist'], hue='Journalist'),\n",
    "    # choices_to_df(choices_all['investment fund manager'], hue='Investment fund manager'),\n",
    "    choices_to_df(choices_all['legislator'], hue='Legislator'),\n",
    "]\n",
    "plot_facet(\n",
    "    df_list=df_list,\n",
    "    binrange=(0, 100),\n",
    "    stat='density',\n",
    "    x_label='Minimum proposal to accept ($)',\n",
    "    palette=['tab:olive', 'tab:purple', 'tab:cyan'],\n",
    "    kde=False,\n",
    ")\n",
    "# plt.savefig('figures/steer-ultimatum-occupation-hist.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Investment (Trust 2-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot facet\n",
    "df_list = [\n",
    "    choices_to_df(df_trust_2_gpt4['choices']*100./30, hue='$10 invested'),\n",
    "    choices_to_df(df_trust_3_gpt4['choices']*100./150, hue='$50 invested'),\n",
    "    choices_to_df(df_trust_4_gpt4['choices']*100./300, hue='$100 invested'),\n",
    "]\n",
    "plot_facet(\n",
    "    df_list=df_list,\n",
    "    binrange=(0, 100),\n",
    "    stat='density',\n",
    "    x_label='Return to investor (% revenue)',\n",
    "    palette=['tab:olive', orange, 'tab:purple'],\n",
    "    kde=False,\n",
    ")\n",
    "# plt.savefig('figures/steer-trust-gpt4-hist.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot facet\n",
    "df_list = [\n",
    "    choices_to_df(df_trust_2_turbo['choices']*100./30, hue='$10 invested'),\n",
    "    choices_to_df(df_trust_3_turbo['choices']*100./150, hue='$50 invested'),\n",
    "    choices_to_df(df_trust_4_turbo['choices']*100./300, hue='$100 invested'),\n",
    "]\n",
    "plot_facet(\n",
    "    df_list=df_list,\n",
    "    binrange=(0, 100),\n",
    "    stat='density',\n",
    "    x_label='Return to investor (% revenue)',\n",
    "    palette=['tab:olive', green, 'tab:purple'],\n",
    "    kde=False,\n",
    ")\n",
    "# plt.savefig('figures/steer-trust-turbo-hist.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot facet\n",
    "df_list = [\n",
    "    choices_to_df(df_trust_2_human['choices']*100./30, hue='$10 invested'),\n",
    "    choices_to_df(df_trust_3_human['choices']*100./150, hue='$50 invested'),\n",
    "    choices_to_df(df_trust_4_human['choices']*100./300, hue='$100 invested'),\n",
    "]\n",
    "plot_facet(\n",
    "    df_list=df_list,\n",
    "    binrange=(0, 100),\n",
    "    stat='density',\n",
    "    x_label='Return to investor (% revenue)',\n",
    "    palette=['tab:olive', blue, 'tab:purple'],\n",
    "    kde=False,\n",
    ")\n",
    "# plt.savefig('figures/steer-trust-human-hist.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig. S7: Learning\n",
    "\n",
    "### Ultimatum Game\n",
    "\n",
    "#### Session 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = json.load(open('records/ultimatum_21_gpt4_2023_04_07-03_32_21_AM.json', 'r'))\n",
    "\n",
    "choices = [[\n",
    "    extract_amout(\n",
    "        messages[i]['content'], \n",
    "        prefix='$', \n",
    "        print_except=True,\n",
    "        type=float\n",
    "    ) for i in range(-3, 0, 2) \n",
    "    ] for messages in records['messages']\n",
    "]\n",
    "\n",
    "choices = [x for x in choices if None not in x]\n",
    "choices_2 = [x[0] for x in choices if x[0] is not None]\n",
    "choices_1 = [x[1] for x in choices if x[1] is not None]\n",
    "print(len(choices))\n",
    "df_ultimatum_1_gpt4_21 = choices_to_df(choices_1, hue='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_facet(\n",
    "    df_list=[\n",
    "        # df_ultimatum_1_human,\n",
    "        df_ultimatum_1_gpt4,\n",
    "        df_ultimatum_1_gpt4_21,\n",
    "    ],\n",
    "    palette=[orange, orange],\n",
    "    binrange=(0, 100),\n",
    "    binwidth=10,\n",
    "    stat='density',\n",
    "    x_label='Proposal to give ($)',\n",
    "    kde=False,\n",
    ")\n",
    "# plt.savefig('figures/steer-order-U21-gpt4.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "choices_1 = [35.0, 40.0, 40.0, 40.0, 40.0, 40.0, 60.0, 40.0, 40.0, 30.0, 30.0, 45.0, 50.0, 30.0, 40.0, 50.0, 50.0, 50.0, 55.0, 70.0] + [40, None, 30, 40, None, 50, 50, 50]\n",
    "df_ultimatum_1_turbo_21 = choices_to_df(choices_1, hue='')\n",
    "print([x for x in choices_1 if x is not None])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = json.load(open('records/ultimatum_21_2023_03_29-05_41_19_PM.json', 'r'))\n",
    "choices_1 = extract_choices(records)\n",
    "print(', '.join([str(x) for x in choices_1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_facet(\n",
    "    df_list=[\n",
    "        # df_ultimatum_1_human,\n",
    "        df_ultimatum_1_turbo,\n",
    "        df_ultimatum_1_turbo_21,\n",
    "    ],\n",
    "    palette=[green, green],\n",
    "    binrange=(0, 100),\n",
    "    binwidth=10,\n",
    "    stat='density',\n",
    "    x_label='Propose to give ($)',\n",
    "    kde=False,\n",
    ")\n",
    "# plt.savefig('figures/steer-order-U21-turbo.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Session 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = json.load(open('records/ultimatum_12_gpt4_2023_04_07-03_14_30_AM.json', 'r'))\n",
    "\n",
    "choices = [[\n",
    "    extract_amout(\n",
    "        messages[i]['content'], \n",
    "        prefix='$', \n",
    "        print_except=True,\n",
    "        type=float\n",
    "    ) for i in range(-3, 0, 2) \n",
    "    ] for messages in records['messages']\n",
    "]\n",
    "\n",
    "choices = [x for x in choices if None not in x]\n",
    "choices_1 = [x[0] for x in choices if x[0] is not None]\n",
    "choices_2 = [x[1] for x in choices if x[1] is not None]\n",
    "print(len(choices))\n",
    "df_ultimatum_2_gpt4_12 = choices_to_df(choices_2, hue='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = plot_facet(\n",
    "    df_list=[\n",
    "        # df_ultimatum_2_human,\n",
    "        df_ultimatum_2_gpt4,\n",
    "        df_ultimatum_2_gpt4_12,\n",
    "    ],\n",
    "    palette=[orange, orange],\n",
    "    binrange=(0, 100),\n",
    "    binwidth=10,\n",
    "    stat='density',\n",
    "    x_label='Minimum proposal to accept ($)',\n",
    "    kde=False,\n",
    ")\n",
    "# g.set(ylim=(0, 0.05))\n",
    "# plt.savefig('figures/steer-order-U12-gpt4.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "choices_2 = [40.0, 40.0, 30.0, 40.0, 40.0, 50.0, 50.0, 35.0, 30.0, 30.0, 30.0, 30.0, 50.0, 30.0, 50.0, 30.0, 30.0, 40.0, 30.0, 20.0, 50.0, 40.0, 50.0, 40.0, 30.0, 50.0, 30.0, 20.0, 40.0, 30.0, 30.0, 30.0, 40.0, 50.0, 30.0, 30.0, 30.0, 30.0, 50.0, 50.0, 40.0, 30.0, 50.0] + [40, 30, 40, 30, 40, 40, 30, 30, 30, 40, 40, 40, 40, 30, 40]\n",
    "df_ultimatum_2_turbo_12 = choices_to_df(choices_2, hue='')\n",
    "print(choices_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = json.load(open('records/ultimatum_12_2023_03_29-06_27_31_PM.json', 'r'))\n",
    "choices_2 = extract_choices(records)\n",
    "print(', '.join([str(x) for x in choices_2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = plot_facet(\n",
    "    df_list=[\n",
    "        # df_ultimatum_2_human,\n",
    "        df_ultimatum_2_turbo,\n",
    "        df_ultimatum_2_turbo_12,\n",
    "    ],\n",
    "    palette=[green, green],\n",
    "    binrange=(0, 100),\n",
    "    binwidth=10,\n",
    "    stat='density',\n",
    "    x_label='Minimum proposal to accept ($)',\n",
    "    kde=False,\n",
    ")\n",
    "# g.set(ylim=(0, 0.05))\n",
    "plt.savefig('figures/steer-order-U12-turbo.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trust Game"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = json.load(open('records/trust_gpt4_12_2023_04_12-01_41_44_AM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "df_trust_12 = choices_to_df(choices, hue=str('ChatGPT-4 (after S1)'))\n",
    "\n",
    "records = json.load(open('records/trust_gpt4_13_2023_04_12-02_03_38_AM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "df_trust_13 = choices_to_df(choices, hue=str(''))\n",
    "\n",
    "records = json.load(open('records/trust_gpt4_14_2023_04_12-02_20_03_AM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "df_trust_14 = choices_to_df(choices, hue=str('ChatGPT-4 (after S1)'))\n",
    "\n",
    "records = json.load(open('records/trust_gpt4_21_2023_04_12-02_35_54_AM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "df_trust_21 = choices_to_df(choices, hue=str('ChatGPT-4 (after S2)'))\n",
    "\n",
    "records = json.load(open('records/trust_gpt4_31_2023_04_12-02_51_38_AM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "df_trust_31 = choices_to_df(choices, hue=str(''))\n",
    "\n",
    "records = json.load(open('records/trust_gpt4_41_2023_04_12-03_09_46_AM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "df_trust_41 = choices_to_df(choices, hue=str('ChatGPT-4 (after S4)'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = plot_facet(\n",
    "    df_list=[\n",
    "        # df_trust_1_human,\n",
    "        df_trust_1_gpt4,\n",
    "        df_trust_31,\n",
    "    ],\n",
    "    palette=[orange, orange],\n",
    "    binrange=(0, 100),\n",
    "    binwidth=10,\n",
    "    stat='density',\n",
    "    x_label='Investment ($)',\n",
    "    kde=False,\n",
    ")\n",
    "# g.set(ylim=(0, 0.12))\n",
    "# plt.savefig('figures/steer-order-T31-gpt4.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_facet(\n",
    "    df_list=[\n",
    "        # df_trust_3_human,\n",
    "        df_trust_3_gpt4,\n",
    "        df_trust_13,\n",
    "    ],\n",
    "    palette=[orange, orange],\n",
    "    binrange=(0, 150),\n",
    "    bins=15,\n",
    "    xticks_locs=[0, 25, 50, 75, 100, 125, 150],\n",
    "    stat='density',\n",
    "    x_label='Return to investor ($)',\n",
    "    kde=False,\n",
    ")\n",
    "# plt.savefig('figures/steer-order-T13-gpt4.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "choices = [25, 24, 27, 20, 27, 25, 25, 24, 24, 18, 25, 27, 24, 25, 25, 25, 25, 25, 24, 20, 24, 25, 24, 25, 22, 20, 25, 20, 21, 20]\n",
    "print(', '.join([str(x) for x in choices if x is not None]))\n",
    "df_trust_12_turbo = choices_to_df(choices, hue=str('ChatGPT-3 (after S1)'))\n",
    "\n",
    "choices = [100, 100, 120, 125, 100, 125, 125, 120, 120, 120, 125, 112, 120, 120, 110, 135, 100, None, 130, 130, 120, 125, 115, 120, 120, 120, 100, 130, 100, 125]\n",
    "print(', '.join([str(x) for x in choices if x is not None]))\n",
    "df_trust_13_turbo = choices_to_df(choices, hue=str(''))\n",
    "\n",
    "choices = [240, 225, 210, 200, 250, 240, 225, 240, None, 240, 250, 250, 225, 240, 240, 240, 225, 225, 220, 225, 225, 240, 250, 240, 250, 210, 240, 220, 240, 240]\n",
    "print(', '.join([str(x) for x in choices if x is not None]))\n",
    "df_trust_14_turbo = choices_to_df(choices, hue=str('ChatGPT-3 (after S1)'))\n",
    "\n",
    "choices = [50, 50, 50, 50, 50, 50, 50, 50, 30, 50, 20, 50, 30, 20, 50, 30, 50, 50, 50, 50, None, 50, 50, 50, 50, 50, 50, 30, 50, 50]\n",
    "print(', '.join([str(x) for x in choices if x is not None]))\n",
    "df_trust_21_turbo = choices_to_df(choices, hue=str('ChatGPT-3 (after S2)'))\n",
    "\n",
    "choices = [50, None, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 30, 50, 50, 30, 20, 50, 50, 50, 50, 100, 50, 50, 50, None, 50, 50]\n",
    "print(', '.join([str(x) for x in choices if x is not None]))\n",
    "df_trust_31_turbo = choices_to_df(choices, hue=str(''))\n",
    "\n",
    "choices = [50, 50, 50, 50, 50, 50, 50, 50, 50, None, 50, None, 50, 50, 50, 50, 50, 50, 50, 70, 50, 20, 50, 100, 50, 50, 50, 50, 50, 50]\n",
    "print(', '.join([str(x) for x in choices if x is not None]))\n",
    "df_trust_41_turbo = choices_to_df(choices, hue=str('ChatGPT-3 (after S4)'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = json.load(open('records/trust_turbo_31_2023_04_12-04_53_06_PM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "print(', '.join([str(x) for x in choices if x is not None]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = json.load(open('records/trust_turbo_13_2023_04_12-04_37_21_PM.json', 'r'))\n",
    "choices = extract_choices(records)\n",
    "print(', '.join([str(x) for x in choices if x is not None]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = plot_facet(\n",
    "    df_list=[\n",
    "        # df_trust_1_human,\n",
    "        df_trust_1_turbo,\n",
    "        df_trust_31_turbo,\n",
    "    ],\n",
    "    palette=[green, green],\n",
    "    binrange=(0, 100),\n",
    "    binwidth=10,\n",
    "    stat='density',\n",
    "    x_label='Investment ($)',\n",
    "    kde=False,\n",
    ")\n",
    "# g.set(ylim=(0, 0.10))\n",
    "# plt.savefig('figures/steer-order-T31-turbo.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_facet(\n",
    "    df_list=[\n",
    "        # df_trust_3_human,\n",
    "        df_trust_3_turbo,\n",
    "        df_trust_13_turbo,\n",
    "    ],\n",
    "    palette=[green, green],\n",
    "    binrange=(0, 150),\n",
    "    bins=15,\n",
    "    xticks_locs=[0, 25, 50, 75, 100, 125, 150],\n",
    "    stat='density',\n",
    "    x_label='Return to investor ($)',\n",
    "    kde=False,\n",
    ")\n",
    "# plt.savefig('figures/steer-order-T13-turbo.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chatgpt",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}