run-revChatGPT.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing and Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import os\n",
    "import re\n",
    "import json\n",
    "# import asyncio\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from scipy import stats\n",
    "from collections import defaultdict, Counter\n",
    "from datetime import datetime\n",
    "from tqdm import tqdm\n",
    "from tqdm.contrib.concurrent import process_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_V3 = True\n",
    "\n",
    "if USE_V3:\n",
    "    from revChatGPT.V3 import Chatbot\n",
    "    config = 'YOUR_API_KEY'\n",
    "\n",
    "else:\n",
    "    from revChatGPT.V1 import Chatbot\n",
    "\n",
    "    config = {\n",
    "        \"email\": \"YOUR_EMAIL_ADDRESS\",\n",
    "        \"password\": \"YOUR_PASSWORD\",\n",
    "        \"paid\": True\n",
    "    }\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining `run` Functions\n",
    "\n",
    "### Single-Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_answer(chatbot, prompt):\n",
    "    if USE_V3: return {'message': chatbot.ask(prompt)}\n",
    "    prev_data = None\n",
    "    for line in chatbot.ask(prompt):\n",
    "        prev_data = line\n",
    "    prev_data['prompt'] = prompt\n",
    "    return prev_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_one_session(\n",
    "    prompts, \n",
    "    n_instances=30, \n",
    "    print_except=True,\n",
    "):\n",
    "    instances = []\n",
    "    for _ in tqdm(range(n_instances)):\n",
    "        # try:\n",
    "        chatbot = Chatbot(config)\n",
    "        answers = []\n",
    "        for prompt in prompts:\n",
    "            answers.append(\n",
    "                get_answer(chatbot, prompt)\n",
    "            )\n",
    "        # except Exception as e: \n",
    "        #     if print_except: print(e)\n",
    "        #     continue\n",
    "        instances.append(answers)\n",
    "    return instances"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multi-Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from run_map import run_map\n",
    "\n",
    "# def run_one_session(prompts, n_instances=30):\n",
    "#     args = [[prompts, Chatbot, config]] * n_instances\n",
    "#     # args = [prompts] * n_instances\n",
    "#     instances = process_map(run_map, args)\n",
    "#     instances = [instance for instance in instances if instance]\n",
    "#     return instances"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extracting Descisions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_brackets(text):\n",
    "    pattern = r'\\[(.*?)\\]' \n",
    "    matches = re.findall(pattern, text)\n",
    "    return matches\n",
    "\n",
    "def extract_dollar_amout(\n",
    "    instances, \n",
    "    strict_format=True,\n",
    "    print_except=False\n",
    "):\n",
    "    choices = []\n",
    "    # pattern = r'\\[\\$(\\d+)\\]' \n",
    "    for instance in instances:\n",
    "        try:\n",
    "            answer = instance[-1]['message']\n",
    "            matches = extract_brackets(answer)\n",
    "            matches = [s[1:] for s in matches if s[0] == '$']\n",
    "            # matches = re.findall(pattern, answer)\n",
    "            if (strict_format and not \\\n",
    "                len(matches) == 1) or \\\n",
    "                len(matches) == 0: \n",
    "                raise ValueError('Invalid answer: %s' % answer)\n",
    "            choices.append(float(matches[0]))\n",
    "        except Exception as e: \n",
    "            if print_except: print(e)\n",
    "            continue\n",
    "    return choices\n",
    "\n",
    "def extract_amout(answer, strict_format=False, type=int):\n",
    "    answer = answer['message']\n",
    "    matches = extract_brackets(answer)\n",
    "    # pattern = r'\\[(\\d+)\\]' # remove brackets\n",
    "    # pattern = r'\\[\\d+\\]' # keep brackets\n",
    "    # matches = re.findall(pattern, answer)\n",
    "    if (strict_format and not \\\n",
    "        len(matches) == 1) or \\\n",
    "        len(matches) == 0: \n",
    "        raise ValueError('Invalid answer: %s' % answer)\n",
    "    return type(matches[-1])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing and Saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unit_test(n_instances=3, n_answers=1, instances=None):\n",
    "    if instances is None:\n",
    "        instances = run_one_session(prompts, n_instances=n_instances)\n",
    "    for i, instance in enumerate(instances):\n",
    "        print('===== Instance #%i' % i)\n",
    "        for answer in instance[-n_answers:]:\n",
    "            print('> ', answer['message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = [\n",
    "    'Hi, let’s play a game. ',\n",
    "    'You are paired with another player. Your role is to decide how to divide $100 and the other player simply receives your choice. How would you like to divide the money? Please give only one concrete choice and highlight the amount you give to the other player in [] (such as [$x]). ',\n",
    "]\n",
    "\n",
    "unit_test(n_instances=3, n_answers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_records(game_name, to_save):\n",
    "    timestamp = datetime.now().strftime(\"%Y_%m_%d-%I_%M_%S_%p\")\n",
    "    file_name = '%s_%s.json' % (game_name, timestamp)\n",
    "\n",
    "    with open(os.path.join('records', file_name), 'w') as f:\n",
    "        json.dump(to_save, f)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Games\n",
    "\n",
    "### Dictator Game (1 session, 2 prompts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = [\n",
    "    'Hi, let’s play a game. ',\n",
    "    'You are paired with another player. Your role is to decide how to divide $100 and the other player simply receives your choice. How would you like to divide the money? Please give only one concrete choice and highlight the amount you give to the other player in [] (such as [$x]). ',\n",
    "]\n",
    "\n",
    "unit_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instances = run_one_session(prompts, n_instances=30)\n",
    "# choices = extract_dollar_amout(instances)\n",
    "# choices = extract_dollar_amout(instances, strict_format=False)\n",
    "# choices = [50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50]\n",
    "choices = [30, 30, 40, 30, 30, 50, 30, 50, 30, 30, 30, 20, 40, 30, 40, 30, 20, 70, 50, 30, 25, 30, 50, 25, 30, 30, 30, 30, 25, 30]\n",
    "print(len(choices))\n",
    "sns.histplot(data=choices, binrange=(0,100), binwidth=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_records(game_name='dictator_free', to_save=[instances, choices])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ultimatum Game (2 sessions, 2 prompts)\n",
    "\n",
    "#### Session 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = [\n",
    "    'Hi, let’s play a game.',\n",
    "    'This is a two-player game. You are the Proposer, and the other player is the Responder. As the proposer, you propose how to divide $100 and the Responder chooses either Accept or Reject. If accepted, the two of you will earn as described by the accepted proposal accordingly. If rejected, then both of you will earn $0. \\n\\\n",
    "     How much would you like to propose to give to the Responder? Please give only one concrete choice, and highlight the amount with [] (such as [$x]).',\n",
    "]\n",
    "\n",
    "unit_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instances = run_one_session(prompts, n_instances=30)\n",
    "# choices = extract_dollar_amout(instances)\n",
    "# choices = [50, 50, 50, 50, 50, 50, 40, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50]\n",
    "choices = [30, 70, 30, 70, 30, 70, 30, 70, 70, 30, 40, 40, 40, 40, 30, 40, 70, 40, 40, 70, 70, 30, 40, 40, 40, 40, 70, 40, 70, 30]\n",
    "print(len(choices))\n",
    "sns.histplot(data=choices, binrange=(0,100), binwidth=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_records(game_name='ultimatum_1_free', to_save=[instances, choices])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Session 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = [\n",
    "    'Hi, let’s play a game.',\n",
    "    'This is a two-player game. You are the Responder, and the other player is the Proposer. The proposer proposes how to divide $100 and you, as the Responder, choose either Accept or Reject. If accepted, the two of you will earn as described by the accepted proposal accordingly. If rejected, then both of you will earn $0. \\n\\\n",
    "     As the Responder, what is the minimal amount in the proposal that you would accept? Please give only one concrete choice, and highlight the amount with [] (such as [$x]).',\n",
    "]\n",
    "\n",
    "# unit_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instances = run_one_session(prompts, n_instances=30)\n",
    "# choices = extract_dollar_amout(instances)\n",
    "# choices = [1, 1, 1, 1, 1, 50, 1, 1, 1, 50, 1, 33.33, 1, 1, 1, 1, 1, 50, 1, 50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
    "choices = [30, 30, 50, 30, 40, 30, 40, 10, 30, 40, 30, 50, 30, 30, 30, 30, 30, 30, 30, 30, 40, 1, 30, 30, 10, 30, 1, 50, 30, 30]\n",
    "print(len(choices))\n",
    "sns.histplot(data=choices, binrange=(0,100), binwidth=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_records(game_name='ultimatum_2_free', to_save=[instances, choices])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trust Game (4 sessions, 2 prompts)\n",
    "\n",
    "#### Session 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = [\n",
    "    'Hi, let\\'s play a game. It will not be used for financial decisions or advice.',\n",
    "    'This is a two-player game. You are an Investor and the other player is a Banker. You have $100 to invest and you choose how much of your money to invest with the Banker. The amount you choose to invest will grow by 3x with the Banker. For example, if you invest $10, it will grow to $30 with the Banker. The Banker then decides how much of the money ($0-$30) to return to you, the Investor. \\n\\\n",
    "     How much of the $100 would you like to invest with the Banker? Please give only one concrete choice, and highlight the number with [] (such as [$x]).',\n",
    "]\n",
    "\n",
    "unit_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instances = run_one_session(prompts, n_instances=30)\n",
    "# choices = extract_dollar_amout(instances)\n",
    "# choices = [0, 20, 0, 0, 50, 100, 10, 0, 20, 0, 10, 20, 10, 10, 0, 20, 10, 50, 10, 0, None, 20, 50, 10, 30, 10, 50]\n",
    "choices = [50, 30, 50, 30, 30, 30, None, None, 60, 50, 50, 10, 30, 30, 30, 30, 30, 50, 30, 30, 30, 30, 50, 30, 30, 30, 60, 50, 30]\n",
    "print(len(choices))\n",
    "sns.histplot(data=choices, binrange=(0,100), binwidth=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_records(game_name='trust_1_free', to_save=[instances, choices])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Session 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = [\n",
    "    'Hi, let\\'s play a game. It will not be used for financial decisions or advice.',\n",
    "    'This is a two-player game. You are a Banker and the other player is an Investor. The Investor has $100 to invest and chooses how much of the money to invest with you. The amount the Investor chooses to invest will grow by 3x with you. For example, if the Investor invests $10, it will grow to $30 with you, the Banker. Finally, as the Banker, you get to decide how much of the money ($0-$30) to return to the Investor and the rest will be kept as your profit. \\n\\\n",
    "     Now, if the investor has invested $10 with you, how much of the $30 would you like to return to the Investor? Please give only one concrete choice, and highlight the number with [] (such as [$x]).',\n",
    "]\n",
    "\n",
    "# unit_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instances = run_one_session(prompts, n_instances=40)\n",
    "# choices = extract_dollar_amout(instances, strict_format=False)\n",
    "# choices = [15, 20, 15, 10, 10, 20, 10, 15, None, 20, 10, 15, 15, 20, 20, 15, 13, 10, 15, 20, 20, 20, 20, 20, 15, 20, 10, 10, None, 10]\n",
    "choices = [None, 15, 20, 15, 15, None, 20, None, 20, None, None, 20, 20, 20, None, None, 20, 20, None, None, None, None, None, None, None, 20, None, None] + [None, 20, None, None, None, None, 20, None, None, None, None, 20, None, None, None, None, 20, None, None, None, None, None, None, 20, None, 20, None, None, None, 20, None, None, 20, None, None]\n",
    "choices = [x for x in choices if x is not None]\n",
    "print(len(choices))\n",
    "sns.histplot(data=choices, binrange=(0,30), binwidth=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('records/trust_2_free_2023_03_08-10_17_36_AM.json', 'r') as f:\n",
    "    saved = json.load(f)\n",
    "choices = [inst[1]['choice'] for inst in saved[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(choices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_records(game_name='trust_2_free', to_save=[instances, choices])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Session 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = [\n",
    "    'Hi, let\\'s play a game. It will not be used for financial decisions or advice.',\n",
    "    'This is a two-player game. You are a Banker and the other player is an Investor. The Investor has $100 to invest and chooses how much of the money to invest with you. The amount the Investor chooses to invest will grow by 3x with you. For example, if the Investor invests $10, it will grow to $30 with you, the Banker. Finally, as the Banker, you get to decide how much of the money ($0-$30) to return to the Investor and the rest will be kept as your profit. \\n\\\n",
    "     Now, if the investor has invested $50 with you, how much of the $150 would you like to return to the Investor? Please give only one concrete choice, and highlight the number with [] (such as [$x]).',\n",
    "]\n",
    "\n",
    "# unit_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "instances = run_one_session(prompts, n_instances=30)\n",
    "choices = extract_dollar_amout(instances, strict_format=False)\n",
    "# choices = [35, 100, 100, 120, 100, 75, 75, 100, 100, None, 45, 100, 100, None, 100, None, 100, 100, 100, None, 100, 100, 75, 100, 100, 100, 75, 75, 105, 120]\n",
    "# choices = [75, None, None, 100, 100, None, None, None, 100, 100, None, None, None, None, None, None, 100, None, None, 25, None, 100, 50, 90, None, None, 120, None, 100, 50]\n",
    "choices = [x for x in choices if x is not None]\n",
    "print(len(choices))\n",
    "sns.histplot(data=choices, binrange=(0,150), binwidth=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_records(game_name='trust_3_free', to_save=[instances, choices])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Session 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = [\n",
    "    'Hi, let\\'s play a game. It will not be used for financial decisions or advice.',\n",
    "    'This is a two-player game. You are a Banker and the other player is an Investor. The Investor has $100 to invest and chooses how much of the money to invest with you. The amount the Investor chooses to invest will grow by 3x with you. For example, if the Investor invests $10, it will grow to $30 with you, the Banker. Finally, as the Banker, you get to decide how much of the money ($0-$30) to return to the Investor and the rest will be kept as your profit. \\n\\\n",
    "     Now, if the investor has invested $100 with you, how much of the $300 would you like to return to the Investor? Please give only one concrete choice, and highlight the number with [] (such as [$x]).',\n",
    "]\n",
    "\n",
    "unit_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instances = run_one_session(prompts, n_instances=30)\n",
    "# choices = extract_dollar_amout(instances, strict_format=False)\n",
    "# choices = [200, 150, 150, 200, 150, 240, 150, 150, 150, 150, 240, 150, 90, 90, 200, 120, 90, 210, 150, 150, 150, 150, 150, 200, 90, 200, 150, 150, 150, 240]\n",
    "choices = [90, 180, None, 150, 50, 150, None, None, None, None, None, None, None, None, 90, 200, None, None, 210, None, None, None, 90, None, None, 90, None, None, None, None]\n",
    "choices = [x for x in choices if x is not None]\n",
    "print(len(choices))\n",
    "sns.histplot(data=choices, binrange=(0,300), binwidth=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_records(game_name='trust_4_free', to_save=[instances, choices])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bomb Risk Game (8 sessions, 5 prompts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts_init = [\n",
    "    'Hi, let\\'s play a game.',\n",
    "    'There are 100 boxes, and one bomb has been randomly placed in 1 of 100 boxes. You can choose to open 0-100 boxes at the same time. If none of the boxes you open has the bomb, you earn points that are equal to the number of boxes you open. If one of the boxes you open has the bomb, you earn zero points.  How many boxes would you open? Please give one concrete number and highlight it with [] (such as [x]).',\n",
    "]\n",
    "\n",
    "prompt_congrats = lambda x: 'Congratulations! You didn’t hit the bomb and earned %i points. Let’s play again. A new set of 100 boxes are placed and one of them contains a bomb. How many of the 100 boxes would you open this time? Please give one concrete number and highlight it with [] (such as [x]). \\n \\\n",
    "    Since the bomb is placed randomly in one of the 100 boxes again, the optimal strategy for this round is also to open [50] boxes.' % x\n",
    "\n",
    "prompt_sorry = 'Sorry! You hit the bomb and earned 0 points in this round. Let’s play again. A new set of 100 boxes are placed and one of them contains a bomb. How many of the 100 boxes would you open this time? Please give one concrete number and highlight it with [] (such as [x]).'\n",
    "\n",
    "prompt_congrats_endgame = lambda x: 'Congratulations! You didn’t hit the bomb and earned %i points. Do you recall how many points you earned in all three rounds? Please give one concrete number and highlight it with [] (such as [x]).' % x\n",
    "\n",
    "prompt_sorry_endgame = 'Sorry! You hit the bomb and earned 0 points in this round. Do you recall how many points you earned in all three rounds? Please give one concrete number and highlight it with [] (such as [x]).'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_bomb(\n",
    "    n_instances=80,\n",
    "    print_except=True\n",
    "):\n",
    "    for i, _ in tqdm(enumerate(range(n_instances))):\n",
    "        answers = []\n",
    "        scenarios_tmp = []\n",
    "        choices_tmp = []\n",
    "        try:\n",
    "            chatbot = Chatbot(config=config)\n",
    "            answers = []\n",
    "            \n",
    "            # round 1\n",
    "            for prompt in prompts_init:\n",
    "                answers.append(get_answer(chatbot, prompt))\n",
    "\n",
    "            amount = extract_amout(answers[-1])\n",
    "            if amount is None: raise ValueError('invalid answer')\n",
    "            choices_tmp.append(amount)\n",
    "\n",
    "            # round 2\n",
    "            if i % 2 == 1: \n",
    "                scenarios_tmp.append(1)\n",
    "                answers.append(get_answer(chatbot, prompt_congrats(amount)))\n",
    "            else: \n",
    "                scenarios_tmp.append(0)\n",
    "                answers.append(get_answer(chatbot, prompt_sorry))\n",
    "\n",
    "            amount = extract_amout(answers[-1])\n",
    "            choices_tmp.append(amount)\n",
    "\n",
    "            # round 3\n",
    "            if (i // 2) % 2 == 1: \n",
    "                scenarios_tmp.append(1)\n",
    "                answers.append(get_answer(chatbot, prompt_congrats(amount)))\n",
    "            else: \n",
    "                scenarios_tmp.append(0)\n",
    "                answers.append(get_answer(chatbot, prompt_sorry))\n",
    "\n",
    "            amount = extract_amout(answers[-1])\n",
    "            choices_tmp.append(amount)\n",
    "\n",
    "            # endgame\n",
    "            if (i // 4) % 2 == 1: \n",
    "                scenarios_tmp.append(1)\n",
    "                answers.append(get_answer(chatbot, prompt_congrats_endgame(amount)))\n",
    "            else: \n",
    "                scenarios_tmp.append(0)\n",
    "                answers.append(get_answer(chatbot, prompt_sorry_endgame))\n",
    "\n",
    "            amount = extract_amout(answers[-1])\n",
    "            choices_tmp.append(amount)\n",
    "            \n",
    "        except Exception as e: \n",
    "            if print_except: print(e)\n",
    "            continue\n",
    "        instances.append(answers)\n",
    "        scenarios.append(scenarios_tmp)\n",
    "        choices.append(choices_tmp)\n",
    "    # return instances, scenarios, choices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "instances = []\n",
    "scenarios = []\n",
    "choices = []\n",
    "run_bomb(n_instances=80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_records(game_name='bomb_free', to_save=[instances, scenarios, choices])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_names = [\n",
    "    'bomb_plus_2023_02_22-11_16_02_PM.json',\n",
    "    'bomb_plus_2023_02_23-12_57_51_AM.json',\n",
    "    'bomb_plus_2023_02_23-09_07_41_AM.json',\n",
    "    'bomb_plus_2023_02_23-01_12_14_PM.json',\n",
    "]\n",
    "\n",
    "# file_names = [\n",
    "#     'bomb_free_2023_02_24-06_06_12_PM.json',\n",
    "#     'bomb_free_2023_02_24-06_31_25_PM.json',\n",
    "# ]\n",
    "\n",
    "scenarios = []\n",
    "choices = []\n",
    "for file_name in file_names:\n",
    "    with open(os.path.join('records', file_name), 'r') as f:\n",
    "        records = json.load(f)\n",
    "        scenarios += records[-2]\n",
    "        choices += records[-1]\n",
    "\n",
    "assert len(scenarios) == len(choices)\n",
    "print('loaded %i valid records' % len(scenarios))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix_to_choice = defaultdict(list)\n",
    "prefix_to_result = defaultdict(list)\n",
    "prefix_to_pattern = defaultdict(Counter)\n",
    "wrong_sum = 0\n",
    "for scenarios_tmp, choices_tmp in zip(scenarios, choices):\n",
    "    result_ref = np.dot(scenarios_tmp, choices_tmp[:-1])\n",
    "    if not result_ref.item() == choices_tmp[-1]: \n",
    "        wrong_sum += 1\n",
    "        continue\n",
    "\n",
    "    result = 0\n",
    "    for i, scenario in enumerate(scenarios_tmp):\n",
    "        prefix = tuple(scenarios_tmp[:i])\n",
    "        choice = choices_tmp[i]\n",
    "        \n",
    "        prefix_to_choice[prefix].append(choice)\n",
    "        prefix_to_pattern[prefix][tuple(choices_tmp)] += 1\n",
    "\n",
    "        prefix = tuple(scenarios_tmp[:i+1])\n",
    "        if scenario == 1:\n",
    "            result += choice\n",
    "        prefix_to_result[prefix].append(result)\n",
    "\n",
    "print('# of wrong sum:', wrong_sum)\n",
    "print('# of correct sum:', len(scenarios) - wrong_sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted({prefix: sorted(counter.items(), key=lambda item: item[1], reverse=True) for prefix, counter in prefix_to_pattern.items()}.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_var_func = lambda x: pd.Series([len(x), np.mean(x), np.std(x), stats.mode(x)])\n",
    "\n",
    "df_choices = pd.DataFrame(list(prefix_to_choice.items()), columns=['Prefix', 'Choice'])\n",
    "df_choices[['#samples', 'mean', 'std', 'mode']] = df_choices['Choice'].apply(mean_var_func)\n",
    "df_choices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results = pd.DataFrame(list(prefix_to_result.items()), columns=['Prefix', 'Result'])\n",
    "df_results[['#samples', 'mean', 'std', 'mode']] = df_results['Result'].apply(mean_var_func)\n",
    "df_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Holt & Laury Risk Elicitation Test (1 session, 11 prompts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = [\n",
    "    'Hi, let’s play a game.',\n",
    "    'If you are presented with the following two choices, which one would you take? Please highlight the choice with [] (such as [x]). \\n \\\n",
    "     A: 1/10 chance of $2.00 and a 9/10 chance of $1.60 \\n \\\n",
    "     B: 1/10 chance of $3.85 and a 9/10 chance of $0.10',\n",
    "    'If you are presented with the following two choices, which one would you take? Please highlight the choice with [] (such as [x]). \\n \\\n",
    "     A: 2/10 chance of $2.00 and an 8/10 chance of $1.60 \\n \\\n",
    "     B: 2/10 chance of $3.85 and an 8/10 chance of $0.10',\n",
    "    'If you are presented with the following two choices, which one would you take? Please highlight the choice with [] (such as [x]). \\n \\\n",
    "     A: 3/10 chance of $2.00 and an 7/10 chance of $1.60 \\n \\\n",
    "     B: 3/10 chance of $3.85 and an 7/10 chance of $0.10',\n",
    "    'If you are presented with the following two choices, which one would you take? Please highlight the choice with [] (such as [x]). \\n \\\n",
    "     A: 4/10 chance of $2.00 and an 6/10 chance of $1.60 \\n \\\n",
    "     B: 4/10 chance of $3.85 and an 6/10 chance of $0.10',\n",
    "    'If you are presented with the following two choices, which one would you take? Please highlight the choice with [] (such as [x]). \\n \\\n",
    "     A: 5/10 chance of $2.00 and an 5/10 chance of $1.60 \\n \\\n",
    "     B: 5/10 chance of $3.85 and an 5/10 chance of $0.10',\n",
    "    'If you are presented with the following two choices, which one would you take? Please highlight the choice with [] (such as [x]). \\n \\\n",
    "     A: 6/10 chance of $2.00 and an 4/10 chance of $1.60 \\n \\\n",
    "     B: 6/10 chance of $3.85 and an 4/10 chance of $0.10',\n",
    "    'If you are presented with the following two choices, which one would you take? Please highlight the choice with [] (such as [x]). \\n \\\n",
    "     A: 7/10 chance of $2.00 and an 3/10 chance of $1.60 \\n \\\n",
    "     B: 7/10 chance of $3.85 and an 3/10 chance of $0.10',\n",
    "    'If you are presented with the following two choices, which one would you take? Please highlight the choice with [] (such as [x]). \\n \\\n",
    "     A: 8/10 chance of $2.00 and an 2/10 chance of $1.60 \\n \\\n",
    "     B: 8/10 chance of $3.85 and an 2/10 chance of $0.10',\n",
    "    'If you are presented with the following two choices, which one would you take? Please highlight the choice with [] (such as [x]). \\n \\\n",
    "     A: 9/10 chance of $2.00 and an 1/10 chance of $1.60 \\n \\\n",
    "     B: 9/10 chance of $3.85 and an 1/10 chance of $0.10',\n",
    "    'If you are presented with the following two choices, which one would you take? Please highlight the choice with [] (such as [x]). \\n \\\n",
    "     A: 10/10 chance of $2.00 and an 0/10 chance of $1.60 \\n \\\n",
    "     B: 10/10 chance of $3.85 and an 0/10 chance of $0.10',\n",
    "]\n",
    "\n",
    "unit_test(n_instances=1, n_answers=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(prompts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_holt(instances, strict_format=True):\n",
    "    choices = []\n",
    "    # pattern = r'\\[(\\w)\\]'\n",
    "    for instance in instances:\n",
    "        choices_tmp = []\n",
    "        try:\n",
    "            for answer in instance[1:]:\n",
    "                answer = answer['message']\n",
    "                matches = extract_brackets(answer)\n",
    "                matches = [s for s in matches if s == 'A' or s == 'B']\n",
    "                # matches = re.findall(pattern, answer)\n",
    "                if strict_format and not \\\n",
    "                    len(matches) == 1: raise ValueError\n",
    "                elif len(matches) == 0: raise ValueError\n",
    "                choices_tmp.append(matches[0])\n",
    "            assert len(choices_tmp) == 10\n",
    "        except: continue\n",
    "        choices.append(choices_tmp)\n",
    "    return choices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instances = run_one_session(prompts, n_instances=30)\n",
    "choices = extract_holt(instances)\n",
    "ord_func = np.vectorize(ord)\n",
    "choices_array = ord_func(np.array(choices)) - ord('A')\n",
    "choices_array.mean(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_records(game_name='holt', to_save=[instances, choices])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.colors import ListedColormap\n",
    "\n",
    "arr = choices_array.astype(bool)\n",
    "\n",
    "colors = ['blue', 'orange']\n",
    "cmap = ListedColormap(colors)\n",
    "\n",
    "plt.imshow(arr, cmap=cmap, aspect=0.5)\n",
    "\n",
    "legend = plt.legend(handles=[plt.Rectangle((0,0),1,1, color=colors[0]), \n",
    "                             plt.Rectangle((0,0),1,1, color=colors[1])], \n",
    "                    labels=['A', 'B'], loc='upper left', bbox_to_anchor=(-0.25, 1))\n",
    "legend.set_title('Legend')\n",
    "\n",
    "plt.xticks(np.arange(arr.shape[1]), labels=list(range(1,11)))\n",
    "plt.yticks([])\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Beauty Contest (8 sessions, 5 prompts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts_init = [\n",
    "    'Hi, let’s play a game.',\n",
    "    'This is a number-guessing game where you will play with 4 other players. Each player chooses an integer between 0 and 100 which is hidden from other players. The player who chooses a number that is closest to ⅔ of the average wins the game. For example, if the average of the guesses is 60 and if you guess 40, i.e. ⅔ of 60, you win the game.\\n \\\n",
    "     What number would you like to guess? Please give a concrete number and highlight it with [] (e.g., [x]).',\n",
    "]\n",
    "\n",
    "prompt_2_congrats = lambda x: 'Congratulations! You won the game by guessing %i. The group average is 34.6, and ⅔ of that is 23.1. You guessed the closest to that number. Let’s play the game again, what number would you like to guess?  Please give a concrete number and highlight it with [] (e.g., [x]).' % x\n",
    "\n",
    "prompt_2_sorry = lambda x: 'Sorry! You didn’t win the game by guessing %i. The group average is 34.6, and ⅔ of that is 23.1. Your guess is NOT the closest to that number. Let’s play again. What number would you like to guess? Please give a concrete number and highlight it with [] (e.g., [x]).' % x\n",
    "\n",
    "prompt_3_congrats = lambda x: 'Congratulations! You won the game by guessing %i. The group average is 24.8, and ⅔ of that is 16.5. You guessed the closest to that number. Let’s play the game again, what number would you like to guess?  Please give a concrete number and highlight it with [] (e.g., [x]).' % x\n",
    "\n",
    "prompt_3_sorry = lambda x: 'Sorry! You didn’t win the game by guessing %i. The group average is 24.8, and ⅔ of that is 16.5. Your guess is NOT the closest to that number. Let’s play again. What number would you like to guess? Please give a concrete number and highlight it with [] (e.g., [x]).' % x\n",
    "\n",
    "prompt_endgame_congrats = lambda x: 'Congratulations! You won the game by guessing %i. The group average is 18.5, and ⅔ of that is 12.3. You guessed the closest to that number. Thank you for playing the game! Any final thoughts about this game?' % x\n",
    "\n",
    "prompt_endgame_sorry = lambda x: 'Sorry! You didn’t win the game by guessing %i. The group average is 18.5, and ⅔ of that is 12.3. Your guess is NOT the closest to that number. Thank you for playing the game! Any final thoughts about this game? ' % x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_beauty(\n",
    "    n_instances=80,\n",
    "    print_except=True\n",
    "):\n",
    "    for i, _ in tqdm(enumerate(range(n_instances))):\n",
    "        answers = []\n",
    "        scenarios_tmp = []\n",
    "        choices_tmp = []\n",
    "        try:\n",
    "            chatbot = Chatbot(config=config)\n",
    "            answers = []\n",
    "            \n",
    "            # round 1\n",
    "            for prompt in prompts_init:\n",
    "                answers.append(get_answer(chatbot, prompt))\n",
    "\n",
    "            amount = extract_amout(answers[-1], type=float)\n",
    "            if amount is None: raise ValueError('invalid answer')\n",
    "            choices_tmp.append(amount)\n",
    "\n",
    "            # round 2\n",
    "            if i % 2 == 1: \n",
    "                scenarios_tmp.append(1)\n",
    "                answers.append(get_answer(chatbot, prompt_2_congrats(amount)))\n",
    "            else: \n",
    "                scenarios_tmp.append(0)\n",
    "                answers.append(get_answer(chatbot, prompt_2_sorry(amount)))\n",
    "\n",
    "            amount = extract_amout(answers[-1], type=float)\n",
    "            choices_tmp.append(amount)\n",
    "\n",
    "            # round 3\n",
    "            if (i // 2) % 2 == 1: \n",
    "                scenarios_tmp.append(1)\n",
    "                answers.append(get_answer(chatbot, prompt_3_congrats(amount)))\n",
    "            else: \n",
    "                scenarios_tmp.append(0)\n",
    "                answers.append(get_answer(chatbot, prompt_3_sorry(amount)))\n",
    "\n",
    "            amount = extract_amout(answers[-1], type=float)\n",
    "            choices_tmp.append(amount)\n",
    "\n",
    "            # endgame\n",
    "            if (i // 4) % 2 == 1: \n",
    "                scenarios_tmp.append(1)\n",
    "                answers.append(get_answer(chatbot, prompt_endgame_congrats(amount)))\n",
    "            else: \n",
    "                scenarios_tmp.append(0)\n",
    "                answers.append(get_answer(chatbot, prompt_endgame_sorry(amount)))\n",
    "            \n",
    "        except Exception as e: \n",
    "            if print_except: print(e)\n",
    "            continue\n",
    "        instances.append(answers)\n",
    "        scenarios.append(scenarios_tmp)\n",
    "        choices.append(choices_tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "instances = []\n",
    "scenarios = []\n",
    "choices = []\n",
    "run_beauty(n_instances=80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_records(game_name='beauty_free', to_save=[instances, scenarios, choices])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_names = [\n",
    "    'beauty_plus_2023_02_27-01_23_09_AM.json',\n",
    "    'beauty_plus_2023_02_27-09_28_26_AM.json',\n",
    "]\n",
    "\n",
    "# file_names = [\n",
    "#     'beauty_free_2023_02_27-01_41_15_AM.json',\n",
    "#     'beauty_free_2023_02_27-05_13_35_AM.json',\n",
    "#     'beauty_free_2023_02_27-09_51_36_AM.json',\n",
    "# ]\n",
    "\n",
    "scenarios = []\n",
    "choices = []\n",
    "for file_name in file_names:\n",
    "    with open(os.path.join('records', file_name), 'r') as f:\n",
    "        records = json.load(f)\n",
    "        scenarios += records[-2]\n",
    "        choices += records[-1]\n",
    "\n",
    "assert len(scenarios) == len(choices)\n",
    "print('loaded %i valid records' % len(scenarios))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix_to_choice = defaultdict(list)\n",
    "prefix_to_pattern = defaultdict(Counter)\n",
    "for scenarios_tmp, choices_tmp in zip(scenarios, choices):\n",
    "    choices_tmp = [round(x, 1) for x in choices_tmp]\n",
    "\n",
    "    result = 0\n",
    "    for i, scenario in enumerate(scenarios_tmp):\n",
    "        prefix = tuple(scenarios_tmp[:i])\n",
    "        choice = choices_tmp[i]\n",
    "        prefix_to_choice[prefix].append(choice)\n",
    "        prefix_to_pattern[prefix][tuple(choices_tmp)] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted({prefix: sorted(counter.items(), key=lambda item: item[1], reverse=True) for prefix, counter in prefix_to_pattern.items()}.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_var_func = lambda x: pd.Series([len(x), np.mean(x), np.std(x), stats.mode(x)])\n",
    "\n",
    "df_choices = pd.DataFrame(list(prefix_to_choice.items()), columns=['Prefix', 'Choice'])\n",
    "df_choices[['#samples', 'mean', 'std', 'mode']] = df_choices['Choice'].apply(mean_var_func)\n",
    "df_choices.sort_values('Prefix')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chatgpt",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9 (main, Jan 11 2023, 09:18:18) [Clang 14.0.6 ]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "0724a2e07c0d614e836da95ca6039b23bcd016e1361c95d842049edda6416a68"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}