experiments/evaluate_lvlm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "import os\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"\" # Set your OpenAI API key here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import base64\n",
    "from PIL import Image\n",
    "import io\n",
    "question =\"\"\"The person is holding something in his hand. What will the person do next? Answer with a pair of actions and an object.\"\"\"\n",
    "\n",
    "# Read the JSON file\n",
    "with open(\"test/few_shot_samples.json\", \"r\") as f:\n",
    "    few_shot_samples = json.load(f)\n",
    "\n",
    "qa_examples = []\n",
    "i = 0\n",
    "for entity_set in few_shot_samples:\n",
    "    answer = entity_set[\"answer\"]\n",
    "    qa_examples.append({\n",
    "        \"input_path\": f\"./test/{i}.png\",\n",
    "        \"output\": answer\n",
    "    })\n",
    "    i += 1\n",
    "\n",
    "few_shot_messages = []\n",
    "for qa_example in qa_examples:\n",
    "    encoded_string = None\n",
    "    image = Image.open(qa_example['input_path'])\n",
    "    with open(qa_example['input_path'], 'rb') as f:\n",
    "        image = Image.open(f)\n",
    "        jpeg_buffer = io.BytesIO()\n",
    "        image.save(jpeg_buffer, format=\"JPEG\", quality=90)\n",
    "        encoded_string = base64.b64encode(jpeg_buffer.getvalue()).decode('utf-8')\n",
    "    \n",
    "    few_shot_messages.append({\n",
    "        \"role\": \"user\",\n",
    "        \"content\": [\n",
    "            {\"type\": \"text\", \"text\":question},\n",
    "            {\"type\": \"image_url\",\"image_url\": {\"url\": f\"data:image/jpeg;base64,\" + encoded_string, \"detail\": \"low\"}},\n",
    "        ],    \n",
    "    })\n",
    "    few_shot_messages.append({\n",
    "        \"role\": \"assistant\",\n",
    "        \"content\":[{\"type\": \"text\", \"text\": qa_example['output']}]\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"test/ground_truth.json\", \"r\") as f:\n",
    "    ground_truth = json.load(f)\n",
    "\n",
    "results = []\n",
    "\n",
    "i = len(few_shot_samples)\n",
    "for entity_set in ground_truth:\n",
    "    image_path = \"test/\" + str(i) + \".png\"\n",
    "    encoded_string = None\n",
    "    image = Image.open(image_path)\n",
    "    with open(qa_example['input_path'], 'rb') as f:\n",
    "        image = Image.open(f)\n",
    "        jpeg_buffer = io.BytesIO()\n",
    "        image.save(jpeg_buffer, format=\"JPEG\", quality=90)\n",
    "        encoded_string = base64.b64encode(jpeg_buffer.getvalue()).decode('utf-8')\n",
    "        \n",
    "    client = OpenAI()\n",
    "    response = client.chat.completions.create(\n",
    "        model=\"gpt-4o\",\n",
    "        messages= few_shot_messages + [\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\":[\n",
    "                    {\"type\": \"text\", \"text\": question},\n",
    "                    {\"type\": \"image_url\",\"image_url\": {\n",
    "                        \"url\": f\"data:image/jpeg;base64,\"+encoded_string, \n",
    "                        \"detail\":\"low\"\n",
    "                    }},\n",
    "                ],\n",
    "            },\n",
    "        ],\n",
    "        max_tokens=50,\n",
    "    )\n",
    "    \n",
    "    results.append(response.choices[0].message.content)\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "answers = [x[\"answer\"] for x in ground_truth]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "# nltk.download('punkt')\n",
    "from nltk.translate import bleu\n",
    "from nltk import word_tokenize\n",
    "from rouge_score.rouge_scorer import RougeScorer\n",
    "from nltk.translate import meteor\n",
    "\n",
    "sum_bleu = 0\n",
    "sum_rouge1 = 0\n",
    "sum_rouge2 = 0\n",
    "sum_rougeL = 0\n",
    "sum_meteor = 0\n",
    "for result, answer in zip(results, answers):\n",
    "    # Calculate BLEU score\n",
    "    result_token = word_tokenize(result)\n",
    "    answer_token = word_tokenize(answer)\n",
    "    print()\n",
    "    bleu_score = bleu(\n",
    "        [result.split()], \n",
    "        answer.split(),\n",
    "        (1,),\n",
    "        )\n",
    "\n",
    "    # Calculate ROUGE score\n",
    "    scorer = RougeScorer([\"rouge1\", \"rouge2\", \"rougeL\", \"rougeLsum\"])\n",
    "    rouge_scores = scorer.score(result, answer)\n",
    "\n",
    "    # Calculate METEOR score\n",
    "    meteor_score = round(meteor(\n",
    "        [result_token],\n",
    "        answer_token,\n",
    "        ), 4)\n",
    "\n",
    "    # Print the scores\n",
    "    sum_bleu += bleu_score\n",
    "    sum_rouge1 += rouge_scores[\"rouge1\"].fmeasure\n",
    "    sum_rouge2 += rouge_scores[\"rouge2\"].fmeasure\n",
    "    sum_rougeL += rouge_scores[\"rougeL\"].fmeasure\n",
    "    sum_meteor += meteor_score\n",
    "\n",
    "print(f\"BLEU: {sum_bleu/len(results)}\")\n",
    "print(f\"ROUGE-1: {sum_rouge1/len(results)}\")\n",
    "print(f\"ROUGE-2: {sum_rouge2/len(results)}\")\n",
    "print(f\"ROUGE-L: {sum_rougeL/len(results)}\")\n",
    "print(f\"METEOR: {sum_meteor/len(results)}\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}