AI_interact.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "20ec0368-ac52-435a-abd4-b7f5cdeb757a",
   "metadata": {},
   "source": [
    "## Install necessary packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f6e36d8-e3c1-4576-8a95-8f0dd46b99d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install pillow pytesseract pyautogui pynput keyboard\n",
    "!brew install tesseract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9abf18d1-e57a-4623-99dd-32b8afaa30f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image, ImageGrab\n",
    "import pytesseract\n",
    "import pyautogui\n",
    "import requests\n",
    "from pynput.mouse import Listener\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ef35885-0216-4a60-af73-5f42d3846344",
   "metadata": {},
   "source": [
    "## Screen capture part\n",
    "User can click on 2 positions on the screen and ImageGrab capture the rectangle defined by those coordinates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "298ebd56-e292-413b-a4b7-294c904a6073",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ClickListener:\n",
    "    def __init__(self):\n",
    "        self.coordinates = None\n",
    "        self.keep_listening = True\n",
    "\n",
    "    def on_click(self, x, y, button, pressed):\n",
    "        if pressed:\n",
    "            self.coordinates = int(x), int(y)\n",
    "            self.keep_listening = False\n",
    "            return False  # Stop the listener\n",
    "\n",
    "    def get_coordinates(self):\n",
    "        with Listener(on_click=self.on_click) as listener:\n",
    "            listener.join()  # This will block until on_click returns False\n",
    "        return self.coordinates\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "80440ffc-a465-45b9-88e1-bdea26721492",
   "metadata": {},
   "outputs": [],
   "source": [
    "def capture_screen():\n",
    "    # Create an instance of ClickListener\n",
    "    click_listener = ClickListener()\n",
    "    \n",
    "    # Get the mouse coordinates after the first click\n",
    "    x1, y1 = click_listener.get_coordinates()\n",
    "    x2, y2 = click_listener.get_coordinates()\n",
    "    \n",
    "    # Print the coordinates\n",
    "    # print(\"First click coordinates:\", x1, y1)\n",
    "    print(f'Rectangle: x1: {x1} y1: {y1} x2: {x2} y2: {y2}')\n",
    "    try:\n",
    "        image = ImageGrab.grab(bbox=(x1, y1, x2, y2))\n",
    "    except:\n",
    "        return None\n",
    "    return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0d99fe6b-6953-496a-b47f-b63f4c38720c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# image = capture_screen()\n",
    "# display(image)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45832ac8-6c64-4642-9f6c-36251e958df1",
   "metadata": {},
   "source": [
    "## Image to Text\n",
    "Use Tesseract to recognize text on the image and convert it to a default prompt.\n",
    "This prompt can be sent to the defined LLM to seek help. LLM can run local or can be a paid service like OpenAi or it can have an escalation from small local model -> organization hosted medium model -> very large LLM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b2572325-7939-4844-a7e0-c34653a019a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# text = pytesseract.image_to_string(image)\n",
    "# # And with this text you can call the LLM\n",
    "# PROMPT_DEFAULT = \"Help me with this: \\n\"\n",
    "# prompt = PROMPT_DEFAULT + text\n",
    "# print(prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "922adeec-9334-4a43-8265-7d6918a22277",
   "metadata": {},
   "source": [
    "## Integrate with LM Studio (https://lmstudio.ai/)\n",
    "- Start LM studio with a model of your choice locally.\n",
    "- Start the Local Inference Server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "fa609426-70cb-4035-865f-51b2613a1a3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def send_request(user_input):\n",
    "        url = f\"http://localhost:1234/v1/chat/completions\"\n",
    "        headers = {\"Content-Type\": \"application/json\"}\n",
    "        data = {\n",
    "            \"messages\": [\n",
    "                {\"role\": \"user\", \"content\": f\"### Instruction: {user_input}\\n###Response: \"}\n",
    "            ],\n",
    "            \"stop\": [\"### Instruction:\"],\n",
    "            \"temperature\": 0.7,\n",
    "            \"max_tokens\": -1,\n",
    "            \"stream\": False\n",
    "        }\n",
    "        response = requests.post(url, headers=headers, json=data)\n",
    "        response.raise_for_status()\n",
    "        return response.json()['choices'][0]['message']['content']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "15eb8dfe-b0ec-4f17-89f0-b5b0c1309581",
   "metadata": {},
   "outputs": [],
   "source": [
    "# response = send_request(prompt)\n",
    "# print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "566c9a15-137e-40db-8fc9-0197fc49257d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def help_me():\n",
    "    image = capture_screen()\n",
    "    if image != None:\n",
    "        text = pytesseract.image_to_string(image)\n",
    "        # PROMPT_DEFAULT = \"Summarize this: \\n\"\n",
    "        PROMPT_DEFAULT = \"Help me with this: \\n\"\n",
    "        prompt = PROMPT_DEFAULT + text\n",
    "        return send_request(prompt)\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "8c129be7-0795-4949-8b50-4e8d1cf88e8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# help_me()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a060f9b-4198-45c7-b448-fc9a6cb254f8",
   "metadata": {},
   "source": [
    "## Run a listener and activate capture and inference onde key combo ([Shift][A]) has pressed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f939fc8b-f3ba-4469-b80a-b5a2741de172",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "This process is not trusted! Input event monitoring will not be possible until it is added to accessibility clients.\n",
      "This process is not trusted! Input event monitoring will not be possible until it is added to accessibility clients.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{<Key.shift: <56>>}\n",
      "{'A', <Key.shift: <56>>}\n",
      "Make a screenshot: define the screen area by click 2 orcners of a triange\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "This process is not trusted! Input event monitoring will not be possible until it is added to accessibility clients.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rectangle: x1: 203 y1: 451 x2: 992 y2: 630\n"
     ]
    }
   ],
   "source": [
    "from pynput import keyboard\n",
    "COMBINATIONS = [\n",
    "    {keyboard.Key.shift, keyboard.KeyCode(char='a')},\n",
    "    {keyboard.Key.shift, keyboard.KeyCode(char='A')},\n",
    "    {keyboard.Key.shift, keyboard.KeyCode(char='x')},\n",
    "    {keyboard.Key.shift, keyboard.KeyCode(char='X')}\n",
    "]\n",
    "# The currently active modifiers\n",
    "current = set()\n",
    "def on_press(key):\n",
    "    if any([key in COMBO for COMBO in COMBINATIONS]):\n",
    "        current.add(key)\n",
    "        print(current)\n",
    "        # if any(all(k in current for k in COMBO) for COMBO in COMBINATIONS):\n",
    "        if any(x in current for x in {keyboard.KeyCode(char='A')}):\n",
    "            print('Make a screenshot: define the screen area by click 2 orcners of a triange')\n",
    "            print(help_me())\n",
    "        # if key == keyboard.Key.shift and any(x in current for x in {keyboard.KeyCode(char='X')}):\n",
    "        if any(x in current for x in {keyboard.KeyCode(char='X')}):\n",
    "            print('Shift + X Apressed. Stopping the listener.')\n",
    "            listener.stop()\n",
    "\n",
    "def on_release(key):\n",
    "    try:\n",
    "        if any([key in COMBO for COMBO in COMBINATIONS]):\n",
    "            current.remove(key)\n",
    "    except:\n",
    "        pass\n",
    "    # if key == keyboard.Key.shift and any(x in current for x in {keyboard.KeyCode(char='X')}):\n",
    "    #     print('Shift + X pressed. Stopping the listener.')\n",
    "    #     listener.stop()\n",
    "\n",
    "with keyboard.Listener(on_press=on_press, on_release=on_release) as listener:\n",
    "    listener.join()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63ec9eca-2b99-4dac-93a6-b3c7f94b8475",
   "metadata": {},
   "source": [
    "## "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f528e918-9b86-45e8-9d0a-0fdac87a4fb0",
   "metadata": {},
   "source": [
    "## "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff288d23-045c-48a1-aba8-9794a76d77c9",
   "metadata": {},
   "source": [
    "## This is just a dummy sandbox to provide some sort of GUI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6789256c-e26a-4697-8001-1652a6cfbbf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tkinter as tk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cca1ed01-2451-4fdd-96ee-de3269b91ced",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_to_new_window(text):\n",
    "    new_window = tk.Toplevel(root)\n",
    "    new_window.title(\"Prompt\")\n",
    "\n",
    "    text_label = tk.Label(new_window, text=text)\n",
    "    text_label.pack()\n",
    "\n",
    "# Create the main window\n",
    "root = tk.Tk()\n",
    "root.title(\"AI interact\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3e8eb06-32f2-4991-be74-8bc1cacb9a51",
   "metadata": {},
   "outputs": [],
   "source": [
    "print_to_new_window(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbbc3777-e848-4987-8be8-b417d936bbde",
   "metadata": {},
   "outputs": [],
   "source": [
    "root.mainloop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d17bc978-729d-47fc-bab0-a629217c7d8e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d365a1d1-44ee-4f4e-a1a3-fbae7550e184",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38fa6378-21f2-4a59-a8ff-7e4ed0b08145",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}