watch-api.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Assume we've already made the summaries\n",
    "- So we only pass in the recent file changes, and ask it to autocomplete\n",
    "- The response schema is the same though\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from groq import Groq\n",
    "import json\n",
    "import os\n",
    "import watchdog\n",
    "import asyncio\n",
    "from src.loader import get_dir_summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"GROQ_API_KEY\"] = \"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\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from watchdog.observers import Observer\n",
    "from watchdog.events import FileSystemEvent, FileSystemEventHandler\n",
    "\n",
    "\n",
    "class Watcher:\n",
    "    def __init__(self, base_path, callback):\n",
    "        self.observer = Observer()\n",
    "        self.base_path = base_path\n",
    "        self.callback = callback\n",
    "\n",
    "    def run(self):\n",
    "        event_handler = Handler(self.base_path, self.callback)\n",
    "        self.observer.schedule(event_handler, self.base_path, recursive=True)\n",
    "        self.observer.start()\n",
    "        try:\n",
    "            while True:\n",
    "                time.sleep(5)\n",
    "        except KeyboardInterrupt:\n",
    "            self.observer.stop()\n",
    "            print(f\"Observer on directory {self.base_path} stopped\")\n",
    "        self.observer.join()\n",
    "\n",
    "\n",
    "class Handler(FileSystemEventHandler):\n",
    "    def __init__(self, base_path, callback):\n",
    "        self.base_path = base_path\n",
    "        self.callback = callback\n",
    "        self.events = []\n",
    "\n",
    "        print(f\"Watching directory {base_path}\")\n",
    "\n",
    "        # Hack to get async function to work\n",
    "        loop = asyncio.new_event_loop()\n",
    "        # Ensure the async setup is run thread-safely\n",
    "        future = loop.run_until_complete(get_doc_summaries(base_path))\n",
    "        self.summaries = future.result()  # This will block until the coroutine is done\n",
    "        print(self.summaries)\n",
    "\n",
    "    def on_created(self, event: FileSystemEvent) -> None:\n",
    "        src_path = os.path.relpath(event.src_path, self.base_path)\n",
    "        print(f\"Created {src_path}\")\n",
    "        # self.callback(event.src_path)\n",
    "\n",
    "    def on_deleted(self, event: FileSystemEvent) -> None:\n",
    "        src_path = os.path.relpath(event.src_path, self.base_path)\n",
    "        print(f\"Deleted {src_path}\")\n",
    "        # self.callback(event.src_path)\n",
    "\n",
    "    def on_moved(self, event: FileSystemEvent) -> None:\n",
    "        src_path = os.path.relpath(event.src_path, self.base_path)\n",
    "        dest_path = os.path.relpath(event.dest_path, self.base_path)\n",
    "        print(f\"Moved {src_path} > {dest_path}\")\n",
    "        self.events.append({\"src_path\": src_path, \"dst_path\": dest_path})\n",
    "        self.callback(self.summaries, {\"files\": self.events})\n",
    "\n",
    "    # def on_any_event(self, event):\n",
    "    #     print(event)\n",
    "    #     print(event.event_type, event.src_path, event.dest_path)\n",
    "    #     if not event.is_directory:\n",
    "    #         self.callback(event.src_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# BASE_PATH = \"./sample_data\"\n",
    "# summaries = await get_doc_summaries(BASE_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "FILE_PROMPT = \"\"\"\n",
    "You will be provided with list of source files and a summary of their contents. For each file, propose a new path and filename, using a directory structure that optimally organizes the files using known conventions and best practices.\n",
    "\n",
    "If the file is already named well or matches a known convention, set the destination path to the same as the source path.\n",
    "\n",
    "Your response must be a JSON object with the following schema:\n",
    "```json\n",
    "{\n",
    "    \"files\": [\n",
    "        {\n",
    "            \"src_path\": \"original file path\",\n",
    "            \"dst_path\": \"new file path under proposed directory structure with proposed file name\"\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "```\n",
    "\"\"\".strip()\n",
    "\n",
    "\n",
    "WATCH_PROMPT = \"\"\"\n",
    "Here are a few examples of good file naming conventions to emulate, based on the files provided:\n",
    "\n",
    "```json\n",
    "{fs_events}\n",
    "```\n",
    "\n",
    "Include the above items in your response exactly as is, along all other proposed changes.\n",
    "\"\"\".strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_file_tree(summaries, fs_events):\n",
    "    client = Groq()\n",
    "    cmpl = client.chat.completions.create(\n",
    "        messages=[\n",
    "            {\"content\": FILE_PROMPT, \"role\": \"system\"},\n",
    "            {\"content\": json.dumps(summaries), \"role\": \"user\"},\n",
    "            {\"content\": WATCH_PROMPT.format(fs_events), \"role\": \"system\"},\n",
    "            {\"content\": json.dumps(summaries), \"role\": \"user\"},\n",
    "        ],\n",
    "        model=\"llama3-70b-8192\",\n",
    "        response_format={\"type\": \"json_object\"},\n",
    "        temperature=0,\n",
    "    )\n",
    "    print(cmpl.choices[0].message.content)\n",
    "    return json.loads(cmpl.choices[0].message.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Watching directory ./sample_data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/iyaja/mambaforge/lib/python3.9/ast.py:50: RuntimeWarning: coroutine 'get_doc_summaries' was never awaited\n",
      "  return compile(source, filename, mode, flags,\n",
      "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "Cannot run the event loop while another loop is running",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Input \u001b[0;32mIn [44]\u001b[0m, in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m w \u001b[38;5;241m=\u001b[39m Watcher(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./sample_data\u001b[39m\u001b[38;5;124m\"\u001b[39m, create_file_tree)\n\u001b[0;32m----> 2\u001b[0m \u001b[43mw\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [40]\u001b[0m, in \u001b[0;36mWatcher.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrun\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m---> 13\u001b[0m     event_handler \u001b[38;5;241m=\u001b[39m \u001b[43mHandler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbase_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcallback\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     14\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mobserver\u001b[38;5;241m.\u001b[39mschedule(event_handler, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbase_path, recursive\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m     15\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mobserver\u001b[38;5;241m.\u001b[39mstart()\n",
      "Input \u001b[0;32mIn [40]\u001b[0m, in \u001b[0;36mHandler.__init__\u001b[0;34m(self, base_path, callback)\u001b[0m\n\u001b[1;32m     34\u001b[0m loop \u001b[38;5;241m=\u001b[39m asyncio\u001b[38;5;241m.\u001b[39mnew_event_loop()\n\u001b[1;32m     35\u001b[0m \u001b[38;5;66;03m# Ensure the async setup is run thread-safely\u001b[39;00m\n\u001b[0;32m---> 36\u001b[0m future \u001b[38;5;241m=\u001b[39m \u001b[43mloop\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_until_complete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mget_doc_summaries\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbase_path\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     37\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msummaries \u001b[38;5;241m=\u001b[39m future\u001b[38;5;241m.\u001b[39mresult()  \u001b[38;5;66;03m# This will block until the coroutine is done\u001b[39;00m\n\u001b[1;32m     38\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msummaries)\n",
      "File \u001b[0;32m~/mambaforge/lib/python3.9/asyncio/base_events.py:623\u001b[0m, in \u001b[0;36mBaseEventLoop.run_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m    612\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Run until the Future is done.\u001b[39;00m\n\u001b[1;32m    613\u001b[0m \n\u001b[1;32m    614\u001b[0m \u001b[38;5;124;03mIf the argument is a coroutine, it is wrapped in a Task.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    620\u001b[0m \u001b[38;5;124;03mReturn the Future's result, or raise its exception.\u001b[39;00m\n\u001b[1;32m    621\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    622\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_closed()\n\u001b[0;32m--> 623\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_running\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    625\u001b[0m new_task \u001b[38;5;241m=\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m futures\u001b[38;5;241m.\u001b[39misfuture(future)\n\u001b[1;32m    626\u001b[0m future \u001b[38;5;241m=\u001b[39m tasks\u001b[38;5;241m.\u001b[39mensure_future(future, loop\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m)\n",
      "File \u001b[0;32m~/mambaforge/lib/python3.9/asyncio/base_events.py:585\u001b[0m, in \u001b[0;36mBaseEventLoop._check_running\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    583\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThis event loop is already running\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m    584\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m events\u001b[38;5;241m.\u001b[39m_get_running_loop() \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 585\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m    586\u001b[0m         \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mCannot run the event loop while another loop is running\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Cannot run the event loop while another loop is running"
     ]
    }
   ],
   "source": [
    "w = Watcher(\"./sample_data\", create_file_tree)\n",
    "w.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}