summarize_videos_clean.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import re\n",
    "import pathlib\n",
    "import openai\n",
    "import os\n",
    "path = pathlib.Path(\"path_to_good_stuff\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "from yt_dlp import YoutubeDL\n",
    "\n",
    "URLS = ['channel, video, or playlist urls']\n",
    "with YoutubeDL(params={'format': '140', \"paths\": {\"home\": path.as_posix()}}\n",
    "               ) as ydl:\n",
    "    ydl.download(URLS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline\n",
    "\n",
    "model_id = \"distil-whisper/distil-large-v2\"\n",
    "torch_dtype = torch.float16\n",
    "device = \"cuda:0\"\n",
    "\n",
    "model = AutoModelForSpeechSeq2Seq.from_pretrained(\n",
    "    model_id, torch_dtype=torch_dtype, \n",
    "    low_cpu_mem_usage=True, use_safetensors=True, \n",
    "    use_flash_attention_2=False\n",
    "    ).to_bettertransformer()\n",
    "model.to(device)\n",
    "\n",
    "processor = AutoProcessor.from_pretrained(model_id)\n",
    "\n",
    "pipe = pipeline(\"automatic-speech-recognition\",\n",
    "                model=model,\n",
    "                tokenizer=processor.tokenizer,\n",
    "                feature_extractor=processor.feature_extractor,\n",
    "                torch_dtype=torch_dtype,\n",
    "                device=device)\n",
    "\n",
    "for fname in path.glob(\"*.m4a\"):\n",
    "    if fname.with_suffix(\".txt\").exists():\n",
    "        continue\n",
    "    \n",
    "    print(fname)\n",
    "    outputs = pipe(fname.as_posix(),\n",
    "                chunk_length_s=15,\n",
    "                batch_size=32,\n",
    "                return_timestamps=True)\n",
    "\n",
    "    text = outputs[\"text\"]\n",
    "    #chunks = [text[i:i+6000] for i in range(0, len(text), 6000)]\n",
    "\n",
    "    with open(fname.with_suffix(\".txt\"), \"w\") as f:\n",
    "        #for c in chunks:\n",
    "        #    f.write(c)\n",
    "        #    f.write(\"\\n\\n\\n\")\n",
    "        f.write(text)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "i = 0\n",
    "for fname in path.glob(\"*.txt\"):\n",
    "    if (path / \"summaries\" / fname.name).exists():\n",
    "        continue\n",
    "    if i > 4:\n",
    "        break\n",
    "    print(fname)\n",
    "    transcript = fname.read_text().strip()\n",
    "    title = re.sub(r'\\[.*?\\]', '', fname.stem).strip()\n",
    "    \n",
    "    \n",
    "    system_msg = f\"\"\"Give me a detailed summary of the following transcript. The title is: {title}\"\"\"\n",
    "    #print(system_msg)\n",
    "    prompt = f\"\"\"TRANSCRIPT:\\n\\n{transcript}\"\"\"\n",
    "\n",
    "    completion = openai.ChatCompletion.create(\n",
    "        model=\"gpt-4-1106-preview\",\n",
    "        temperature=0.0,\n",
    "        max_tokens=1000,\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": system_msg},\n",
    "            {\"role\": \"user\", \"content\": prompt},\n",
    "        ],\n",
    "    )\n",
    "\n",
    "    summary = completion['choices'][0].message['content']\n",
    "\n",
    "\n",
    "    with open(path / \"summaries\" / fname.name, \"w\") as f:\n",
    "        f.write(summary)\n",
    "    i += 1    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "seo",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}