5 - Large Text with Langchan and Upstash Vector Store.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "307a9639-3143-4a1b-904e-3b95787740e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %pip install wikipedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fca2a8fc-5af1-4ca0-8a89-1feb9513a3f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from decouple import config\n",
    "from openai import OpenAI\n",
    "\n",
    "OPENAI_API_KEY = config(\"OPENAI_API_KEY\")\n",
    "UPSTASH_VECTOR_REST_URL = config(\"UPSTASH_VECTOR_REST_URL\")\n",
    "UPSTASH_VECTOR_REST_TOKEN = config(\"UPSTASH_VECTOR_REST_TOKEN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f998c68a-f4a7-4845-a50b-7ddc1c3beeb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['OPENAI_API_KEY'] = OPENAI_API_KEY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "653872ab-5628-4668-8c13-a6d109a3df5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import wikipedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f5c628bb-084a-451a-82f4-6e4099d24f98",
   "metadata": {},
   "outputs": [],
   "source": [
    "ny = wikipedia.page(title=\"New York City, New York\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "321c545d-a894-4151-9495-0392e0028243",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ny.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "86800cf5-f88f-41cb-952e-bbb9953df6f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://python.langchain.com/v0.2/docs/integrations/vectorstores/upstash/\n",
    "# pip install langchain-openai langchain langchain-community upstash-vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fbc7e953-fda7-4a4b-8cf7-a4855265a714",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_openai import OpenAIEmbeddings\n",
    "\n",
    "# dim -> 1536\n",
    "embeddings = OpenAIEmbeddings(model=\"text-embedding-3-small\")\n",
    "\n",
    "# dim -> 3072\n",
    "# embeddings = OpenAIEmbeddings(model=\"text-embedding-3-large\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ba786edb-8841-4418-9fa4-52601599bc44",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.vectorstores import UpstashVectorStore\n",
    "\n",
    "store = UpstashVectorStore(\n",
    "    embedding=embeddings,\n",
    "    index_url=UPSTASH_VECTOR_REST_URL,\n",
    "    index_token=UPSTASH_VECTOR_REST_TOKEN\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a19460cf-f148-4812-9c19-083ee8124013",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Boise, Idaho',\n",
       " 'Boise State University',\n",
       " 'Boise metropolitan area',\n",
       " 'Boise County, Idaho',\n",
       " 'Boise Airport',\n",
       " 'Boise State Broncos football',\n",
       " 'List of mayors of Boise, Idaho',\n",
       " 'Idaho',\n",
       " 'List of people from Boise, Idaho',\n",
       " 'Boise State Broncos']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wikipedia.search(\"Boise, Idaho\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3d6297e3-5546-4c32-8875-436a590d2dbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.documents import Document\n",
    "\n",
    "documents = []\n",
    "cities = [\"New York City, New York\", \"Boise, Idaho\"]\n",
    "for city in cities:\n",
    "    wikipedia_page_result = wikipedia.page(title=city)\n",
    "    doc = Document(\n",
    "        page_content=wikipedia_page_result.content,\n",
    "        metadata={\n",
    "            \"source\": f\"{wikipedia_page_result.url}\",\n",
    "            \"title\": city,\n",
    "        }\n",
    "    )\n",
    "    documents.append(doc)\n",
    "    # raw_documents.append(page.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "174523aa-ca13-413b-9d43-567d64718212",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'source': 'https://en.wikipedia.org/wiki/New_York_City',\n",
       " 'title': 'New York City, New York'}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents[0].metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "989e8c85-ef5b-4a9a-a3b6-bad4a540631b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8fdcc9b1-d507-45ee-841a-99835bcbdb6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tiktoken in ./venv/lib/python3.12/site-packages (0.7.0)\n",
      "Requirement already satisfied: regex>=2022.1.18 in ./venv/lib/python3.12/site-packages (from tiktoken) (2024.7.24)\n",
      "Requirement already satisfied: requests>=2.26.0 in ./venv/lib/python3.12/site-packages (from tiktoken) (2.32.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in ./venv/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in ./venv/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in ./venv/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken) (2.2.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in ./venv/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken) (2024.7.4)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install tiktoken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "43868bbb-de27-4add-a464-ba143073a435",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import TextLoader\n",
    "from langchain_text_splitters import TokenTextSplitter\n",
    "\n",
    "OPENAI_LLM_MODEL = \"gpt-4o\"\n",
    "text_splitter = TokenTextSplitter.from_tiktoken_encoder(\n",
    "    model_name=OPENAI_LLM_MODEL, chunk_size=100, chunk_overlap=0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "65729c1a-4722-44ca-988e-9c794187b90a",
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = text_splitter.split_documents(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2bb53b05-40ee-43b0-9b72-0cbdcecbfd9b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "312"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ede6d035-a716-499e-9e74-ca610c60ffa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "inserted_vectors = store.add_documents(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "009c2c62-e7d9-487f-bbca-3532681e349c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# result = store.similarity_search(\"The city with big buildings\", k=5)\n",
    "# result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a5bb8246-a959-4479-b8de-7c92b36af599",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'source': 'https://en.wikipedia.org/wiki/Boise,_Idaho', 'title': 'Boise, Idaho'} - 0.749708\n",
      "{'source': 'https://en.wikipedia.org/wiki/Boise,_Idaho', 'title': 'Boise, Idaho'} - 0.7259105\n"
     ]
    }
   ],
   "source": [
    "result = store.similarity_search_with_score(\"The city named after trees\", k=2)\n",
    "for doc, score in result:\n",
    "    print(f\"{doc.metadata} - {score}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "055a0b9a-b1f6-4218-b276-5374df7b81de",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}