langchain_pdf_loader.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMaDmtAC/tobGsdd9ob0iPl",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/vyz1194/vyz/blob/master/langchain_pdf_loader.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install unstructured"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jbxky3xc6HVw",
        "outputId": "fe3a9b9c-7b67-4eb5-ff29-bcebe41139f7"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting unstructured\n",
            "  Downloading unstructured-0.9.2-py3-none-any.whl (1.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: chardet in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.0.0)\n",
            "Collecting filetype (from unstructured)\n",
            "  Downloading filetype-1.2.0-py2.py3-none-any.whl (19 kB)\n",
            "Collecting python-magic (from unstructured)\n",
            "  Downloading python_magic-0.4.27-py2.py3-none-any.whl (13 kB)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.9.3)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.8.1)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.9.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.31.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (8.1.6)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (1.3.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (2023.6.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (4.66.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2023.7.22)\n",
            "Installing collected packages: filetype, python-magic, unstructured\n",
            "Successfully installed filetype-1.2.0 python-magic-0.4.27 unstructured-0.9.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install pdfminer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mNu4lsKE6eun",
        "outputId": "3f93a420-08bb-4bf6-e188-fbe164547806"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pdfminer\n",
            "  Downloading pdfminer-20191125.tar.gz (4.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.2/4.2 MB\u001b[0m \u001b[31m21.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pycryptodome (from pdfminer)\n",
            "  Downloading pycryptodome-3.18.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m39.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: pdfminer\n",
            "  Building wheel for pdfminer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pdfminer: filename=pdfminer-20191125-py3-none-any.whl size=6140093 sha256=fd1b9390cddd9bd4dfebe31511febf498ac38b05e5d383e98b1f15caa404d5f5\n",
            "  Stored in directory: /root/.cache/pip/wheels/4e/c1/68/f7bd0a8f514661f76b5cbe3b5f76e0033d79f1296012cbbf72\n",
            "Successfully built pdfminer\n",
            "Installing collected packages: pycryptodome, pdfminer\n",
            "Successfully installed pdfminer-20191125 pycryptodome-3.18.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install pdf2image"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p_tjITMK6ZTN",
        "outputId": "88fd8cff-7df8-4e0e-9a73-5b22776243e5"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pdf2image\n",
            "  Downloading pdf2image-1.16.3-py3-none-any.whl (11 kB)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from pdf2image) (9.4.0)\n",
            "Installing collected packages: pdf2image\n",
            "Successfully installed pdf2image-1.16.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install pdfminer.six"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "id": "q79sQi5Z60Be",
        "outputId": "9ae1fdd3-406c-4c9e-f0fc-487af076fe63"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pdfminer.six\n",
            "  Downloading pdfminer.six-20221105-py3-none-any.whl (5.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six) (3.2.0)\n",
            "Requirement already satisfied: cryptography>=36.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six) (41.0.3)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/dist-packages (from cryptography>=36.0.0->pdfminer.six) (1.15.1)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six) (2.21)\n",
            "Installing collected packages: pdfminer.six\n",
            "Successfully installed pdfminer.six-20221105\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "pdfminer"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import UnstructuredPDFLoader"
      ],
      "metadata": {
        "id": "cY5qRCA75UNb"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loader = UnstructuredPDFLoader(\"/content/sample.pdf\")"
      ],
      "metadata": {
        "id": "TzzX3_2a54z4"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = loader.load()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VdETmVhG5-iO",
        "outputId": "bf17b781-896b-499a-f720-1f70e2bd293c"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data[0].page_content"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "O3_tlS3z6RNq",
        "outputId": "55d4f059-d654-4015-e0d8-9313b92e7750"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Sample PDF Document\\n\\nRobert Maron Grzegorz Grudzi´nski\\n\\nFebruary 20, 1999\\n\\n2\\n\\nContents\\n\\n1 Template\\n\\n5 5 5 5 6 6 6 6 7 7 7 7 7\\n\\n1.1 How to compile a .tex ﬁle to a .pdf ﬁle . . 1.1.1 Tools . . . . . . . 1.1.2 How to use the tools .\\n\\n. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\\n\\n. . . .\\n\\n. . . . . . 1.2.1 The main document . . 1.2.2 Chapters 1.2.3\\n\\n.\\n\\n. . . . . . . . . . . . . . . . . . . . . . .\\n\\n. . . . . . . . . . . . . . . . . . . . . . . .\\n\\n. . .\\n\\n1.2 How to write a document\\n\\n. . . . . Spell-checking . .\\n\\n. . .\\n\\n. .\\n\\n. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\\n\\n. . . .\\n\\n1.3 LATEX and pdfLATEX capabilities . 1.3.1 Overview . . . . . . . 1.3.2 LATEX . . . . . 1.3.3 pdfLATEX . . . . . 1.3.4 Examples . . . . .\\n\\n. . . . . . . . . . . . . . .\\n\\n. . . . . . . . . .\\n\\n.\\n\\n. . .\\n\\n. .\\n\\n3\\n\\n4\\n\\nCONTENTS\\n\\nChapter 1\\n\\nTemplate\\n\\n1.1 How to compile a .tex ﬁle to a .pdf ﬁle\\n\\n1.1.1 Tools\\n\\nTo process the ﬁles you (may) need:\\n\\npdflatex (for example from tetex package ≥ 0.9-6, which you can\\n\\nget from Red Hat 5.2);\\n\\nacroread (a PDF viewer, available from http://www.adobe.com/);\\n\\nghostscript ≥ 5.10 (for example from Red Hat Contrib) and ghostview\\n\\nor gv (from RedHat Linux);\\n\\nefax package could be useful, if you plan to fax documents.\\n\\n1.1.2 How to use the tools\\n\\nFollow these steps:\\n\\n1. put all source .tex ﬁles in one directory, then chdir to the directory (or put some of them in the LATEXsearch path — if you know how to do this);\\n\\n2. run “pdflatex file.tex” on the main ﬁle of the document three times\\n\\n(three — to prepare valid table of contents);\\n\\n3. to see or print the result use acroread (unfortunately some versions of\\n\\nacroread may produce PostScript which is too complex), or\\n\\n5\\n\\n6\\n\\nCHAPTER1. TEMPLATE\\n\\n4. run ghostscript: “gv file.pdf” to display or:\\n\\n“gs -dNOPAUSE -sDEVICE=pswrite -q -dBATCH -sOutputFile=ﬁle.ps ﬁle.pdf” to produce a PostScript ﬁle;\\n\\n5. run “fax send phone-number file.ps” as root to send a fax, or — if you know how to do this — modify the fax script to be able to fax .pdf ﬁles directly (you have to insert “|%PDF*” somewhere. . . ).\\n\\n1.2 How to write a document\\n\\n1.2.1 The main document\\n\\nChoose the name of the document, say document. Copy template.tex to document.tex, then edit it, change the title, the authors and set proper in- clude(s) for all the chapters.\\n\\n1.2.2 Chapters\\n\\nEach chapter should be included in the main document as a separate ﬁle. You can choose any name for the ﬁle, but we suggest adding a sufﬁx to the name of the main ﬁle. For our example we use the ﬁle name document_chapter1.tex. First, copy template_chapter.tex to document_chapter1.tex\\n\\nand add the line\\n\\n\\\\include{document_chapter1}\\n\\nin the document.tex, then edit document_chapter1.tex, change the\\n\\nchapter title and edit the body of the chapter appropriately.\\n\\n1.2.3 Spell-checking\\n\\nDo use a spell-checker, please!\\n\\nYou may also want to check grammar, style and so on. Actually you should\\n\\ndo it (if you have enough spare time). But you must check spelling!\\n\\nYou can use the ispell package for this, from within emacs, or from the\\n\\ncommand line:\\n\\nispell -t document_chapter1.tex\\n\\n1.3. LATEXANDPDFLATEXCAPABILITIES\\n\\n7\\n\\n1.3 LATEX and pdfLATEX capabilities\\n\\n1.3.1 Overview\\n\\nFirst you edit your source .tex ﬁle. In LATEX you compile it using the latex command to a .dvi ﬁle (which stands for device-independent). The .dvi ﬁle can be converted to any device-dependent format you like using an appropriate driver, for example dvips.\\n\\nWhen producing .pdf ﬁles you should use pdflatex, which produces di- rectly .pdf ﬁles out of .tex sources. Note that in the .tex ﬁle you may need to use some PDF speciﬁc packages.\\n\\nFor viewing .tex ﬁles use your favourite text editor, for viewing .dvi ﬁles under X Window System use xdvi command, .ps ﬁles can be viewed with gv (or ghostview) and .pdf ﬁles with acroread, gv or xpdf.\\n\\n1.3.2 LATEX A lot of examples can be found in this document.\\n\\nYou should also print • doc/latex/general/latex2e.dvi and\\n\\ndoc/latex/general/lshort2e.dvi\\n\\nfrom your tetex distribution (usually in\\n\\n/usr/share/texmf or\\n\\n/usr/lib/texmf/texmf).\\n\\n1.3.3 pdfLATEX Consult doc/pdftex/manual.pdf from your tetex distribution for more details. Very useful informations can be found in the hyperref and graphics package manuals:\\n\\ndoc/latex/hyperref/manual.pdf and\\n\\ndoc/latex/graphics/grfguide.dvi.\\n\\n1.3.4 Examples\\n\\nReferences\\n\\nMIMUW\\n\\n8\\n\\nCHAPTER1. TEMPLATE\\n\\nHyperlinks\\n\\nThis is a target. And this is a link.\\n\\nDashes, etc.\\n\\nThere are three kinds of horizontal dash:\\n\\n- (use inside words; for example “home-page”, “X-rated”)\\n\\n– (use this one between numbers; for example “pages 2–22”)\\n\\n— (use this one as a sentence separator — like here)\\n\\nNational characters\\n\\nó, é, í, . . .\\n\\nè, à, ì, . . .\\n\\nô, ê, . . .\\n\\nõ, ñ, . . .\\n\\nö, ë, . . .\\n\\n˙z\\n\\n˛a, ˛e\\n\\nł, ø, ß\\n\\nThere are other ways to do this, see the documentation for inputenc pack-\\n\\nage.\\n\\nReserved characters\\n\\nSome characters have some special meaning, thus cannot be entered in the usual way.\\n\\n$ & % # _ { }\\n\\n\\\\\\n\\n˜ ˆ\\n\\n1.3. LATEXANDPDFLATEXCAPABILITIES\\n\\n9\\n\\nMath\\n\\n12, 12n, . . .\\n\\ni1, i2n, . . .\\n\\n1\\n\\n2 , 2n\\n\\n2−3 , . . . • α, β, γ, Ω, . . .\\n\\n→, ⇒, ≥, (cid:54)=, ∈, (cid:63), . . .\\n\\n√\\n\\n\\n\\n2, . . .\\n\\n2 + 2, . . .\\n\\nFor more examples and symbols see chapter 3 of lshort2e.dvi.\\n\\nFonts\\n\\nRoman\\n\\nEmphasis\\n\\nMedium weight — the default\\n\\nBoldface\\n\\nUpright\\n\\nSlanted\\n\\nSans serif\\n\\nSMALL CAPS\\n\\nTypewriter\\n\\nand sizes:\\n\\n– tiny\\n\\n– scriptsize\\n\\n– footnotesize\\n\\n– small\\n\\n– normalsize\\n\\n10\\n\\nCHAPTER1. TEMPLATE\\n\\n– large – Large – LARGE – huge – Huge'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "jtCvEtU769sD"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}