5.LangChain_data_connection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# 安装 LangChain"
      ],
      "metadata": {
        "id": "rfO7MCKbgiDr"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "vyrgFxpMxIiM",
        "outputId": "48f7508b-153b-4e1c-d2a5-8ed2ac123279"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting langchain[llms]\n",
            "  Downloading langchain-0.0.327-py3-none-any.whl (2.0 MB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/2.0 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.1/2.0 MB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m1.9/2.0 MB\u001b[0m \u001b[31m28.3 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m23.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain[llms]) (6.0.1)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain[llms]) (2.0.22)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain[llms]) (3.8.6)\n",
            "Requirement already satisfied: anyio<4.0 in /usr/local/lib/python3.10/dist-packages (from langchain[llms]) (3.7.1)\n",
            "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain[llms]) (4.0.3)\n",
            "Collecting dataclasses-json<0.7,>=0.5.7 (from langchain[llms])\n",
            "  Downloading dataclasses_json-0.6.1-py3-none-any.whl (27 kB)\n",
            "Collecting jsonpatch<2.0,>=1.33 (from langchain[llms])\n",
            "  Downloading jsonpatch-1.33-py2.py3-none-any.whl (12 kB)\n",
            "Collecting langsmith<0.1.0,>=0.0.52 (from langchain[llms])\n",
            "  Downloading langsmith-0.0.54-py3-none-any.whl (43 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain[llms]) (1.23.5)\n",
            "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain[llms]) (1.10.13)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain[llms]) (2.31.0)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain[llms]) (8.2.3)\n",
            "Collecting clarifai>=9.1.0 (from langchain[llms])\n",
            "  Downloading clarifai-9.9.3-py3-none-any.whl (2.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.5/2.5 MB\u001b[0m \u001b[31m59.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting cohere<5,>=4 (from langchain[llms])\n",
            "  Downloading cohere-4.32-py3-none-any.whl (47 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.0/48.0 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface_hub<1,>=0 (from langchain[llms])\n",
            "  Downloading huggingface_hub-0.18.0-py3-none-any.whl (301 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.0/302.0 kB\u001b[0m \u001b[31m32.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting manifest-ml<0.0.2,>=0.0.1 (from langchain[llms])\n",
            "  Downloading manifest_ml-0.0.1-py2.py3-none-any.whl (42 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.9/42.9 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nlpcloud<2,>=1 (from langchain[llms])\n",
            "  Downloading nlpcloud-1.1.44-py3-none-any.whl (4.4 kB)\n",
            "Collecting openai<1,>=0 (from langchain[llms])\n",
            "  Downloading openai-0.28.1-py3-none-any.whl (76 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.0/77.0 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting openlm<0.0.6,>=0.0.5 (from langchain[llms])\n",
            "  Downloading openlm-0.0.5-py3-none-any.whl (10 kB)\n",
            "Requirement already satisfied: torch<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain[llms]) (2.1.0+cu118)\n",
            "Collecting transformers<5,>=4 (from langchain[llms])\n",
            "  Downloading transformers-4.34.1-py3-none-any.whl (7.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.7/7.7 MB\u001b[0m \u001b[31m81.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain[llms]) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain[llms]) (3.3.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain[llms]) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain[llms]) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain[llms]) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain[llms]) (1.3.1)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<4.0->langchain[llms]) (3.4)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<4.0->langchain[llms]) (1.3.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4.0->langchain[llms]) (1.1.3)\n",
            "Collecting clarifai-grpc==9.8.1 (from clarifai>=9.1.0->langchain[llms])\n",
            "  Downloading clarifai_grpc-9.8.1-py3-none-any.whl (216 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m216.9/216.9 kB\u001b[0m \u001b[31m29.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pandas==1.3.5 (from clarifai>=9.1.0->langchain[llms])\n",
            "  Downloading pandas-1.3.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.5/11.5 MB\u001b[0m \u001b[31m99.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting numpy<2,>=1 (from langchain[llms])\n",
            "  Downloading numpy-1.22.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (16.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m16.8/16.8 MB\u001b[0m \u001b[31m48.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.65.0 in /usr/local/lib/python3.10/dist-packages (from clarifai>=9.1.0->langchain[llms]) (4.66.1)\n",
            "Collecting omegaconf==2.2.3 (from clarifai>=9.1.0->langchain[llms])\n",
            "  Downloading omegaconf-2.2.3-py3-none-any.whl (79 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.3/79.3 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pycocotools==2.0.6 (from clarifai>=9.1.0->langchain[llms])\n",
            "  Downloading pycocotools-2.0.6.tar.gz (24 kB)\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting opencv-python==4.7.0.68 (from clarifai>=9.1.0->langchain[llms])\n",
            "  Downloading opencv_python-4.7.0.68-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (61.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.8/61.8 MB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tritonclient==2.34.0 (from clarifai>=9.1.0->langchain[llms])\n",
            "  Downloading tritonclient-2.34.0-py3-none-manylinux1_x86_64.whl (12.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m46.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting rich==13.4.2 (from clarifai>=9.1.0->langchain[llms])\n",
            "  Downloading rich-13.4.2-py3-none-any.whl (239 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m239.4/239.4 kB\u001b[0m \u001b[31m24.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pytest==7.4.1 (from clarifai>=9.1.0->langchain[llms])\n",
            "  Downloading pytest-7.4.1-py3-none-any.whl (324 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m324.2/324.2 kB\u001b[0m \u001b[31m35.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting schema==0.7.5 (from clarifai>=9.1.0->langchain[llms])\n",
            "  Downloading schema-0.7.5-py2.py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: grpcio>=1.44.0 in /usr/local/lib/python3.10/dist-packages (from clarifai-grpc==9.8.1->clarifai>=9.1.0->langchain[llms]) (1.59.0)\n",
            "Requirement already satisfied: protobuf>=3.20.3 in /usr/local/lib/python3.10/dist-packages (from clarifai-grpc==9.8.1->clarifai>=9.1.0->langchain[llms]) (3.20.3)\n",
            "Requirement already satisfied: googleapis-common-protos>=1.53.0 in /usr/local/lib/python3.10/dist-packages (from clarifai-grpc==9.8.1->clarifai>=9.1.0->langchain[llms]) (1.61.0)\n",
            "Collecting antlr4-python3-runtime==4.9.* (from omegaconf==2.2.3->clarifai>=9.1.0->langchain[llms])\n",
            "  Downloading antlr4-python3-runtime-4.9.3.tar.gz (117 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 kB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.10/dist-packages (from pandas==1.3.5->clarifai>=9.1.0->langchain[llms]) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.10/dist-packages (from pandas==1.3.5->clarifai>=9.1.0->langchain[llms]) (2023.3.post1)\n",
            "Requirement already satisfied: matplotlib>=2.1.0 in /usr/local/lib/python3.10/dist-packages (from pycocotools==2.0.6->clarifai>=9.1.0->langchain[llms]) (3.7.1)\n",
            "Requirement already satisfied: iniconfig in /usr/local/lib/python3.10/dist-packages (from pytest==7.4.1->clarifai>=9.1.0->langchain[llms]) (2.0.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from pytest==7.4.1->clarifai>=9.1.0->langchain[llms]) (23.2)\n",
            "Requirement already satisfied: pluggy<2.0,>=0.12 in /usr/local/lib/python3.10/dist-packages (from pytest==7.4.1->clarifai>=9.1.0->langchain[llms]) (1.3.0)\n",
            "Requirement already satisfied: tomli>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from pytest==7.4.1->clarifai>=9.1.0->langchain[llms]) (2.0.1)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich==13.4.2->clarifai>=9.1.0->langchain[llms]) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich==13.4.2->clarifai>=9.1.0->langchain[llms]) (2.16.1)\n",
            "Requirement already satisfied: contextlib2>=0.5.5 in /usr/local/lib/python3.10/dist-packages (from schema==0.7.5->clarifai>=9.1.0->langchain[llms]) (21.6.0)\n",
            "Collecting python-rapidjson>=0.9.1 (from tritonclient==2.34.0->clarifai>=9.1.0->langchain[llms])\n",
            "  Downloading python_rapidjson-1.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m79.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting backoff<3.0,>=2.0 (from cohere<5,>=4->langchain[llms])\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Collecting fastavro==1.8.2 (from cohere<5,>=4->langchain[llms])\n",
            "  Downloading fastavro-1.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m85.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: importlib_metadata<7.0,>=6.0 in /usr/local/lib/python3.10/dist-packages (from cohere<5,>=4->langchain[llms]) (6.8.0)\n",
            "Requirement already satisfied: urllib3<3,>=1.26 in /usr/local/lib/python3.10/dist-packages (from cohere<5,>=4->langchain[llms]) (2.0.7)\n",
            "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json<0.7,>=0.5.7->langchain[llms])\n",
            "  Downloading marshmallow-3.20.1-py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-inspect<1,>=0.4.0 (from dataclasses-json<0.7,>=0.5.7->langchain[llms])\n",
            "  Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<1,>=0->langchain[llms]) (3.12.4)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<1,>=0->langchain[llms]) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<1,>=0->langchain[llms]) (4.5.0)\n",
            "Collecting jsonpointer>=1.9 (from jsonpatch<2.0,>=1.33->langchain[llms])\n",
            "  Downloading jsonpointer-2.4-py2.py3-none-any.whl (7.8 kB)\n",
            "Collecting dill>=0.3.5 (from manifest-ml<0.0.2,>=0.0.1->langchain[llms])\n",
            "  Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m16.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting redis>=4.3.1 (from manifest-ml<0.0.2,>=0.0.1->langchain[llms])\n",
            "  Downloading redis-5.0.1-py3-none-any.whl (250 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m250.3/250.3 kB\u001b[0m \u001b[31m26.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sqlitedict>=2.0.0 (from manifest-ml<0.0.2,>=0.0.1->langchain[llms])\n",
            "  Downloading sqlitedict-2.1.0.tar.gz (21 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain[llms]) (2023.7.22)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain[llms]) (3.0.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch<3,>=1->langchain[llms]) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch<3,>=1->langchain[llms]) (3.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch<3,>=1->langchain[llms]) (3.1.2)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch<3,>=1->langchain[llms]) (2.1.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5,>=4->langchain[llms]) (2023.6.3)\n",
            "Collecting tokenizers<0.15,>=0.14 (from transformers<5,>=4->langchain[llms])\n",
            "  Downloading tokenizers-0.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m69.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers<5,>=4->langchain[llms])\n",
            "  Downloading safetensors-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m62.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib_metadata<7.0,>=6.0->cohere<5,>=4->langchain[llms]) (3.17.0)\n",
            "Collecting huggingface_hub<1,>=0 (from langchain[llms])\n",
            "  Downloading huggingface_hub-0.17.3-py3-none-any.whl (295 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.0/295.0 kB\u001b[0m \u001b[31m20.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting mypy-extensions>=0.3.0 (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain[llms])\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch<3,>=1->langchain[llms]) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch<3,>=1->langchain[llms]) (1.3.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich==13.4.2->clarifai>=9.1.0->langchain[llms]) (0.1.2)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools==2.0.6->clarifai>=9.1.0->langchain[llms]) (1.1.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools==2.0.6->clarifai>=9.1.0->langchain[llms]) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools==2.0.6->clarifai>=9.1.0->langchain[llms]) (4.43.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools==2.0.6->clarifai>=9.1.0->langchain[llms]) (1.4.5)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools==2.0.6->clarifai>=9.1.0->langchain[llms]) (9.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools==2.0.6->clarifai>=9.1.0->langchain[llms]) (3.1.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7.3->pandas==1.3.5->clarifai>=9.1.0->langchain[llms]) (1.16.0)\n",
            "Building wheels for collected packages: pycocotools, antlr4-python3-runtime, sqlitedict\n",
            "  Building wheel for pycocotools (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pycocotools: filename=pycocotools-2.0.6-cp310-cp310-linux_x86_64.whl size=377131 sha256=71c4a8898d46e5d5b7f6ce6576c588b8bf464a44fef05715f65009ce3f31d6f9\n",
            "  Stored in directory: /root/.cache/pip/wheels/58/e6/f9/f87c8f8be098b51b616871315318329cae12cdb618f4caac93\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.9.3-py3-none-any.whl size=144554 sha256=15c4129322650f6553791051e4c97a7812bee5c8e935134f355d971782125c2d\n",
            "  Stored in directory: /root/.cache/pip/wheels/12/93/dd/1f6a127edc45659556564c5730f6d4e300888f4bca2d4c5a88\n",
            "  Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sqlitedict: filename=sqlitedict-2.1.0-py3-none-any.whl size=16864 sha256=367d4f191a19fc24fb3af5e4b0ce23bfa7a40f878d6ef6471a4d00289e445ebf\n",
            "  Stored in directory: /root/.cache/pip/wheels/79/d6/e7/304e0e6cb2221022c26d8161f7c23cd4f259a9e41e8bbcfabd\n",
            "Successfully built pycocotools antlr4-python3-runtime sqlitedict\n",
            "Installing collected packages: sqlitedict, antlr4-python3-runtime, schema, safetensors, redis, python-rapidjson, pytest, omegaconf, numpy, mypy-extensions, marshmallow, jsonpointer, fastavro, dill, backoff, typing-inspect, tritonclient, rich, pandas, openlm, opencv-python, nlpcloud, manifest-ml, langsmith, jsonpatch, huggingface_hub, clarifai-grpc, tokenizers, openai, dataclasses-json, cohere, transformers, pycocotools, langchain, clarifai\n",
            "  Attempting uninstall: pytest\n",
            "    Found existing installation: pytest 7.4.3\n",
            "    Uninstalling pytest-7.4.3:\n",
            "      Successfully uninstalled pytest-7.4.3\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 1.23.5\n",
            "    Uninstalling numpy-1.23.5:\n",
            "      Successfully uninstalled numpy-1.23.5\n",
            "  Attempting uninstall: rich\n",
            "    Found existing installation: rich 13.6.0\n",
            "    Uninstalling rich-13.6.0:\n",
            "      Successfully uninstalled rich-13.6.0\n",
            "  Attempting uninstall: pandas\n",
            "    Found existing installation: pandas 1.5.3\n",
            "    Uninstalling pandas-1.5.3:\n",
            "      Successfully uninstalled pandas-1.5.3\n",
            "  Attempting uninstall: opencv-python\n",
            "    Found existing installation: opencv-python 4.8.0.76\n",
            "    Uninstalling opencv-python-4.8.0.76:\n",
            "      Successfully uninstalled opencv-python-4.8.0.76\n",
            "  Attempting uninstall: pycocotools\n",
            "    Found existing installation: pycocotools 2.0.7\n",
            "    Uninstalling pycocotools-2.0.7:\n",
            "      Successfully uninstalled pycocotools-2.0.7\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "lida 0.0.10 requires fastapi, which is not installed.\n",
            "lida 0.0.10 requires kaleido, which is not installed.\n",
            "lida 0.0.10 requires python-multipart, which is not installed.\n",
            "lida 0.0.10 requires uvicorn, which is not installed.\n",
            "llmx 0.0.15a0 requires tiktoken, which is not installed.\n",
            "bigframes 0.10.0 requires pandas>=1.5.0, but you have pandas 1.3.5 which is incompatible.\n",
            "google-colab 1.0.0 requires pandas==1.5.3, but you have pandas 1.3.5 which is incompatible.\n",
            "librosa 0.10.1 requires numpy!=1.22.0,!=1.22.1,!=1.22.2,>=1.20.3, but you have numpy 1.22.0 which is incompatible.\n",
            "plotnine 0.12.3 requires numpy>=1.23.0, but you have numpy 1.22.0 which is incompatible.\n",
            "plotnine 0.12.3 requires pandas>=1.5.0, but you have pandas 1.3.5 which is incompatible.\n",
            "tensorflow 2.14.0 requires numpy>=1.23.5, but you have numpy 1.22.0 which is incompatible.\n",
            "xarray 2023.7.0 requires pandas>=1.4, but you have pandas 1.3.5 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed antlr4-python3-runtime-4.9.3 backoff-2.2.1 clarifai-9.9.3 clarifai-grpc-9.8.1 cohere-4.32 dataclasses-json-0.6.1 dill-0.3.7 fastavro-1.8.2 huggingface_hub-0.17.3 jsonpatch-1.33 jsonpointer-2.4 langchain-0.0.327 langsmith-0.0.54 manifest-ml-0.0.1 marshmallow-3.20.1 mypy-extensions-1.0.0 nlpcloud-1.1.44 numpy-1.22.0 omegaconf-2.2.3 openai-0.28.1 opencv-python-4.7.0.68 openlm-0.0.5 pandas-1.3.5 pycocotools-2.0.6 pytest-7.4.1 python-rapidjson-1.13 redis-5.0.1 rich-13.4.2 safetensors-0.4.0 schema-0.7.5 sqlitedict-2.1.0 tokenizers-0.14.1 transformers-4.34.1 tritonclient-2.34.0 typing-inspect-0.9.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "numpy",
                  "pydevd_plugins"
                ]
              }
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "!pip install langchain[llms]"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 设置 OpenAI Key"
      ],
      "metadata": {
        "id": "9iHD5lgMyPAX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ['OPENAI_API_BASE'] = ''\n",
        "os.environ['OPENAI_API_KEY'] = ''\n"
      ],
      "metadata": {
        "id": "qx2xkwSzyOZe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 安装依赖包"
      ],
      "metadata": {
        "id": "j-WQPHLP0GDE"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Document Loaders（文档加载器）"
      ],
      "metadata": {
        "id": "i0X4BUraxJZI"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## CSV加载器"
      ],
      "metadata": {
        "id": "YkzfqZ1lyzJw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders.csv_loader import CSVLoader\n",
        "\n",
        "\n",
        "loader = CSVLoader(\n",
        "    'sample_data/california_housing_test.csv',\n",
        "    source_column=\"housing_median_age\"\n",
        ")\n",
        "\n",
        "for x in loader.load()[:10]:\n",
        "    print(x)"
      ],
      "metadata": {
        "id": "blr_joRQC-vl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 文件目录加载器"
      ],
      "metadata": {
        "id": "Wd0kEIIpy4BI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pdfminer.six"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ngJS0qckIkQP",
        "outputId": "e7e4c63a-5e5b-479e-d1fc-2956daa38005"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pdfminer.six in /usr/local/lib/python3.10/dist-packages (20221105)\n",
            "Requirement already satisfied: charset-normalizer>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six) (3.3.0)\n",
            "Requirement already satisfied: cryptography>=36.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six) (41.0.4)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/dist-packages (from cryptography>=36.0.0->pdfminer.six) (1.16.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six) (2.21)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import DirectoryLoader\n",
        "\n",
        "loader = DirectoryLoader('sample_data/', glob=\"**/*.md\")\n",
        "\n",
        "for x in loader.load():\n",
        "    print(x)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IJfaLKuOHTfJ",
        "outputId": "9c97d76f-a743-40c7-b93a-1d051030ab51"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "page_content=\"This directory includes a few sample datasets to get you started.\\n\\ncalifornia_housing_data*.csv is California housing data from the 1990 US\\n    Census; more information is available at:\\n    https://developers.google.com/machine-learning/crash-course/california-housing-data-description\\n\\nmnist_*.csv is a small sample of the\\n    MNIST database, which is\\n    described at: http://yann.lecun.com/exdb/mnist/\\n\\nanscombe.json contains a copy of\\n    Anscombe's quartet; it\\n    was originally described in\\nAnscombe, F. J. (1973). 'Graphs in Statistical Analysis'. American\\nStatistician. 27 (1): 17-21. JSTOR 2682899.\\nand our copy was prepared by the\\nvega_datasets library.\" metadata={'source': 'sample_data/README.md'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## HTML加载器"
      ],
      "metadata": {
        "id": "a7dnAAfCzJbI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import UnstructuredHTMLLoader\n",
        "\n",
        "loader = UnstructuredHTMLLoader(\n",
        "    'example_data/fake-content.html'\n",
        ")\n",
        "\n",
        "for x in loader.load():\n",
        "    print(x)"
      ],
      "metadata": {
        "id": "fkdgXPtazIZ4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## JSON加载器"
      ],
      "metadata": {
        "id": "O3qabhphzNO4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install jq"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eXx1N7OQSw77",
        "outputId": "197b8105-03af-4f0d-b004-8f644b7c8bd0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting jq\n",
            "  Downloading jq-1.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (656 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m656.0/656.0 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: jq\n",
            "Successfully installed jq-1.6.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import JSONLoader\n",
        "\n",
        "def metadata_func(record, metadata):\n",
        "    metadata['Series'] = record['Series']\n",
        "    return metadata\n",
        "\n",
        "\n",
        "loader = JSONLoader(\n",
        "    file_path='sample_data/anscombe.json',\n",
        "    jq_schema='.[]',\n",
        "    metadata_func=metadata_func,\n",
        "    text_content=False\n",
        ")\n",
        "\n",
        "for x in loader.load():\n",
        "    print(x)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MZKwMH3rSJB0",
        "outputId": "aed42841-db6c-42ff-f998-a3086156103b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "page_content='{\"Series\": \"I\", \"X\": 10, \"Y\": 8.04}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 1, 'Series': 'I'}\n",
            "page_content='{\"Series\": \"I\", \"X\": 8, \"Y\": 6.95}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 2, 'Series': 'I'}\n",
            "page_content='{\"Series\": \"I\", \"X\": 13, \"Y\": 7.58}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 3, 'Series': 'I'}\n",
            "page_content='{\"Series\": \"I\", \"X\": 9, \"Y\": 8.81}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 4, 'Series': 'I'}\n",
            "page_content='{\"Series\": \"I\", \"X\": 11, \"Y\": 8.33}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 5, 'Series': 'I'}\n",
            "page_content='{\"Series\": \"I\", \"X\": 14, \"Y\": 9.96}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 6, 'Series': 'I'}\n",
            "page_content='{\"Series\": \"I\", \"X\": 6, \"Y\": 7.24}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 7, 'Series': 'I'}\n",
            "page_content='{\"Series\": \"I\", \"X\": 4, \"Y\": 4.26}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 8, 'Series': 'I'}\n",
            "page_content='{\"Series\": \"I\", \"X\": 12, \"Y\": 10.84}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 9, 'Series': 'I'}\n",
            "page_content='{\"Series\": \"I\", \"X\": 7, \"Y\": 4.81}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 10, 'Series': 'I'}\n",
            "page_content='{\"Series\": \"I\", \"X\": 5, \"Y\": 5.68}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 11, 'Series': 'I'}\n",
            "page_content='{\"Series\": \"II\", \"X\": 10, \"Y\": 9.14}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 12, 'Series': 'II'}\n",
            "page_content='{\"Series\": \"II\", \"X\": 8, \"Y\": 8.14}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 13, 'Series': 'II'}\n",
            "page_content='{\"Series\": \"II\", \"X\": 13, \"Y\": 8.74}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 14, 'Series': 'II'}\n",
            "page_content='{\"Series\": \"II\", \"X\": 9, \"Y\": 8.77}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 15, 'Series': 'II'}\n",
            "page_content='{\"Series\": \"II\", \"X\": 11, \"Y\": 9.26}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 16, 'Series': 'II'}\n",
            "page_content='{\"Series\": \"II\", \"X\": 14, \"Y\": 8.1}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 17, 'Series': 'II'}\n",
            "page_content='{\"Series\": \"II\", \"X\": 6, \"Y\": 6.13}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 18, 'Series': 'II'}\n",
            "page_content='{\"Series\": \"II\", \"X\": 4, \"Y\": 3.1}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 19, 'Series': 'II'}\n",
            "page_content='{\"Series\": \"II\", \"X\": 12, \"Y\": 9.13}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 20, 'Series': 'II'}\n",
            "page_content='{\"Series\": \"II\", \"X\": 7, \"Y\": 7.26}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 21, 'Series': 'II'}\n",
            "page_content='{\"Series\": \"II\", \"X\": 5, \"Y\": 4.74}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 22, 'Series': 'II'}\n",
            "page_content='{\"Series\": \"III\", \"X\": 10, \"Y\": 7.46}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 23, 'Series': 'III'}\n",
            "page_content='{\"Series\": \"III\", \"X\": 8, \"Y\": 6.77}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 24, 'Series': 'III'}\n",
            "page_content='{\"Series\": \"III\", \"X\": 13, \"Y\": 12.74}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 25, 'Series': 'III'}\n",
            "page_content='{\"Series\": \"III\", \"X\": 9, \"Y\": 7.11}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 26, 'Series': 'III'}\n",
            "page_content='{\"Series\": \"III\", \"X\": 11, \"Y\": 7.81}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 27, 'Series': 'III'}\n",
            "page_content='{\"Series\": \"III\", \"X\": 14, \"Y\": 8.84}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 28, 'Series': 'III'}\n",
            "page_content='{\"Series\": \"III\", \"X\": 6, \"Y\": 6.08}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 29, 'Series': 'III'}\n",
            "page_content='{\"Series\": \"III\", \"X\": 4, \"Y\": 5.39}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 30, 'Series': 'III'}\n",
            "page_content='{\"Series\": \"III\", \"X\": 12, \"Y\": 8.15}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 31, 'Series': 'III'}\n",
            "page_content='{\"Series\": \"III\", \"X\": 7, \"Y\": 6.42}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 32, 'Series': 'III'}\n",
            "page_content='{\"Series\": \"III\", \"X\": 5, \"Y\": 5.73}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 33, 'Series': 'III'}\n",
            "page_content='{\"Series\": \"IV\", \"X\": 8, \"Y\": 6.58}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 34, 'Series': 'IV'}\n",
            "page_content='{\"Series\": \"IV\", \"X\": 8, \"Y\": 5.76}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 35, 'Series': 'IV'}\n",
            "page_content='{\"Series\": \"IV\", \"X\": 8, \"Y\": 7.71}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 36, 'Series': 'IV'}\n",
            "page_content='{\"Series\": \"IV\", \"X\": 8, \"Y\": 8.84}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 37, 'Series': 'IV'}\n",
            "page_content='{\"Series\": \"IV\", \"X\": 8, \"Y\": 8.47}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 38, 'Series': 'IV'}\n",
            "page_content='{\"Series\": \"IV\", \"X\": 8, \"Y\": 7.04}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 39, 'Series': 'IV'}\n",
            "page_content='{\"Series\": \"IV\", \"X\": 8, \"Y\": 5.25}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 40, 'Series': 'IV'}\n",
            "page_content='{\"Series\": \"IV\", \"X\": 19, \"Y\": 12.5}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 41, 'Series': 'IV'}\n",
            "page_content='{\"Series\": \"IV\", \"X\": 8, \"Y\": 5.56}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 42, 'Series': 'IV'}\n",
            "page_content='{\"Series\": \"IV\", \"X\": 8, \"Y\": 7.91}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 43, 'Series': 'IV'}\n",
            "page_content='{\"Series\": \"IV\", \"X\": 8, \"Y\": 6.89}' metadata={'source': '/content/sample_data/anscombe.json', 'seq_num': 44, 'Series': 'IV'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Markdown加载器"
      ],
      "metadata": {
        "id": "HPARsxI9ziOY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "Markdown加载器（UnstructuredMarkdownLoader）使我们可以便捷地将Markdown文本加载为Document对象，具体代码如下。\n",
        "\n",
        "from langchain.document_loaders import UnstructuredMarkdownLoader\n",
        "\n",
        "loader = UnstructuredMarkdownLoader(\n",
        "    'sample_data/README.md'\n",
        ")\n",
        "\n",
        "for x in loader.load():\n",
        "    print(x)"
      ],
      "metadata": {
        "id": "r-7XEDiKzjM3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## URL加载器"
      ],
      "metadata": {
        "id": "vdsy9qBEzliX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import WebBaseLoader\n",
        "\n",
        "loader = WebBaseLoader(\n",
        "    'https://movie.douban.com/subject/1292063/'\n",
        ")\n",
        "\n",
        "for x in loader.load():\n",
        "    print(x.page_content)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VWWimlrTGqqH",
        "outputId": "b64d55e5-7c70-43c5-96c3-747a288340d3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "        美丽人生 (豆瓣)\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "登录/注册\n",
            "\n",
            "\n",
            "下载豆瓣客户端\n",
            "\n",
            "豆瓣 6.0 全新发布\n",
            "×\n",
            "\n",
            "\n",
            "豆瓣\n",
            "扫码直接下载\n",
            "\n",
            "iPhone\n",
            "·\n",
            "Android\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "豆瓣\n",
            "\n",
            "\n",
            "读书\n",
            "\n",
            "\n",
            "电影\n",
            "\n",
            "\n",
            "音乐\n",
            "\n",
            "\n",
            "同城\n",
            "\n",
            "\n",
            "小组\n",
            "\n",
            "\n",
            "阅读\n",
            "\n",
            "\n",
            "FM\n",
            "\n",
            "\n",
            "时间\n",
            "\n",
            "\n",
            "豆品\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "豆瓣电影\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "搜索：\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "影讯&购票\n",
            "\n",
            "选电影\n",
            "\n",
            "电视剧\n",
            "\n",
            "排行榜\n",
            "\n",
            "影评\n",
            "\n",
            "2022年度榜单\n",
            "\n",
            "2022书影音报告\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "No.7豆瓣电影Top250\n",
            "\n",
            "\n",
            "\n",
            "美丽人生 La vita è bella\n",
            "(1997)\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "导演: 罗伯托·贝尼尼\n",
            "编剧: 温琴佐·切拉米 / 罗伯托·贝尼尼\n",
            "主演: 罗伯托·贝尼尼 / 尼可莱塔·布拉斯基 / 乔治·坎塔里尼 / 朱斯蒂诺·杜拉诺 / 赛尔乔·比尼·布斯特里克 / 玛丽萨·帕雷德斯 / 霍斯特·布赫霍尔茨 / 利迪娅·阿方西 / 朱利亚娜·洛约迪切 / 亚美利哥·丰塔尼 / 彼得·德·席尔瓦 / 弗朗西斯·古佐 / 拉法埃拉·莱博罗尼 / 克劳迪奥·阿方西 / 吉尔·巴罗尼 / 马西莫·比安奇 / 恩尼奥·孔萨尔维 / 吉安卡尔洛·科森蒂诺 / 阿伦·克雷格 / 汉尼斯·赫尔曼 / 弗兰科·梅斯科利尼 / 安东尼奥·普雷斯特 / 吉娜·诺维勒 / 理查德·塞梅尔 / 安德烈提多娜 / 迪尔克·范登贝格 / 奥梅罗·安东努蒂\n",
            "类型: 剧情 / 喜剧 / 爱情 / 战争\n",
            "制片国家/地区: 意大利\n",
            "语言: 意大利语 / 德语 / 英语\n",
            "上映日期: 2020-01-03(中国大陆) / 1997-12-20(意大利)\n",
            "片长: 116分钟(国际版) / 125分钟\n",
            "又名: 一个快乐的传说(港) / Life Is Beautiful\n",
            "IMDb: tt0118799\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                豆瓣评分\n",
            "            \n",
            "\n",
            "\n",
            "引用\n",
            "\n",
            "\n",
            "\n",
            "9.6\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "1341175人评价\n",
            "                \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "            5星\n",
            "        \n",
            "\n",
            "80.6%\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "            4星\n",
            "        \n",
            "\n",
            "16.9%\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "            3星\n",
            "        \n",
            "\n",
            "2.2%\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "            2星\n",
            "        \n",
            "\n",
            "0.2%\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "            1星\n",
            "        \n",
            "\n",
            "0.1%\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "            好于 99% 战争片\n",
            "            好于 99% 爱情片\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "想看\n",
            "\n",
            "\n",
            "看过\n",
            "\n",
            "\n",
            "            \n",
            "    \n",
            "    评价:\n",
            "     \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " \n",
            "        写短评\n",
            "\n",
            "\n",
            " \n",
            "        写影评\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "        分享到\n",
            "       \n",
            "    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "推荐\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "美丽人生的剧情简介\n",
            "              · · · · · ·\n",
            "    \n",
            "\n",
            "\n",
            "                                　　犹太青年圭多（罗伯托·贝尼尼）邂逅美丽的女教师多拉（尼可莱塔·布拉斯基），他彬彬有礼的向多拉鞠躬：“早安！公主！”。历经诸多令人啼笑皆非的周折后，天遂人愿，两人幸福美满的生活在一起。\n",
            "                                    \n",
            "                                　　然而好景不长，法西斯政权下，圭多和儿子被强行送往犹太人集中营。多拉虽没有犹太血统，毅然同行，与丈夫儿子分开关押在一个集中营里。聪明乐天的圭多哄骗儿子这只是一场游戏，奖品就是一辆大坦克，儿子快乐、天真的生活在纳粹的阴霾之中。尽管集中营的生活艰苦寂寞，圭多仍然带给他人很多快乐，他还趁机在纳粹的广播里问候妻子：“早安！公主！”\n",
            "                                    \n",
            "                                　　法西斯政权即将倾覆，纳粹的集中营很快就要接受最后的清理，圭多编给儿子的游戏该怎么结束？他们一家能否平安的度过这黑暗的年代呢？\n",
            "                        \n",
            "©豆瓣\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "美丽人生的演职员\n",
            "              · · · · · ·\n",
            "            \n",
            "            (\n",
            "                全部 41\n",
            "            )\n",
            "            \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "罗伯托·贝尼尼\n",
            "导演\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "罗伯托·贝尼尼\n",
            "饰 Guido\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "尼可莱塔·布拉斯基\n",
            "饰 Dora\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "乔治·坎塔里尼\n",
            "饰 Giosuè\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "朱斯蒂诺·杜拉诺\n",
            "饰 Zio\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "赛尔乔·比尼·布斯特里克\n",
            "饰 Ferruccio\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "美丽人生的视频和图片\n",
            "              · · · · · ·\n",
            "            \n",
            "            (\n",
            "                预告片5 | 添加视频评论 | 图片909 · 添加\n",
            "            )\n",
            "            \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "美丽人生的获奖情况\n",
            "              · · · · · ·\n",
            "            \n",
            "            (\n",
            "                全部\n",
            "            )\n",
            "            \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "第51届戛纳电影节\n",
            "\n",
            "主竞赛单元 金棕榈奖(提名)\n",
            "罗伯托·贝尼尼\n",
            "\n",
            "\n",
            "\n",
            "第71届奥斯卡金像奖\n",
            "\n",
            "最佳影片(提名)\n",
            "艾尔达·法瑞 / 吉安路易吉·布拉斯基\n",
            "\n",
            "\n",
            "\n",
            "第23届日本电影学院奖\n",
            "\n",
            "最佳外语片(提名)\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "喜欢这部电影的人也喜欢\n",
            "              · · · · · ·\n",
            "    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "辛德勒的名单\n",
            "9.6\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "忠犬八公的故事\n",
            "9.4\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "穿条纹睡衣的男孩\n",
            "9.2\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "拯救大兵瑞恩\n",
            "9.1\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "这个杀手不太冷\n",
            "9.4\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "触不可及\n",
            "9.3\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "钢琴家\n",
            "9.3\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "泰坦尼克号\n",
            "9.5\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "当幸福来敲门\n",
            "9.2\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "饮食男女\n",
            "9.2\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "我要写短评\n",
            "\n",
            "\n",
            "美丽人生的短评\n",
            "              · · · · · ·\n",
            "            \n",
            "            (\n",
            "                全部 292770 条\n",
            "            )\n",
            "            \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "热门 / \n",
            "                        最新 / \n",
            "                        好友\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "28366\n",
            "\n",
            "有用\n",
            "\n",
            "\n",
            "\n",
            "不靠谱小姐\n",
            "看过\n",
            "\n",
            "\n",
            "                    2010-08-17 11:39:02\n",
            "                \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "我以为如此智慧的一个人，在那几声枪响过后，必定是会走出来，继续对他的公主说早安的...\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "13439\n",
            "\n",
            "有用\n",
            "\n",
            "\n",
            "\n",
            "老鸡｜扶立\n",
            "看过\n",
            "\n",
            "\n",
            "                    2008-01-10 16:54:36\n",
            "                \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "如果谎言可以这样美丽，我也情愿生活在谎言之中\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "14150\n",
            "\n",
            "有用\n",
            "\n",
            "\n",
            "\n",
            "monotonous\n",
            "看过\n",
            "\n",
            "\n",
            "                    2007-12-21 16:05:17\n",
            "                \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "人生，就是喜剧，就是一场1000分的游戏。千万别哭，不然那个迈着正步去赴死的犹太人会笑话你的。\r\n",
            " \r\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "1395\n",
            "\n",
            "有用\n",
            "\n",
            "\n",
            "\n",
            "如是我闻\n",
            "看过\n",
            "\n",
            "\n",
            "                    2009-03-13 21:23:18\n",
            "                \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "在真实的历史面前，不可能有这样的轻松与浪漫！\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "9230\n",
            "\n",
            "有用\n",
            "\n",
            "\n",
            "\n",
            "UrthónaD'Mors\n",
            "看过\n",
            "\n",
            "\n",
            "                    2008-01-27 23:20:12\n",
            "                \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "喜剧的开始，悲剧的结束，\r\n",
            "满满的亲情，无限的父爱\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                \n",
            "                    > \n",
            "                        更多短评\n",
            "                            292770条\n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "为什么被折叠？\n",
            "有一些短评被折叠了\n",
            "\n",
            "                    评论被折叠，是因为发布这条评论的账号行为异常。评论仍可以被展开阅读，对发布人的账号不造成其他影响。如果认为有问题，可以联系豆瓣电影。\n",
            "                \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "            你关注的人还没写过短评\n",
            "        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "我要写影评\n",
            "\n",
            "\n",
            "                    美丽人生的影评 · · · · · ·\n",
            "\n",
            "                    ( 全部 5441 条 )\n",
            "\n",
            "\n",
            "\n",
            "热门\n",
            "最新\n",
            "好友\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "小隐隐于浆糊\n",
            "\n",
            "2008-03-24 21:11:06\n",
            "\n",
            "\n",
            "H4传说\n",
            "\n",
            "\n",
            "\n",
            "                        林语堂给人生列出了几个公式： 1)Reality — Dreams ＝ Animal Being 2)Reality ＋ Dreams ＝A heartache(usually called Idealism) 3)Reality ＋ Humor ＝ Realism(also called Conservatism) 4)Dreams － Humor ＝ Fanaticism 5)Dreams ＋ Humor ＝ Fantasy 6)Reality ＋ Dre...\n",
            "\n",
            "                         (展开)\n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                6962\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                200\n",
            "                        \n",
            "\n",
            "280回应\n",
            "收起\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "赛宁\n",
            "\n",
            "2006-03-05 23:32:27\n",
            "\n",
            "\n",
            "“观碟不语”之《美丽人生》\n",
            "\n",
            "\n",
            "\n",
            "                        就算在最艰难最黑暗的日子里，就算了无希望，死亡近在眼前，他依然深爱着并用生命与智慧保护着他的妻子与儿子。他的勇气与智慧，即使在战争的硝烟弥漫中，即使在集中营的暗无天日中，即使在最后枪声响起死亡来临的那一刻，依然闪现着耀眼夺目的光芒。 他用尽全力，在集中营的悲...\n",
            "\n",
            "                         (展开)\n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                5336\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                160\n",
            "                        \n",
            "\n",
            "289回应\n",
            "收起\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "徐晋\n",
            "\n",
            "2005-11-25 23:13:35\n",
            "\n",
            "\n",
            "我们有资格做父亲吗？\n",
            "\n",
            "\n",
            "\n",
            "                        生于70年代中末期的我们到了应该可以做父亲的年纪了，在这样一个滑稽的让人绝望的年代里。 我们的经历比任何人都更加动荡：之前的60年代绝大多数还是在故乡发展；之后的80年代刚开始迷惘。而我们这代人是一个开始。 当结束十几年的教育之后，突然发现我们跟体制其实没什么关系...\n",
            "\n",
            "                         (展开)\n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                2963\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                482\n",
            "                        \n",
            "\n",
            "443回应\n",
            "收起\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Seeking\n",
            "\n",
            "2009-06-30 01:47:34\n",
            "\n",
            "\n",
            "对《La Vita è Bella》中德国医生和基多说的最后一个谜语的理解\n",
            "\n",
            "\n",
            "\n",
            "                        我也看了些国外的评论，基本观点是那就是个普通的谜语，没有什么隐藏的含义，因此我把百度美丽人生吧的一篇帖子转到了这里  导演本人在被问到这个问题时的回答是，这个谜语没有解，目的是为反映那场战争的荒谬。因此大家的争论可以告一段落了。    http://tieba.baidu.com/f?kz...\n",
            "\n",
            "                         (展开)\n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                2643\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                160\n",
            "                        \n",
            "\n",
            "336回应\n",
            "收起\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "crow\n",
            "\n",
            "2008-12-29 17:04:55\n",
            "\n",
            "\n",
            "该是男人婚前必看的影片\n",
            "\n",
            "\n",
            "\n",
            "                        草草看过其他人的影评，满满一片国旗的星星数 倘若谁再挑出这片的毛病 只怕对不起的就是人生二字了 —————————————————————— 很多年前读过一本哲理小册子里讲到过这个故事 那时候还小，不懂得亲情和爱情的伟大 在我不相信谎言的时候，心想这小孩子真好骗 ...\n",
            "\n",
            "                         (展开)\n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                2222\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                87\n",
            "                        \n",
            "\n",
            "107回应\n",
            "收起\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "朝暮雪\n",
            "\n",
            "2020-01-04 15:35:44\n",
            "\n",
            "\n",
            "伟大喜剧的内核都是悲剧\n",
            "\n",
            "\n",
            "这篇影评可能有剧透\n",
            "\n",
            "                        继《龙猫》《千与千寻》《海上钢琴师》后，《美丽人生》4K修复版也在中国上映了。 2001年2月，《美丽人生》曾在中国上映，但是规模很小。 这次则是大规模上映，而且还是4K修复版。 上小学的时候，学校放了这部电影，当时以为这只是一部喜剧片。 长大以后才明白，这部电影和其它...\n",
            "\n",
            "                         (展开)\n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                1907\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                18\n",
            "                        \n",
            "\n",
            "65回应\n",
            "收起\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "王挣扎\n",
            "\n",
            "2008-07-20 16:49:36\n",
            "\n",
            "\n",
            "早上好，我的公主！\n",
            "\n",
            "\n",
            "这篇影评可能有剧透\n",
            "\n",
            "                        生活是美丽的，无论它怎样不尽如人意。你会在不经意的时候偶遇可爱的女孩儿，你要对她说：早上好，我的公主！然后脱帽致意，向她展示你的无与伦比的才华。      对任何事情充满巨大的热情，把任何事情都看作天赐，对所有人报以微笑，对欣赏你的人充满敬意。公主怎么会不爱上你...\n",
            "\n",
            "                         (展开)\n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                1278\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                92\n",
            "                        \n",
            "\n",
            "162回应\n",
            "收起\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "硕大无朋\n",
            "\n",
            "2008-02-23 03:37:05\n",
            "\n",
            "\n",
            "生活不完美却仍旧很美\n",
            "\n",
            "\n",
            "\n",
            "                        刚看完美丽人生，哭得稀里哗啦。我看电视电影一般不怎么被感动，因为现在充斥荧屏的所谓催人泪下的故事要么是费尽千辛万苦才在一起的男女主人公其中一个身患绝症而亡的韩剧，要么是单亲家长含辛茹苦养大非亲生儿女而其亲生父亲或母亲在孩子长成之后又突然出现产生各种纠葛的逻...\n",
            "\n",
            "                         (展开)\n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                671\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                45\n",
            "                        \n",
            "\n",
            "149回应\n",
            "收起\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Emil\n",
            "\n",
            "2007-10-22 01:24:04\n",
            "\n",
            "\n",
            "Buono giorno, Principessa!\n",
            "\n",
            "\n",
            "这篇影评可能有剧透\n",
            "\n",
            "                        说一些细节：    1. 影片一开始就揶揄了一把Nazi：Guido和Ferruccio的车因为刹车失灵冲下山，车窗被植物挡住，Guido就站起来挥手示意人群让开，那个挥手的动作就是Nazi的军礼，也因此他们才被当成是来视察的高官。    2. 我以前老把“Principessa”记成“Princepessa”，这把看...\n",
            "\n",
            "                         (展开)\n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                416\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                41\n",
            "                        \n",
            "\n",
            "68回应\n",
            "收起\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "不散\n",
            "\n",
            "2020-01-04 18:59:55\n",
            "\n",
            "\n",
            "跨越23年的经典，背后还有这些不为人知的事\n",
            "\n",
            "\n",
            "\n",
            "                        本世纪的20年代还没过上几天，我们却要开始怀旧了。 《美丽人生》，豆瓣上这部影片的上映日期让我感触颇深，1997到2020年，基本能算个90后的青年了。 然而这并不是它首次引入内地，其实2001年3月《美丽人生》就曾在内地上映，而这一次全国公映的是本片的“4K修复版”，他们甚至...\n",
            "\n",
            "                         (展开)\n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                361\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                                1\n",
            "                        \n",
            "\n",
            "18回应\n",
            "收起\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                    >\n",
            "                        \n",
            "                            更多影评\n",
            "                                5441篇\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "添加新讨论\n",
            "\n",
            "        讨论区\n",
            "           ·  ·  ·  ·  ·  ·\n",
            "    \n",
            "\n",
            "\n",
            "\n",
            "去了解犹太教被屠这类的历史应该看哪些书啊？\n",
            "来自阿长呀阿短\n",
            "12 回应\n",
            "2023-10-14 09:15:24\n",
            "\n",
            "\n",
            "请问10分钟都删减了哪些\n",
            "来自李焱汕\n",
            "\n",
            "2023-10-10 00:29:51\n",
            "\n",
            "\n",
            "3星我喜欢开头的喜剧不喜欢后半部分！我喜欢国语配...\n",
            "来自MrZongli\n",
            "4 回应\n",
            "2023-10-04 18:12:33\n",
            "\n",
            "\n",
            "我感觉这部电影应该是比上面几部要好的\n",
            "来自志远\n",
            "5 回应\n",
            "2023-10-02 17:50:05\n",
            "\n",
            "\n",
            "喜剧的背后\n",
            "来自张悦悦\n",
            "\n",
            "2023-10-01 03:26:32\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                            > 去这部影片的讨论区（全部1067条）\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                关于《美丽人生》的问题\n",
            "                · · · · · ·\n",
            "            \n",
            "                (\n",
            "                    全部45个\n",
            "                )\n",
            "            \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                            为什么“做爱”可以很直白地说出来？\n",
            "                    \n",
            "\n",
            "\n",
            "                    18人回答\n",
            "                \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                            最后一夜圭多如果没有把儿子放进箱子里面而是和其他人一起待命他们一家人会不会团聚呢？\n",
            "                    \n",
            "\n",
            "\n",
            "                    23人回答\n",
            "                \n",
            "\n",
            "\n",
            ">\n",
            "            \n",
            "                全部45个问题\n",
            "            \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "        在哪儿看这部电影\n",
            "             · · · · · ·\n",
            "    \n",
            "\n",
            "\n",
            "\n",
            "                            腾讯视频\n",
            "                        \n",
            "\n",
            "                        VIP免费观看\n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "                            爱奇艺视频\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "                            哔哩哔哩\n",
            "                        \n",
            "\n",
            "                        VIP免费观看\n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "以下片单推荐\n",
            "              · · · · · ·\n",
            "            \n",
            "            (\n",
            "                全部\n",
            "            )\n",
            "            \n",
            "\n",
            "\n",
            "\n",
            "★豆瓣高分电影榜★ （上）9.7-8.6分\n",
            "(影志)\n",
            "\n",
            "\n",
            "【内牛满面】\n",
            "(影志)\n",
            "\n",
            "\n",
            "【励志电影】\n",
            "(影志)\n",
            "\n",
            "\n",
            "那些牛叉闪闪的片子\n",
            "(家庭种菜笔记)\n",
            "\n",
            "\n",
            "入选《第十放映室》影片之国外篇Ⅰ\n",
            "(心之旋影)\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "谁在看这部电影\n",
            "              · · · · · ·\n",
            "    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "我睡觉时候不困\n",
            "\n",
            "                        刚刚\n",
            "                        想看\n",
            "                        \n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Nirvana\n",
            "\n",
            "                        2分钟前\n",
            "                        看过\n",
            "                        \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "想讲你知\n",
            "\n",
            "                        3分钟前\n",
            "                        想看\n",
            "                        \n",
            "                    \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "2221585人看过\n",
            "                 / \n",
            "            695822人想看\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "订阅美丽人生的评论: \n",
            " feed: rss 2.0\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "    © 2005－2023 douban.com, all rights reserved 北京豆网科技有限公司\n",
            "\n",
            "\n",
            "\n",
            "关于豆瓣\n",
            "    · 在豆瓣工作\n",
            "    · 联系我们\n",
            "    · 法律声明\n",
            "    \n",
            "    · 帮助中心\n",
            "    · 移动应用\n",
            "    · 豆瓣广告\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install unstructured"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7vj1_3cc0JHj",
        "outputId": "480d03dd-9aa1-4bf4-a004-8fd05ceb0681"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting unstructured\n",
            "  Downloading unstructured-0.10.22-py3-none-any.whl (1.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: chardet in /usr/local/lib/python3.10/dist-packages (from unstructured) (5.2.0)\n",
            "Collecting filetype (from unstructured)\n",
            "  Downloading filetype-1.2.0-py2.py3-none-any.whl (19 kB)\n",
            "Collecting python-magic (from unstructured)\n",
            "  Downloading python_magic-0.4.27-py2.py3-none-any.whl (13 kB)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.9.3)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.8.1)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.9.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.31.0)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.11.2)\n",
            "Collecting emoji (from unstructured)\n",
            "  Downloading emoji-2.8.0-py2.py3-none-any.whl (358 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m358.9/358.9 kB\u001b[0m \u001b[31m14.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: dataclasses-json in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.6.1)\n",
            "Collecting python-iso639 (from unstructured)\n",
            "  Downloading python_iso639-2023.6.15-py3-none-any.whl (275 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m275.1/275.1 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting langdetect (from unstructured)\n",
            "  Downloading langdetect-1.0.9.tar.gz (981 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m981.5/981.5 kB\u001b[0m \u001b[31m21.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.23.5)\n",
            "Collecting rapidfuzz (from unstructured)\n",
            "  Downloading rapidfuzz-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.2/3.2 MB\u001b[0m \u001b[31m30.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: backoff in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.2.1)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->unstructured) (2.5)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->unstructured) (3.20.1)\n",
            "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->unstructured) (0.9.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from langdetect->unstructured) (1.16.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (1.3.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (2023.6.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (4.64.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.3.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2.0.6)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2023.7.22)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->unstructured) (23.2)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json->unstructured) (1.0.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.10/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json->unstructured) (4.5.0)\n",
            "Building wheels for collected packages: langdetect\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for langdetect: filename=langdetect-1.0.9-py3-none-any.whl size=993224 sha256=d033d148a8933fc1141fddb707903260f5362f89f971661569f951945f6ea9fb\n",
            "  Stored in directory: /root/.cache/pip/wheels/95/03/7d/59ea870c70ce4e5a370638b5462a7711ab78fba2f655d05106\n",
            "Successfully built langdetect\n",
            "Installing collected packages: filetype, rapidfuzz, python-magic, python-iso639, langdetect, emoji, unstructured\n",
            "Successfully installed emoji-2.8.0 filetype-1.2.0 langdetect-1.0.9 python-iso639-2023.6.15 python-magic-0.4.27 rapidfuzz-3.4.0 unstructured-0.10.22\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import UnstructuredURLLoader\n",
        "\n",
        "loader = UnstructuredURLLoader(\n",
        "    ['https://python.langchain.com/docs/get_started/introduction'],\n",
        "    )\n",
        "\n",
        "for x in loader.load():\n",
        "    print(x.page_content)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "412iM9DUyYtm",
        "outputId": "a2a0b6ce-c276-4c25-e8a2-6db221c54f06"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "Get started\n",
            "\n",
            "Introduction\n",
            "\n",
            "Introduction\n",
            "\n",
            "LangChain is a framework for developing applications powered by language models. It enables applications that:\n",
            "\n",
            "Are context-aware: connect a language model to sources of context (prompt instructions, few shot examples, content to ground its response in, etc.)\n",
            "\n",
            "Reason: rely on a language model to reason (about how to answer based on provided context, what actions to take, etc.)\n",
            "\n",
            "The main value props of LangChain are:\n",
            "\n",
            "Components: abstractions for working with language models, along with a collection of implementations for each abstraction. Components are modular and easy-to-use, whether you are using the rest of the LangChain framework or not\n",
            "\n",
            "Off-the-shelf chains: a structured assembly of components for accomplishing specific higher-level tasks\n",
            "\n",
            "Off-the-shelf chains make it easy to get started. For complex applications, components make it easy to customize existing chains and build new ones.\n",
            "\n",
            "Get started​\n",
            "\n",
            "Here’s how to install LangChain, set up your environment, and start building.\n",
            "\n",
            "We recommend following our Quickstart guide to familiarize yourself with the framework by building your first LangChain application.\n",
            "\n",
            "Note: These docs are for the LangChain Python package. For documentation on LangChain.js, the JS/TS version, head here.\n",
            "\n",
            "Modules​\n",
            "\n",
            "LangChain provides standard, extendable interfaces and external integrations for the following modules, listed from least to most complex:\n",
            "\n",
            "Model I/O​\n",
            "\n",
            "Interface with language models\n",
            "\n",
            "Retrieval​\n",
            "\n",
            "Interface with application-specific data\n",
            "\n",
            "Chains​\n",
            "\n",
            "Construct sequences of calls\n",
            "\n",
            "Agents​\n",
            "\n",
            "Let chains choose which tools to use given high-level directives\n",
            "\n",
            "Memory​\n",
            "\n",
            "Persist application state between runs of a chain\n",
            "\n",
            "Callbacks​\n",
            "\n",
            "Log and stream intermediate steps of any chain\n",
            "\n",
            "Examples, ecosystem, and resources​\n",
            "\n",
            "Use cases​\n",
            "\n",
            "Walkthroughs and best-practices for common end-to-end use cases, like:\n",
            "\n",
            "Document question answering\n",
            "\n",
            "Chatbots\n",
            "\n",
            "Analyzing structured data\n",
            "\n",
            "and much more...\n",
            "\n",
            "Guides​\n",
            "\n",
            "Learn best practices for developing with LangChain.\n",
            "\n",
            "Ecosystem​\n",
            "\n",
            "LangChain is part of a rich ecosystem of tools that integrate with our framework and build on top of it. Check out our growing list of integrations and dependent repos.\n",
            "\n",
            "Additional resources​\n",
            "\n",
            "Our community is full of prolific developers, creative builders, and fantastic teachers. Check out YouTube tutorials for great tutorials from folks in the community, and Gallery for a list of awesome LangChain projects, compiled by the folks at KyroLabs.\n",
            "\n",
            "Community​\n",
            "\n",
            "Head to the Community navigator to find places to ask questions, share feedback, meet other developers, and dream about the future of LLM’s.\n",
            "\n",
            "API reference​\n",
            "\n",
            "Head to the reference section for full documentation of all classes and methods in the LangChain Python package.\n",
            "\n",
            "PreviousGet started\n",
            "\n",
            "NextInstallation\n",
            "\n",
            "Get started\n",
            "\n",
            "Modules\n",
            "\n",
            "Examples, ecosystem, and resourcesUse casesGuidesEcosystemAdditional resourcesCommunity\n",
            "\n",
            "API reference\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## PDF加载器"
      ],
      "metadata": {
        "id": "xDH7f9APzvVo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pypdf"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CebWaz8lI8qD",
        "outputId": "4dfa6a60-5178-428e-99a9-b321f6b7e4af"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pypdf\n",
            "  Downloading pypdf-3.16.4-py3-none-any.whl (276 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/276.6 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━\u001b[0m \u001b[32m204.8/276.6 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m276.6/276.6 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pypdf\n",
            "Successfully installed pypdf-3.16.4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import PyPDFLoader\n",
        "\n",
        "loader = PyPDFLoader('https://arxiv.org/pdf/2302.03803.pdf')\n",
        "\n",
        "for x in loader.load():\n",
        "    print(x.page_content)\n",
        "\n",
        "for x in loader.load_and_split():\n",
        "    print(x.page_content)\n",
        "    print('\\n\\n')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x7OGBuAjI4x7",
        "outputId": "9fd9a915-2c89-40f7-f97a-16cd309bc956"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "arXiv:2302.03803v1  [math.AG]  7 Feb 2023A WEAK(k,k)-LEFSCHETZ THEOREM FOR PROJECTIVE\n",
            "TORIC ORBIFOLDS\n",
            "William D. Montoya\n",
            "Instituto de Matem´ atica, Estat´ ıstica e Computa¸ c˜ ao Cie nt´ ıﬁca,\n",
            "Universidade Estadual de Campinas (UNICAMP),\n",
            "Rua S´ ergio Buarque de Holanda 651, 13083-859, Campinas, SP , Brazil\n",
            "February 9, 2023\n",
            "Abstract\n",
            "Firstly we show a generalization of the (1,1)-Lefschetz theorem for projective\n",
            "toric orbifolds and secondly we prove that on 2 k-dimensional quasi-smooth hyper-\n",
            "surfaces coming from quasi-smooth intersection surfaces, under the Cayley trick,\n",
            "every rational(k,k)-cohomology class is algebraic, i.e., the Hodge conjecture holds\n",
            "on them.\n",
            "1 Introduction\n",
            "In [3] we proved that, under suitable conditions, on a very general codimension squasi-\n",
            "smooth intersection subvariety Xin a projective toric orbifold Pd\n",
            "Σwithd+s=2(k+1)\n",
            "the Hodge conjecture holds, that is, every (p,p)-cohomology class, under the Poincar´ e\n",
            "duality is a rational linear combination of fundamental classes of alge braic subvarieties\n",
            "ofX. The proof of the above-mentioned result relies, for p≠d+1−s, on a Lefschetz\n",
            "Date: February 9, 2023\n",
            "2020 Mathematics Subject Classiﬁcation: 14C30, 14M10, 14J70, 14M25\n",
            "Keywords: (1,1)- Lefschetz theorem, Hodge conjecture, toric varieties, c omplete intersection\n",
            "Email:wmontoya@ime.unicamp.br\n",
            "1\n",
            "theorem ([7]) and the Hard Lefschetz theorem for projective orb ifolds ([11]). When p=\n",
            "d+1−sthe proof relies on the Cayley trick, a trick which associates to Xa quasi-smooth\n",
            "hypersurface Yin a projective vector bundle, and the Cayley Proposition (4.3) which\n",
            "gives an isomorphism of some primitive cohomologies (4.2) of XandY. The Cayley\n",
            "trick, following the philosophy of Mavlyutov in [7], reduces results kn own for quasi-smooth\n",
            "hypersurfaces to quasi-smooth intersection subvarieties. The id ea in this paper goes the\n",
            "other way around, we translate some results for quasi-smooth int ersection subvarieties to\n",
            "quasi-smooth hypersurfaces, mainly the (1,1)-Lefschetz theorem.\n",
            "Acknowledgement. I thank Prof. Ugo Bruzzo and Tiago Fonseca for useful discus-\n",
            "sions. I also acknowledge support from FAPESP postdoctoral gra nt No. 2019/23499-7.\n",
            "2 Preliminaries and Notation\n",
            "2.1 Toric varieties\n",
            "LetMbe a free abelian group of rank d, letN=Hom(M,Z), andNR=N⊗ZR.\n",
            "Deﬁnition 2.1. •A convex subset σ⊂NRis a rational k-dimensional simplicial cone\n",
            "if there exist klinearly independent primitive elements e1,...,e k∈Nsuch that σ=\n",
            "{µ1e1+⋯+µkek}.\n",
            "•The generators eiare integral if for every iand any nonnegative rational number µ\n",
            "the product µeiis inNonly ifµis an integer.\n",
            "•Given two rational simplicial cones σ,σ′one says that σ′is a face of σ(σ′<σ) if\n",
            "the set of integral generators of σ′is a subset of the set of integral generators of σ.\n",
            "•A ﬁnite set Σ={σ1,...,σ t}of rational simplicial cones is called a rational simplicia l\n",
            "complete d-dimensional fan if:\n",
            "1. all faces of cones in Σare inΣ;\n",
            "2. ifσ,σ′∈Σthenσ∩σ′<σandσ∩σ′<σ′;\n",
            "3.NR=σ1∪⋅⋅⋅∪σt.\n",
            "A rational simplicial complete d-dimensional fan Σ deﬁnes a d-dimensional toric variety\n",
            "Pd\n",
            "Σhaving only orbifold singularities which we assume to be projective. Mo reover,T∶=\n",
            "N⊗ZC∗≃(C∗)dis the torus action on Pd\n",
            "Σ. We denote by Σ (i)thei-dimensional cones\n",
            "2\n",
            "of Σ and each ρ∈Σ corresponds to an irreducible T-invariant Weil divisor DρonPd\n",
            "Σ. Let\n",
            "Cl(Σ)be the group of Weil divisors on Pd\n",
            "Σmodule rational equivalences.\n",
            "The total coordinate ring of Pd\n",
            "Σis the polynomial ring S=C[xρ/divides.alt0ρ∈Σ(1)],Shas the\n",
            "Cl(Σ)-grading, aWeil divisor D=∑ρ∈Σ(1)uρDρdetermines themonomial xu∶=∏ρ∈Σ(1)xuρ\n",
            "ρ∈\n",
            "Sand conversely deg (xu)=[D]∈Cl(Σ).\n",
            "For a cone σ∈Σ, ˆσis the set of 1-dimensional cone in Σ that are not contained in σ\n",
            "andxˆσ∶=∏ρ∈ˆσxρis the associated monomial in S.\n",
            "Deﬁnition 2.2. The irrelevant ideal of Pd\n",
            "Σis the monomial ideal BΣ∶=<xˆσ/divides.alt0σ∈Σ>and\n",
            "the zero locus Z(Σ)∶=V(BΣ)in the aﬃne space Ad∶=Spec(S)is the irrelevant locus.\n",
            "Proposition 2.3 (Theorem 5.1.11 [5]) .The toric variety Pd\n",
            "Σis a categorical quotient\n",
            "Ad∖Z(Σ)by the group Hom(Cl(Σ),C∗)and the group action is induced by the Cl(Σ)-\n",
            "grading of S.\n",
            "2.2 Orbifolds\n",
            "Now we give a brief introduction to complex orbifolds and we mention th e needed theorems\n",
            "for the next section. Namely: de Rham theorem and Dolbeault theor em for complex\n",
            "orbifolds.\n",
            "Deﬁnition 2.4. A complex orbifold of complex dimension dis a singular complex space\n",
            "whose singularities are locally isomorphic to quotient sin gularities Cd/slash.leftG, for ﬁnite sub-\n",
            "groupsG⊂Gl(d,C).\n",
            "Deﬁnition 2.5. A diﬀerential form on a complex orbifold Zis deﬁned locally at z∈Zas\n",
            "aG-invariant diﬀerential form on CdwhereG⊂Gl(d,C)andZis locally isomorphic to\n",
            "Cd/slash.leftGaroundz.\n",
            "Roughly speaking the local geometry of orbifolds reduces to local G-invariant geometry.\n",
            "Wehaveacomplex ofdiﬀerential forms (A●(Z),d)andadoublecomplex (A●,●(Z),∂,¯∂)\n",
            "of bigraded diﬀerential forms which deﬁne the de Rham and the Dolbe ault cohomology\n",
            "groups (for a ﬁxed p∈N) respectively:\n",
            "H●\n",
            "dR(Z,C)∶=kerd\n",
            "imdandHp,●(Z,¯∂)∶=ker¯∂\n",
            "im¯∂\n",
            "Theorem 2.6 (Theorem 3.4.4 in [4] and Theorem 1.2 in [1] ) .LetZbe a compact complex\n",
            "orbifold. There are natural isomorphisms:\n",
            "3\n",
            "•H●\n",
            "dR(Z,C)≃H●(Z,C)\n",
            "•Hp,●(Z,¯∂)≃H●(X,Ωp\n",
            "Z)\n",
            "3 (1,1)-Lefschetztheoremfor projectivetoricorbifolds\n",
            "Deﬁnition 3.1. A subvariety X⊂Pd\n",
            "Σis quasi-smooth if V(IX)⊂A#Σ(1)is smooth outside\n",
            "Z(Σ).\n",
            "Example 3.2.Quasi-smoothhypersurfaces ormoregenerallyquasi-smoothinte rsection sub-\n",
            "varieties are quasi-smooth subvarieties (see [2] or [7] for more det ails).\n",
            "△\n",
            "Remark3.3.Quasi-smooth subvarieties are suborbifolds of Pd\n",
            "Σin the sense of Satake in [8].\n",
            "Intuitively speaking they are subvarieties whose only singularities co me from the ambient\n",
            "space.\n",
            "△\n",
            "Theorem 3.4. LetX⊂Pd\n",
            "Σbe a quasi-smooth subvariety. Then every (1,1)-cohomology\n",
            "classλ∈H1,1(X)∩H2(X,Z)is algebraic\n",
            "Proof.From the exponential short exact sequence\n",
            "0→Z→OX→O∗\n",
            "X→0\n",
            "we have a long exact sequence in cohomology\n",
            "H1(O∗\n",
            "X)→H2(X,Z)→H2(OX)≃H0,2(X)\n",
            "where the last isomorphisms is due to Steenbrink in [9]. Now, it is enoug h to prove the\n",
            "commutativity of the next diagram\n",
            "H2(X,Z)→→\n",
            "↓↓H2(X,OX)\n",
            "≃Dolbeault\n",
            "↓↓H2(X,C)\n",
            "deRham≃\n",
            "↓↓\n",
            "H2\n",
            "dR(X,C)→→H0,2\n",
            "¯∂(X)\n",
            "4\n",
            "The key points are the de Rham and Dolbeault’s isomorphisms for orbif olds. The rest\n",
            "of the proof follows as the (1,1)-Lefschetz theorem in [6].\n",
            "Remark 3.5.Fork=1 andPd\n",
            "Σas the projective space, we recover the classical (1,1)-\n",
            "Lefschetz theorem.\n",
            "△\n",
            "By the Hard Lefschetz Theorem for projective orbifolds (see [11] for details) we get an\n",
            "isomorphism of cohomologies :\n",
            "H●(X,Q)≃H2dimX−●(X,Q)\n",
            "given by the Lefschetz morphism and since it is a morphism of Hodge st ructures, we have:\n",
            "H1,1(X,Q)≃HdimX−1,dimX−1(X,Q)\n",
            "ForXas before:\n",
            "Corollary 3.6. If the dimension of Xis1,2or3. The Hodge conjecture holds on X.\n",
            "Proof.If thedimCX=1 the result is clear by the Hard Lefschetz theorem for projective\n",
            "orbifolds. The dimension 2 and3 cases arecovered by Theorem 3.5 an dtheHardLefschetz.\n",
            "theorem.\n",
            "4 Cayley trick and Cayley proposition\n",
            "The Cayley trick is a way to associate to a quasi-smooth intersection subvariety a quasi-\n",
            "smooth hypersurface. Let L1,...,L sbe line bundles on Pd\n",
            "Σand letπ∶P(E)→Pd\n",
            "Σbe the\n",
            "projective space bundle associated to the vector bundle E=L1⊕⋯⊕Ls. It is known that\n",
            "P(E)is a(d+s−1)-dimensional simplicial toric variety whose fan depends on the degre es\n",
            "of the line bundles and the fan Σ. Furthermore, if the Cox ring, witho ut considering the\n",
            "grading, of Pd\n",
            "ΣisC[x1,...,x m]then the Cox ring of P(E)is\n",
            "C[x1,...,x m,y1,...,y s]\n",
            "Moreoverfor Xaquasi-smoothintersectionsubvarietycutoﬀby f1,...,f swithdeg(fi)=\n",
            "[Li]we relate the hypersurface Ycut oﬀ by F=y1f1+⋅⋅⋅+ysfswhich turns out to be\n",
            "quasi-smooth. For more details see Section 2 in [7].\n",
            "5\n",
            "We will denote P(E)asPd+s−1\n",
            "Σ,Xto keep track of its relation with XandPd\n",
            "Σ.\n",
            "The following is a key remark.\n",
            "Remark4.1.There is a morphism ι∶X→Y⊂Pd+s−1\n",
            "Σ,X. Moreover every point z∶=(x,y)∈Y\n",
            "withy≠0 has a preimage. Hence for any subvariety W=V(IW)⊂X⊂Pd\n",
            "Σthere exists\n",
            "W′⊂Y⊂Pd+s−1\n",
            "Σ,Xsuch that π(W′)=W, i.e.,W′={z=(x,y)/divides.alt0x∈W}.\n",
            "△\n",
            "ForX⊂Pd\n",
            "Σa quasi-smooth intersection variety the morphism in cohomology indu ced\n",
            "by the inclusion i∗∶Hd−s(Pd\n",
            "Σ,C)→Hd−s(X,C)is injective by Proposition 1.4 in [7].\n",
            "Deﬁnition 4.2. Theprimitive cohomologyof Hd−s\n",
            "prim(X)is the quotient Hd−s(X,C)/slash.lefti∗(Hd−s(Pd\n",
            "Σ,C))\n",
            "andHd−s\n",
            "prim(X,Q)with rational coeﬃcients.\n",
            "Hd−s(Pd\n",
            "Σ,C)andHd−s(X,C)have pure Hodge structures, and the morphism i∗is com-\n",
            "patible with them, so that Hd−s\n",
            "prim(X)gets a pure Hodge structure.\n",
            "The next Proposition is the Cayley proposition.\n",
            "Proposition 4.3. [Proposition 2.3 in [3] ] Let X=X1∩⋅⋅⋅∩Xsbe a quasi-smooth intersec-\n",
            "tion subvariety in Pd\n",
            "Σcut oﬀ by homogeneous polynomials f1...fs. Then for p≠d+s−1\n",
            "2,d+s−3\n",
            "2\n",
            "Hp−1,d+s−1−p\n",
            "prim(Y)≃Hp−s,d−p\n",
            "prim(X).\n",
            "Corollary 4.4. Ifd+s=2(k+1),\n",
            "Hk+1−s,k+1−s\n",
            "prim(X)≃Hk,k\n",
            "prim(Y)\n",
            "Remark4.5.Theaboveisomorphismsarealsotruewithrationalcoeﬃcientssince H●(X,C)=\n",
            "H●(X,Q)⊗QC.See the beginning of Section 7.1 in [10] for more details.\n",
            "△\n",
            "5 Main result\n",
            "Theorem 5.1. LetY={F=y1f1+⋯+ykfk=0}⊂P2k+1\n",
            "Σ,Xbe the quasi-smooth hypersurface\n",
            "associated to the quasi-smooth intersection surface X=Xf1∩⋅⋅⋅∩Xfk⊂Pk+2\n",
            "Σ. Then on Y\n",
            "the Hodge conjecture holds.\n",
            "Proof.IfHk,k\n",
            "prim(X,Q)=0 we are done. So let us assume Hk,k\n",
            "prim(X,Q)≠0. By the Cayley\n",
            "proposition Hk,k\n",
            "prim(Y,Q)≃H1,1\n",
            "prim(X,Q)and by the(1,1)-Lefschetz theorem for projective\n",
            "6\n",
            "toric orbifolds there is a non-zero algebraic basis λC1,...,λ Cnwith rational coeﬃcients of\n",
            "H1,1\n",
            "prim(X,Q), that is, there are n∶=h1,1\n",
            "prim(X,Q)algebraic curves C1,...,C ninXsuch that\n",
            "under the Poincar´ e duality the class in homology [Ci]goes toλCi,[Ci]↦λCi. Recall\n",
            "that the Cox ring of Pk+2is contained in the Cox ring of P2k+1\n",
            "Σ,Xwithout considering the\n",
            "grading. Considering the grading we have that if α∈Cl(Pk+2\n",
            "Σ)then(α,0)∈Cl(P2k+1\n",
            "Σ,X). So\n",
            "the polynomials deﬁning Ci⊂Pk+2\n",
            "Σcan be interpreted in P2k+1\n",
            "X,Σbut with diﬀerent degree.\n",
            "Moreover, by Remark 4.1 each Ciis contained in Y={F=y1f1+⋯+ykfk=0}and\n",
            "furthermore it has codimension k.\n",
            "Claim:{λCi}n\n",
            "i=1is a basis of Hk,k\n",
            "prim(Y,Q).\n",
            "It is enough to prove that λCiis diﬀerent from zero in Hk,k\n",
            "prim(Y,Q)or equivalently that the\n",
            "cohomology classes {λCi}n\n",
            "i=1do not come from the ambient space. By contradiction, let us\n",
            "assume that there exists a jandC⊂P2k+1\n",
            "Σ,Xsuch that λC∈Hk,k(P2k+1\n",
            "Σ,X,Q)withi∗(λC)=λCj\n",
            "or in terms of homology there exists a (k+2)-dimensional algebraic subvariety V⊂P2k+1\n",
            "Σ,X\n",
            "such that V∩Y=Cjso they are equal as a homology class of P2k+1\n",
            "Σ,X,i.e.,[V∩Y]=[Cj].\n",
            "It is easy to check that π(V)∩X=Cjas a subvariety of Pk+2\n",
            "Σwhereπ∶(x,y)↦x. Hence\n",
            "[π(V)∩X]=[Cj]which is equivalent to say that λCjcomes from Pk+2\n",
            "Σwhich contradicts\n",
            "the choice of[Cj].\n",
            "Remark 5.2.Into the proof of the previous theorem, the key fact was that on Xthe\n",
            "Hodge conjecture holds and we translate it to Yby contradiction. So, using an analogous\n",
            "argument we have:\n",
            "△\n",
            "Proposition 5.3. LetY={F=y1fs+⋯+ysfs=0}⊂P2k+1\n",
            "Σ,Xbe the quasi-smooth hypersurface\n",
            "associated to a quasi-smooth intersection subvariety X=Xf1∩⋅⋅⋅∩Xfs⊂Pd\n",
            "Σsuch that\n",
            "d+s=2(k+1). If the Hodge conjecture holds on Xthen it holds as well on Y.\n",
            "Corollary 5.4. If the dimension of Yis2s−1,2sor2s+1then the Hodge conjecture\n",
            "holds on Y.\n",
            "Proof.By Proposition 5.3 and Corollary 3.6.\n",
            "7\n",
            "References\n",
            "[1]Angella, D. Cohomologies of certain orbifolds. Journal of Geometry and Physics\n",
            "71(2013), 117–126.\n",
            "[2]Batyrev, V. V., and Cox, D. A. On the Hodge structure of projective hypersur-\n",
            "faces in toric varieties. Duke Mathematical Journal 75, 2 (Aug 199 4).\n",
            "[3]Bruzzo, U., and Montoya, W. On the Hodge conjecture for quasi-smooth in-\n",
            "tersections in toric varieties. S˜ ao Paulo J. Math. Sci. Special Section: Geometry in\n",
            "Algebra and Algebra in Geometry (2021).\n",
            "[4]Caramello Jr, F. C. Introduction to orbifolds. arXiv:1909.08699v6 (2019).\n",
            "[5]Cox, D., Little, J., and Schenck, H. Toric varieties, vol. 124. American Math-\n",
            "ematical Soc., 2011.\n",
            "[6]Griffiths, P., and Harris, J. Principles of Algebraic Geometry. John Wiley &\n",
            "Sons, Ltd, 1978.\n",
            "[7]Mavlyutov, A. R. Cohomology of complete intersections in toric varieties. Pub-\n",
            "lished in Paciﬁc J. of Math. 191 No. 1 (1999), 133–144.\n",
            "[8]Satake, I. On a Generalization of the Notion of Manifold. Proceedings of the\n",
            "National Academy of Sciences of the United States of America 42, 6 (1956), 359–363.\n",
            "[9]Steenbrink, J. H. M. Intersection form for quasi-homogeneous singularities. Com-\n",
            "positio Mathematica 34 , 2 (1977), 211–223.\n",
            "[10]Voisin, C. Hodge Theory and Complex Algebraic Geometry I, vol. 1 of Cambridge\n",
            "Studies in Advanced Mathematics .Cambridge University Press, 2002.\n",
            "[11]Wang, Z. Z., and Zaffran, D. A remark on the Hard Lefschetz theorem for K¨ ahler\n",
            "orbifolds. Proceedings of the American Mathematical Society 137 , 08 (Aug 2009).\n",
            "8\n",
            "arXiv:2302.03803v1  [math.AG]  7 Feb 2023A WEAK(k,k)-LEFSCHETZ THEOREM FOR PROJECTIVE\n",
            "TORIC ORBIFOLDS\n",
            "William D. Montoya\n",
            "Instituto de Matem´ atica, Estat´ ıstica e Computa¸ c˜ ao Cie nt´ ıﬁca,\n",
            "Universidade Estadual de Campinas (UNICAMP),\n",
            "Rua S´ ergio Buarque de Holanda 651, 13083-859, Campinas, SP , Brazil\n",
            "February 9, 2023\n",
            "Abstract\n",
            "Firstly we show a generalization of the (1,1)-Lefschetz theorem for projective\n",
            "toric orbifolds and secondly we prove that on 2 k-dimensional quasi-smooth hyper-\n",
            "surfaces coming from quasi-smooth intersection surfaces, under the Cayley trick,\n",
            "every rational(k,k)-cohomology class is algebraic, i.e., the Hodge conjecture holds\n",
            "on them.\n",
            "1 Introduction\n",
            "In [3] we proved that, under suitable conditions, on a very general codimension squasi-\n",
            "smooth intersection subvariety Xin a projective toric orbifold Pd\n",
            "Σwithd+s=2(k+1)\n",
            "the Hodge conjecture holds, that is, every (p,p)-cohomology class, under the Poincar´ e\n",
            "duality is a rational linear combination of fundamental classes of alge braic subvarieties\n",
            "ofX. The proof of the above-mentioned result relies, for p≠d+1−s, on a Lefschetz\n",
            "Date: February 9, 2023\n",
            "2020 Mathematics Subject Classiﬁcation: 14C30, 14M10, 14J70, 14M25\n",
            "Keywords: (1,1)- Lefschetz theorem, Hodge conjecture, toric varieties, c omplete intersection\n",
            "Email:wmontoya@ime.unicamp.br\n",
            "1\n",
            "\n",
            "\n",
            "\n",
            "theorem ([7]) and the Hard Lefschetz theorem for projective orb ifolds ([11]). When p=\n",
            "d+1−sthe proof relies on the Cayley trick, a trick which associates to Xa quasi-smooth\n",
            "hypersurface Yin a projective vector bundle, and the Cayley Proposition (4.3) which\n",
            "gives an isomorphism of some primitive cohomologies (4.2) of XandY. The Cayley\n",
            "trick, following the philosophy of Mavlyutov in [7], reduces results kn own for quasi-smooth\n",
            "hypersurfaces to quasi-smooth intersection subvarieties. The id ea in this paper goes the\n",
            "other way around, we translate some results for quasi-smooth int ersection subvarieties to\n",
            "quasi-smooth hypersurfaces, mainly the (1,1)-Lefschetz theorem.\n",
            "Acknowledgement. I thank Prof. Ugo Bruzzo and Tiago Fonseca for useful discus-\n",
            "sions. I also acknowledge support from FAPESP postdoctoral gra nt No. 2019/23499-7.\n",
            "2 Preliminaries and Notation\n",
            "2.1 Toric varieties\n",
            "LetMbe a free abelian group of rank d, letN=Hom(M,Z), andNR=N⊗ZR.\n",
            "Deﬁnition 2.1. •A convex subset σ⊂NRis a rational k-dimensional simplicial cone\n",
            "if there exist klinearly independent primitive elements e1,...,e k∈Nsuch that σ=\n",
            "{µ1e1+⋯+µkek}.\n",
            "•The generators eiare integral if for every iand any nonnegative rational number µ\n",
            "the product µeiis inNonly ifµis an integer.\n",
            "•Given two rational simplicial cones σ,σ′one says that σ′is a face of σ(σ′<σ) if\n",
            "the set of integral generators of σ′is a subset of the set of integral generators of σ.\n",
            "•A ﬁnite set Σ={σ1,...,σ t}of rational simplicial cones is called a rational simplicia l\n",
            "complete d-dimensional fan if:\n",
            "1. all faces of cones in Σare inΣ;\n",
            "2. ifσ,σ′∈Σthenσ∩σ′<σandσ∩σ′<σ′;\n",
            "3.NR=σ1∪⋅⋅⋅∪σt.\n",
            "A rational simplicial complete d-dimensional fan Σ deﬁnes a d-dimensional toric variety\n",
            "Pd\n",
            "Σhaving only orbifold singularities which we assume to be projective. Mo reover,T∶=\n",
            "N⊗ZC∗≃(C∗)dis the torus action on Pd\n",
            "Σ. We denote by Σ (i)thei-dimensional cones\n",
            "2\n",
            "\n",
            "\n",
            "\n",
            "of Σ and each ρ∈Σ corresponds to an irreducible T-invariant Weil divisor DρonPd\n",
            "Σ. Let\n",
            "Cl(Σ)be the group of Weil divisors on Pd\n",
            "Σmodule rational equivalences.\n",
            "The total coordinate ring of Pd\n",
            "Σis the polynomial ring S=C[xρ/divides.alt0ρ∈Σ(1)],Shas the\n",
            "Cl(Σ)-grading, aWeil divisor D=∑ρ∈Σ(1)uρDρdetermines themonomial xu∶=∏ρ∈Σ(1)xuρ\n",
            "ρ∈\n",
            "Sand conversely deg (xu)=[D]∈Cl(Σ).\n",
            "For a cone σ∈Σ, ˆσis the set of 1-dimensional cone in Σ that are not contained in σ\n",
            "andxˆσ∶=∏ρ∈ˆσxρis the associated monomial in S.\n",
            "Deﬁnition 2.2. The irrelevant ideal of Pd\n",
            "Σis the monomial ideal BΣ∶=<xˆσ/divides.alt0σ∈Σ>and\n",
            "the zero locus Z(Σ)∶=V(BΣ)in the aﬃne space Ad∶=Spec(S)is the irrelevant locus.\n",
            "Proposition 2.3 (Theorem 5.1.11 [5]) .The toric variety Pd\n",
            "Σis a categorical quotient\n",
            "Ad∖Z(Σ)by the group Hom(Cl(Σ),C∗)and the group action is induced by the Cl(Σ)-\n",
            "grading of S.\n",
            "2.2 Orbifolds\n",
            "Now we give a brief introduction to complex orbifolds and we mention th e needed theorems\n",
            "for the next section. Namely: de Rham theorem and Dolbeault theor em for complex\n",
            "orbifolds.\n",
            "Deﬁnition 2.4. A complex orbifold of complex dimension dis a singular complex space\n",
            "whose singularities are locally isomorphic to quotient sin gularities Cd/slash.leftG, for ﬁnite sub-\n",
            "groupsG⊂Gl(d,C).\n",
            "Deﬁnition 2.5. A diﬀerential form on a complex orbifold Zis deﬁned locally at z∈Zas\n",
            "aG-invariant diﬀerential form on CdwhereG⊂Gl(d,C)andZis locally isomorphic to\n",
            "Cd/slash.leftGaroundz.\n",
            "Roughly speaking the local geometry of orbifolds reduces to local G-invariant geometry.\n",
            "Wehaveacomplex ofdiﬀerential forms (A●(Z),d)andadoublecomplex (A●,●(Z),∂,¯∂)\n",
            "of bigraded diﬀerential forms which deﬁne the de Rham and the Dolbe ault cohomology\n",
            "groups (for a ﬁxed p∈N) respectively:\n",
            "H●\n",
            "dR(Z,C)∶=kerd\n",
            "imdandHp,●(Z,¯∂)∶=ker¯∂\n",
            "im¯∂\n",
            "Theorem 2.6 (Theorem 3.4.4 in [4] and Theorem 1.2 in [1] ) .LetZbe a compact complex\n",
            "orbifold. There are natural isomorphisms:\n",
            "3\n",
            "\n",
            "\n",
            "\n",
            "•H●\n",
            "dR(Z,C)≃H●(Z,C)\n",
            "•Hp,●(Z,¯∂)≃H●(X,Ωp\n",
            "Z)\n",
            "3 (1,1)-Lefschetztheoremfor projectivetoricorbifolds\n",
            "Deﬁnition 3.1. A subvariety X⊂Pd\n",
            "Σis quasi-smooth if V(IX)⊂A#Σ(1)is smooth outside\n",
            "Z(Σ).\n",
            "Example 3.2.Quasi-smoothhypersurfaces ormoregenerallyquasi-smoothinte rsection sub-\n",
            "varieties are quasi-smooth subvarieties (see [2] or [7] for more det ails).\n",
            "△\n",
            "Remark3.3.Quasi-smooth subvarieties are suborbifolds of Pd\n",
            "Σin the sense of Satake in [8].\n",
            "Intuitively speaking they are subvarieties whose only singularities co me from the ambient\n",
            "space.\n",
            "△\n",
            "Theorem 3.4. LetX⊂Pd\n",
            "Σbe a quasi-smooth subvariety. Then every (1,1)-cohomology\n",
            "classλ∈H1,1(X)∩H2(X,Z)is algebraic\n",
            "Proof.From the exponential short exact sequence\n",
            "0→Z→OX→O∗\n",
            "X→0\n",
            "we have a long exact sequence in cohomology\n",
            "H1(O∗\n",
            "X)→H2(X,Z)→H2(OX)≃H0,2(X)\n",
            "where the last isomorphisms is due to Steenbrink in [9]. Now, it is enoug h to prove the\n",
            "commutativity of the next diagram\n",
            "H2(X,Z)→→\n",
            "↓↓H2(X,OX)\n",
            "≃Dolbeault\n",
            "↓↓H2(X,C)\n",
            "deRham≃\n",
            "↓↓\n",
            "H2\n",
            "dR(X,C)→→H0,2\n",
            "¯∂(X)\n",
            "4\n",
            "\n",
            "\n",
            "\n",
            "The key points are the de Rham and Dolbeault’s isomorphisms for orbif olds. The rest\n",
            "of the proof follows as the (1,1)-Lefschetz theorem in [6].\n",
            "Remark 3.5.Fork=1 andPd\n",
            "Σas the projective space, we recover the classical (1,1)-\n",
            "Lefschetz theorem.\n",
            "△\n",
            "By the Hard Lefschetz Theorem for projective orbifolds (see [11] for details) we get an\n",
            "isomorphism of cohomologies :\n",
            "H●(X,Q)≃H2dimX−●(X,Q)\n",
            "given by the Lefschetz morphism and since it is a morphism of Hodge st ructures, we have:\n",
            "H1,1(X,Q)≃HdimX−1,dimX−1(X,Q)\n",
            "ForXas before:\n",
            "Corollary 3.6. If the dimension of Xis1,2or3. The Hodge conjecture holds on X.\n",
            "Proof.If thedimCX=1 the result is clear by the Hard Lefschetz theorem for projective\n",
            "orbifolds. The dimension 2 and3 cases arecovered by Theorem 3.5 an dtheHardLefschetz.\n",
            "theorem.\n",
            "4 Cayley trick and Cayley proposition\n",
            "The Cayley trick is a way to associate to a quasi-smooth intersection subvariety a quasi-\n",
            "smooth hypersurface. Let L1,...,L sbe line bundles on Pd\n",
            "Σand letπ∶P(E)→Pd\n",
            "Σbe the\n",
            "projective space bundle associated to the vector bundle E=L1⊕⋯⊕Ls. It is known that\n",
            "P(E)is a(d+s−1)-dimensional simplicial toric variety whose fan depends on the degre es\n",
            "of the line bundles and the fan Σ. Furthermore, if the Cox ring, witho ut considering the\n",
            "grading, of Pd\n",
            "ΣisC[x1,...,x m]then the Cox ring of P(E)is\n",
            "C[x1,...,x m,y1,...,y s]\n",
            "Moreoverfor Xaquasi-smoothintersectionsubvarietycutoﬀby f1,...,f swithdeg(fi)=\n",
            "[Li]we relate the hypersurface Ycut oﬀ by F=y1f1+⋅⋅⋅+ysfswhich turns out to be\n",
            "quasi-smooth. For more details see Section 2 in [7].\n",
            "5\n",
            "\n",
            "\n",
            "\n",
            "We will denote P(E)asPd+s−1\n",
            "Σ,Xto keep track of its relation with XandPd\n",
            "Σ.\n",
            "The following is a key remark.\n",
            "Remark4.1.There is a morphism ι∶X→Y⊂Pd+s−1\n",
            "Σ,X. Moreover every point z∶=(x,y)∈Y\n",
            "withy≠0 has a preimage. Hence for any subvariety W=V(IW)⊂X⊂Pd\n",
            "Σthere exists\n",
            "W′⊂Y⊂Pd+s−1\n",
            "Σ,Xsuch that π(W′)=W, i.e.,W′={z=(x,y)/divides.alt0x∈W}.\n",
            "△\n",
            "ForX⊂Pd\n",
            "Σa quasi-smooth intersection variety the morphism in cohomology indu ced\n",
            "by the inclusion i∗∶Hd−s(Pd\n",
            "Σ,C)→Hd−s(X,C)is injective by Proposition 1.4 in [7].\n",
            "Deﬁnition 4.2. Theprimitive cohomologyof Hd−s\n",
            "prim(X)is the quotient Hd−s(X,C)/slash.lefti∗(Hd−s(Pd\n",
            "Σ,C))\n",
            "andHd−s\n",
            "prim(X,Q)with rational coeﬃcients.\n",
            "Hd−s(Pd\n",
            "Σ,C)andHd−s(X,C)have pure Hodge structures, and the morphism i∗is com-\n",
            "patible with them, so that Hd−s\n",
            "prim(X)gets a pure Hodge structure.\n",
            "The next Proposition is the Cayley proposition.\n",
            "Proposition 4.3. [Proposition 2.3 in [3] ] Let X=X1∩⋅⋅⋅∩Xsbe a quasi-smooth intersec-\n",
            "tion subvariety in Pd\n",
            "Σcut oﬀ by homogeneous polynomials f1...fs. Then for p≠d+s−1\n",
            "2,d+s−3\n",
            "2\n",
            "Hp−1,d+s−1−p\n",
            "prim(Y)≃Hp−s,d−p\n",
            "prim(X).\n",
            "Corollary 4.4. Ifd+s=2(k+1),\n",
            "Hk+1−s,k+1−s\n",
            "prim(X)≃Hk,k\n",
            "prim(Y)\n",
            "Remark4.5.Theaboveisomorphismsarealsotruewithrationalcoeﬃcientssince H●(X,C)=\n",
            "H●(X,Q)⊗QC.See the beginning of Section 7.1 in [10] for more details.\n",
            "△\n",
            "5 Main result\n",
            "Theorem 5.1. LetY={F=y1f1+⋯+ykfk=0}⊂P2k+1\n",
            "Σ,Xbe the quasi-smooth hypersurface\n",
            "associated to the quasi-smooth intersection surface X=Xf1∩⋅⋅⋅∩Xfk⊂Pk+2\n",
            "Σ. Then on Y\n",
            "the Hodge conjecture holds.\n",
            "Proof.IfHk,k\n",
            "prim(X,Q)=0 we are done. So let us assume Hk,k\n",
            "prim(X,Q)≠0. By the Cayley\n",
            "proposition Hk,k\n",
            "prim(Y,Q)≃H1,1\n",
            "prim(X,Q)and by the(1,1)-Lefschetz theorem for projective\n",
            "6\n",
            "\n",
            "\n",
            "\n",
            "toric orbifolds there is a non-zero algebraic basis λC1,...,λ Cnwith rational coeﬃcients of\n",
            "H1,1\n",
            "prim(X,Q), that is, there are n∶=h1,1\n",
            "prim(X,Q)algebraic curves C1,...,C ninXsuch that\n",
            "under the Poincar´ e duality the class in homology [Ci]goes toλCi,[Ci]↦λCi. Recall\n",
            "that the Cox ring of Pk+2is contained in the Cox ring of P2k+1\n",
            "Σ,Xwithout considering the\n",
            "grading. Considering the grading we have that if α∈Cl(Pk+2\n",
            "Σ)then(α,0)∈Cl(P2k+1\n",
            "Σ,X). So\n",
            "the polynomials deﬁning Ci⊂Pk+2\n",
            "Σcan be interpreted in P2k+1\n",
            "X,Σbut with diﬀerent degree.\n",
            "Moreover, by Remark 4.1 each Ciis contained in Y={F=y1f1+⋯+ykfk=0}and\n",
            "furthermore it has codimension k.\n",
            "Claim:{λCi}n\n",
            "i=1is a basis of Hk,k\n",
            "prim(Y,Q).\n",
            "It is enough to prove that λCiis diﬀerent from zero in Hk,k\n",
            "prim(Y,Q)or equivalently that the\n",
            "cohomology classes {λCi}n\n",
            "i=1do not come from the ambient space. By contradiction, let us\n",
            "assume that there exists a jandC⊂P2k+1\n",
            "Σ,Xsuch that λC∈Hk,k(P2k+1\n",
            "Σ,X,Q)withi∗(λC)=λCj\n",
            "or in terms of homology there exists a (k+2)-dimensional algebraic subvariety V⊂P2k+1\n",
            "Σ,X\n",
            "such that V∩Y=Cjso they are equal as a homology class of P2k+1\n",
            "Σ,X,i.e.,[V∩Y]=[Cj].\n",
            "It is easy to check that π(V)∩X=Cjas a subvariety of Pk+2\n",
            "Σwhereπ∶(x,y)↦x. Hence\n",
            "[π(V)∩X]=[Cj]which is equivalent to say that λCjcomes from Pk+2\n",
            "Σwhich contradicts\n",
            "the choice of[Cj].\n",
            "Remark 5.2.Into the proof of the previous theorem, the key fact was that on Xthe\n",
            "Hodge conjecture holds and we translate it to Yby contradiction. So, using an analogous\n",
            "argument we have:\n",
            "△\n",
            "Proposition 5.3. LetY={F=y1fs+⋯+ysfs=0}⊂P2k+1\n",
            "Σ,Xbe the quasi-smooth hypersurface\n",
            "associated to a quasi-smooth intersection subvariety X=Xf1∩⋅⋅⋅∩Xfs⊂Pd\n",
            "Σsuch that\n",
            "d+s=2(k+1). If the Hodge conjecture holds on Xthen it holds as well on Y.\n",
            "Corollary 5.4. If the dimension of Yis2s−1,2sor2s+1then the Hodge conjecture\n",
            "holds on Y.\n",
            "Proof.By Proposition 5.3 and Corollary 3.6.\n",
            "7\n",
            "\n",
            "\n",
            "\n",
            "References\n",
            "[1]Angella, D. Cohomologies of certain orbifolds. Journal of Geometry and Physics\n",
            "71(2013), 117–126.\n",
            "[2]Batyrev, V. V., and Cox, D. A. On the Hodge structure of projective hypersur-\n",
            "faces in toric varieties. Duke Mathematical Journal 75, 2 (Aug 199 4).\n",
            "[3]Bruzzo, U., and Montoya, W. On the Hodge conjecture for quasi-smooth in-\n",
            "tersections in toric varieties. S˜ ao Paulo J. Math. Sci. Special Section: Geometry in\n",
            "Algebra and Algebra in Geometry (2021).\n",
            "[4]Caramello Jr, F. C. Introduction to orbifolds. arXiv:1909.08699v6 (2019).\n",
            "[5]Cox, D., Little, J., and Schenck, H. Toric varieties, vol. 124. American Math-\n",
            "ematical Soc., 2011.\n",
            "[6]Griffiths, P., and Harris, J. Principles of Algebraic Geometry. John Wiley &\n",
            "Sons, Ltd, 1978.\n",
            "[7]Mavlyutov, A. R. Cohomology of complete intersections in toric varieties. Pub-\n",
            "lished in Paciﬁc J. of Math. 191 No. 1 (1999), 133–144.\n",
            "[8]Satake, I. On a Generalization of the Notion of Manifold. Proceedings of the\n",
            "National Academy of Sciences of the United States of America 42, 6 (1956), 359–363.\n",
            "[9]Steenbrink, J. H. M. Intersection form for quasi-homogeneous singularities. Com-\n",
            "positio Mathematica 34 , 2 (1977), 211–223.\n",
            "[10]Voisin, C. Hodge Theory and Complex Algebraic Geometry I, vol. 1 of Cambridge\n",
            "Studies in Advanced Mathematics .Cambridge University Press, 2002.\n",
            "[11]Wang, Z. Z., and Zaffran, D. A remark on the Hard Lefschetz theorem for K¨ ahler\n",
            "orbifolds. Proceedings of the American Mathematical Society 137 , 08 (Aug 2009).\n",
            "8\n",
            "\n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install unstructured\n",
        "!pip install pdf2image\n",
        "!pip install pdfminer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "izQwPvlj7Fle",
        "outputId": "8d69cabb-18f0-423d-f03a-e087ee6a45c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: unstructured in /usr/local/lib/python3.10/dist-packages (0.10.23)\n",
            "Requirement already satisfied: chardet in /usr/local/lib/python3.10/dist-packages (from unstructured) (5.2.0)\n",
            "Requirement already satisfied: filetype in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.2.0)\n",
            "Requirement already satisfied: python-magic in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.4.27)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.9.3)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.8.1)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.9.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.31.0)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.11.2)\n",
            "Requirement already satisfied: emoji in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.8.0)\n",
            "Requirement already satisfied: dataclasses-json in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.6.1)\n",
            "Requirement already satisfied: python-iso639 in /usr/local/lib/python3.10/dist-packages (from unstructured) (2023.6.15)\n",
            "Requirement already satisfied: langdetect in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.0.9)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.23.5)\n",
            "Requirement already satisfied: rapidfuzz in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.4.0)\n",
            "Requirement already satisfied: backoff in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.2.1)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->unstructured) (2.5)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->unstructured) (3.20.1)\n",
            "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->unstructured) (0.9.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from langdetect->unstructured) (1.16.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (1.3.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (2023.6.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (4.64.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.3.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2.0.6)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2023.7.22)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->unstructured) (23.2)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json->unstructured) (1.0.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.10/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json->unstructured) (4.5.0)\n",
            "Requirement already satisfied: pdf2image in /usr/local/lib/python3.10/dist-packages (1.16.3)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from pdf2image) (9.4.0)\n",
            "Collecting pdfminer\n",
            "  Downloading pdfminer-20191125.tar.gz (4.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.2/4.2 MB\u001b[0m \u001b[31m18.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pycryptodome (from pdfminer)\n",
            "  Downloading pycryptodome-3.19.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m33.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: pdfminer\n",
            "  Building wheel for pdfminer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pdfminer: filename=pdfminer-20191125-py3-none-any.whl size=6140074 sha256=2bc42dba17be89653ede87666a3db8be141acab4a609763ab06b458bfbcfdcfa\n",
            "  Stored in directory: /root/.cache/pip/wheels/4e/c1/68/f7bd0a8f514661f76b5cbe3b5f76e0033d79f1296012cbbf72\n",
            "Successfully built pdfminer\n",
            "Installing collected packages: pycryptodome, pdfminer\n",
            "Successfully installed pdfminer-20191125 pycryptodome-3.19.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import UnstructuredPDFLoader\n",
        "\n",
        "loader = UnstructuredPDFLoader(\n",
        "    'https://arxiv.org/pdf/2302.03803.pdf'\n",
        ")\n",
        "\n",
        "for x in loader.load():\n",
        "    print(x.page_content)\n"
      ],
      "metadata": {
        "id": "Sle0XRMM3Qbr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 自定义加载器"
      ],
      "metadata": {
        "id": "MOCOWYE90L-h"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from typing import List, Optional\n",
        "\n",
        "from langchain.docstore.document import Document\n",
        "from langchain.document_loaders.base import BaseLoader\n",
        "\n",
        "\n",
        "class TxtLoader(BaseLoader):\n",
        "    def __init__(\n",
        "            self,\n",
        "            filepaths: List[str],\n",
        "            split_str: Optional[str] = None\n",
        "    ):\n",
        "        self.filepaths = filepaths\n",
        "        self.split_str = split_str\n",
        "\n",
        "    def load(self) -> List[Document]:\n",
        "        documents = []\n",
        "        for filepath in self.filepaths:\n",
        "            self._generate_documents(filepath, documents)\n",
        "\n",
        "        return documents\n",
        "\n",
        "    def _generate_documents(self, filepath, documents):\n",
        "        with open(filepath) as f:\n",
        "            data = f.read().strip()\n",
        "\n",
        "        metadata = {\n",
        "            'filepath': filepath,\n",
        "            'split_str': self.split_str\n",
        "        }\n",
        "        if self.split_str is not None:\n",
        "            for split_data in data.strip(self.split_str):\n",
        "                documents.append(\n",
        "                    Document(\n",
        "                        page_content=split_data,\n",
        "                        metadata=metadata\n",
        "                    ))\n",
        "        else:\n",
        "            documents.append(\n",
        "                Document(page_content=data, metadata=metadata)\n",
        "            )\n",
        "\n",
        "\n",
        "txt_loader = TxtLoader([\n",
        "    'sample_data/sample.txt',\n",
        "    'sample_data/sample2.txt'\n",
        "    ])\n",
        "print(txt_loader.load())\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9ULLb9GxV9iq",
        "outputId": "eb15b724-b43c-4701-8c64-7fc77f62f706"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Document(page_content='好猴王，急纵筋斗云，霎时间过了二百里水面。果然那厢有座城池，六街三市，万户千门，来来往往，人都在光天化日之下。悟空心中想道：“这里定有现成的兵器，我待下去买他几件，还不如使个神通觅他几件倒好。”他就捻起诀来，念动咒语，向巽地上吸一口气，呼的吹将去，便是一阵风，飞沙走石，好惊人也。'), Document(page_content='好猴王，急纵筋斗云，霎时间过了二百里水面。果然那厢有座城池，六街三市，万户千门，来来往往，人都在光天化日之下。悟空心中想道：“这里定有现成的兵器，我待下去买他几件，还不如使个神通觅他几件倒好。”他就捻起诀来，念动咒语，向巽地上吸一口气，呼的吹将去，便是一阵风，飞沙走石，好惊人也。')]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Document Transformer（文档转换器）"
      ],
      "metadata": {
        "id": "M141H4w10Qxn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 文本分割"
      ],
      "metadata": {
        "id": "A9RPAY2F0R9X"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "\n",
        "\n",
        "doc_str = \"\"\"Spacewar! \\n\\nis a space combat video game developed \\n\\n\n",
        "in 1962 by Steve Russell in collaboration with Martin Graetz, \\n\\n\n",
        "Wayne Wiitanen, Bob Saunders, Steve Piner, and others. \\n\\n\n",
        "The first video game known to be played at multiple \\n\\n\n",
        "computer installations, \\n\\nit was popular in the small \\n\\n\n",
        "American programming community in the 1960s. \\n\\n\n",
        "Players wage a dogfight between two spaceships with limited \\n\\n\n",
        "weaponry and fuel in the gravity well of a star.\\n\\n\"\"\"\n",
        "\n",
        "text_splitter = CharacterTextSplitter(\n",
        "    separator=\"\\n\\n\",\n",
        "    chunk_size=250,\n",
        "    chunk_overlap=80,\n",
        "    length_function=len,\n",
        ")\n",
        "\n",
        "docs = text_splitter.create_documents([doc_str])\n",
        "for doc in docs:\n",
        "    print(doc)\n",
        "    print(f'length: {len(doc.page_content)}')\n"
      ],
      "metadata": {
        "id": "_HQ0ZdIHWDsp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "84daa21e-b2a4-4168-d08b-175221463bea"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "page_content='Spacewar! \\n\\nis a space combat video game developed \\n\\n\\nin 1962 by Steve Russell in collaboration with Martin Graetz, \\n\\n\\nWayne Wiitanen, Bob Saunders, Steve Piner, and others. \\n\\n\\nThe first video game known to be played at multiple'\n",
            "length: 228\n",
            "page_content='The first video game known to be played at multiple \\n\\n\\ncomputer installations, \\n\\nit was popular in the small \\n\\n\\nAmerican programming community in the 1960s. \\n\\n\\nPlayers wage a dogfight between two spaceships with limited'\n",
            "length: 219\n",
            "page_content='Players wage a dogfight between two spaceships with limited \\n\\n\\nweaponry and fuel in the gravity well of a star.'\n",
            "length: 111\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "\n",
        "doc_str = \"\"\"Spacewar!\n",
        "is a space combat\n",
        "video game developed\n",
        "in 1962, by Steve\n",
        "Russell in\n",
        "collaboration\n",
        "with Martin Graetz, Wayne Wiitanen,\n",
        "Bob Saunders,\n",
        "Steve Piner,\n",
        "and others. \"\"\"\n",
        "\n",
        "text_splitter = RecursiveCharacterTextSplitter(\n",
        "    chunk_size=80,\n",
        "    chunk_overlap=30,\n",
        ")\n",
        "\n",
        "docs = text_splitter.create_documents([doc_str])\n",
        "for doc in docs:\n",
        "    print(doc)\n",
        "    print(f'length: {len(doc.page_content)}')\n",
        "    print('=' * 66)"
      ],
      "metadata": {
        "id": "gsZlbuj00ylA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import tiktoken\n",
        "from langchain.text_splitter import TokenTextSplitter\n",
        "\n",
        "doc_str = \"\"\"Spacewar! is a space combat video game developed\n",
        "in 1962, by Steve Russell in collaboration with Martin Graetz,\n",
        "Wayne Wiitanen, Bob Saunders,Steve Piner, and others. \"\"\"\n",
        "\n",
        "text_splitter = TokenTextSplitter(\n",
        "    encoding_name='gpt2',\n",
        "    chunk_size=12,\n",
        "    chunk_overlap=0\n",
        ")\n",
        "\n",
        "docs = text_splitter.create_documents([doc_str])\n",
        "\n",
        "token_encoder = tiktoken.get_encoding('gpt2')\n",
        "for doc in docs:\n",
        "    print(doc)\n",
        "    token_len = len(token_encoder.encode(doc.page_content))\n",
        "    print(f'length: {token_len}')\n",
        "    print('=' * 66)"
      ],
      "metadata": {
        "id": "-CQEvajw06Cm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "from langchain.text_splitter import TextSplitter\n",
        "\n",
        "\n",
        "class SimpleSentenceTextSplitter(TextSplitter):\n",
        "    def __init__(self, separators=None, **kwargs):\n",
        "        super().__init__(**kwargs)\n",
        "        self.separators = separators or [\n",
        "            ',', '.', '?', '!', ';',\n",
        "            '，', '。', '？', '！', '；'\n",
        "        ]\n",
        "\n",
        "    def split_text(self, text):\n",
        "        pattern = '|'.join(map(re.escape, self.separators))\n",
        "        return [t.strip() for t in re.split(pattern, text)\n",
        "                if t.strip()]\n",
        "\n",
        "\n",
        "doc_str = \"\"\"有一个兕怪，把唐师父拿于洞里，是老孙寻上门与他交战一场，\n",
        "那厮的神通广大，把老孙的金箍棒抢去了，因此难缚魔王。\"\"\"\n",
        "\n",
        "text_splitter = SimpleSentenceTextSplitter()\n",
        "\n",
        "docs = text_splitter.create_documents([doc_str])\n",
        "for doc in docs:\n",
        "    print(repr(doc))\n",
        "    print('=' * 66)"
      ],
      "metadata": {
        "id": "0pQhTPTM1D0-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 文本元数据提取"
      ],
      "metadata": {
        "id": "7vcPYcSG1C6m"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install doctran"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xTnUCxNG_AfZ",
        "outputId": "9b22b211-ab7b-47bf-9857-43acbb7dce24"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting doctran\n",
            "  Downloading doctran-0.0.11-py3-none-any.whl (10 kB)\n",
            "Requirement already satisfied: lxml<5.0.0,>=4.9.2 in /usr/local/lib/python3.10/dist-packages (from doctran) (4.9.3)\n",
            "Collecting openai<0.28.0,>=0.27.8 (from doctran)\n",
            "  Downloading openai-0.27.10-py3-none-any.whl (76 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.5/76.5 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting presidio-analyzer<3.0.0,>=2.2.33 (from doctran)\n",
            "  Downloading presidio_analyzer-2.2.33-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.0/76.0 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting presidio-anonymizer<3.0.0,>=2.2.33 (from doctran)\n",
            "  Downloading presidio_anonymizer-2.2.33-py3-none-any.whl (28 kB)\n",
            "Requirement already satisfied: pydantic<2.0.0,>=1.10.9 in /usr/local/lib/python3.10/dist-packages (from doctran) (1.10.13)\n",
            "Requirement already satisfied: spacy<4.0.0,>=3.5.4 in /usr/local/lib/python3.10/dist-packages (from doctran) (3.6.1)\n",
            "Collecting tiktoken<0.6.0,>=0.5.0 (from doctran)\n",
            "  Downloading tiktoken-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m55.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->doctran) (2.31.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->doctran) (4.66.1)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->doctran) (3.8.6)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from presidio-analyzer<3.0.0,>=2.2.33->doctran) (2023.6.3)\n",
            "Collecting tldextract (from presidio-analyzer<3.0.0,>=2.2.33->doctran)\n",
            "  Downloading tldextract-5.0.1-py3-none-any.whl (97 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m97.1/97.1 kB\u001b[0m \u001b[31m11.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from presidio-analyzer<3.0.0,>=2.2.33->doctran) (6.0.1)\n",
            "Collecting phonenumbers>=8.12 (from presidio-analyzer<3.0.0,>=2.2.33->doctran)\n",
            "  Downloading phonenumbers-8.13.23-py2.py3-none-any.whl (2.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.6/2.6 MB\u001b[0m \u001b[31m82.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pycryptodome>=3.10.1 (from presidio-anonymizer<3.0.0,>=2.2.33->doctran)\n",
            "  Downloading pycryptodome-3.19.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m55.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<2.0.0,>=1.10.9->doctran) (4.5.0)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (8.1.12)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (1.1.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (2.4.8)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (2.0.10)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (0.9.0)\n",
            "Requirement already satisfied: pathy>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (0.10.2)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (6.4.0)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (1.22.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (3.1.2)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (23.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0.0,>=3.5.4->doctran) (3.3.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->doctran) (3.3.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->doctran) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->doctran) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->doctran) (2023.7.22)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<4.0.0,>=3.5.4->doctran) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<4.0.0,>=3.5.4->doctran) (0.1.3)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.3.0->spacy<4.0.0,>=3.5.4->doctran) (8.1.7)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->doctran) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->doctran) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->doctran) (4.0.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->doctran) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->doctran) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->doctran) (1.3.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy<4.0.0,>=3.5.4->doctran) (2.1.3)\n",
            "Collecting requests-file>=1.4 (from tldextract->presidio-analyzer<3.0.0,>=2.2.33->doctran)\n",
            "  Downloading requests_file-1.5.1-py2.py3-none-any.whl (3.7 kB)\n",
            "Requirement already satisfied: filelock>=3.0.8 in /usr/local/lib/python3.10/dist-packages (from tldextract->presidio-analyzer<3.0.0,>=2.2.33->doctran) (3.12.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from requests-file>=1.4->tldextract->presidio-analyzer<3.0.0,>=2.2.33->doctran) (1.16.0)\n",
            "Installing collected packages: phonenumbers, pycryptodome, tiktoken, requests-file, presidio-anonymizer, tldextract, openai, presidio-analyzer, doctran\n",
            "  Attempting uninstall: openai\n",
            "    Found existing installation: openai 0.28.1\n",
            "    Uninstalling openai-0.28.1:\n",
            "      Successfully uninstalled openai-0.28.1\n",
            "Successfully installed doctran-0.0.11 openai-0.27.10 phonenumbers-8.13.23 presidio-analyzer-2.2.33 presidio-anonymizer-2.2.33 pycryptodome-3.19.0 requests-file-1.5.1 tiktoken-0.5.1 tldextract-5.0.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ['OPENAI_API_BASE'] = ''\n",
        "os.environ['OPENAI_API_KEY'] = ''"
      ],
      "metadata": {
        "id": "MVXyIS4v_qnJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "import asyncio\n",
        "\n",
        "from langchain.schema import Document\n",
        "from langchain.document_transformers import DoctranPropertyExtractor\n",
        "\n",
        "doc_str = '《在细雨中呼喊》是作家余华的第一部长篇小说，首发于《收获》1991年第6期。'\n",
        "\n",
        "documents = [Document(page_content=doc_str)]\n",
        "\n",
        "properties = [\n",
        "    {\n",
        "        'name': '名称',\n",
        "        'description': '作品名称',\n",
        "        'type': 'string',\n",
        "        'required': True\n",
        "    },\n",
        "    {\n",
        "        'name': '作者',\n",
        "        'description': '作者名称',\n",
        "        'type': 'string',\n",
        "        'required': True\n",
        "    },\n",
        "    {\n",
        "        'name': '期刊名称',\n",
        "        'description': '发表到的期刊名称',\n",
        "        'type': 'string',\n",
        "        'required': True\n",
        "    },\n",
        "]\n",
        "async def extract_metadata():\n",
        "    property_extractor = DoctranPropertyExtractor(\n",
        "        openai_api_model='gpt-3.5-turbo',\n",
        "        properties=properties,\n",
        "    )\n",
        "    extracted_document = await property_extractor.atransform_documents(\n",
        "        documents\n",
        "    )\n",
        "    print(json.dumps(\n",
        "        extracted_document[0].metadata,\n",
        "        indent=2,\n",
        "        ensure_ascii=False,\n",
        "    ))\n",
        "\n",
        "\n",
        "asyncio.run(extract_metadata())"
      ],
      "metadata": {
        "id": "mOdoppqG_B0B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.schema import Document\n",
        "from langchain.chat_models import ChatOpenAI\n",
        "from langchain.document_transformers.openai_functions import (\n",
        "    create_metadata_tagger)\n",
        "\n",
        "properties = {\n",
        "    'properties': {\n",
        "        'name': {'type': 'string', 'description': '作品名称'},\n",
        "        'author': {'type': 'string', 'description': '作者姓名'},\n",
        "        'journal': {'type': 'string', 'description': '发表到的期刊名称'},\n",
        "    }\n",
        "}\n",
        "\n",
        "llm = ChatOpenAI(temperature=0)\n",
        "\n",
        "document_transformer = create_metadata_tagger(\n",
        "    metadata_schema=properties, llm=llm)\n",
        "\n",
        "doc_str = '《在细雨中呼喊》是作家余华的第一部长篇小说，首发于《收获》1991年第6期。'\n",
        "documents = [Document(page_content=doc_str)]\n",
        "\n",
        "enhanced_documents = document_transformer.transform_documents(\n",
        "    documents)\n",
        "\n",
        "print(json.dumps(\n",
        "    enhanced_documents[0].metadata,\n",
        "    indent=2,\n",
        "    ensure_ascii=False,\n",
        "))"
      ],
      "metadata": {
        "id": "se3FB_3t1ycu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.schema import Document\n",
        "from langchain.document_transformers import DoctranTextTranslator\n",
        "\n",
        "doc_str = '《在细雨中呼喊》是作家余华的第一部长篇小说，首发于《收获》1991年第6期。'\n",
        "documents = [Document(page_content=doc_str)]\n",
        "\n",
        "async def translate_documents():\n",
        "    translator = DoctranTextTranslator(\n",
        "        openai_api_model='gpt-3.5-turbo',\n",
        "        language='english'\n",
        "    )\n",
        "    translated_documents = await translator.atransform_documents(\n",
        "        documents)\n",
        "    print(translated_documents[0].page_content)\n",
        "\n",
        "asyncio.run(translate_documents())"
      ],
      "metadata": {
        "id": "KA5TcXkA19KN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.schema import Document\n",
        "from langchain.document_transformers import DoctranQATransformer\n",
        "\n",
        "doc_str = '《在细雨中呼喊》是作家余华的第一部长篇小说，首发于《收获》1991年第6期。'\n",
        "documents = [Document(page_content=doc_str)]\n",
        "\n",
        "\n",
        "async def transform_documents():\n",
        "    transformer = DoctranQATransformer(\n",
        "        openai_api_model='gpt-3.5-turbo'\n",
        "    )\n",
        "    transformed_documents = await transformer.atransform_documents(\n",
        "        documents)\n",
        "    print(json.dumps(\n",
        "        transformed_documents[0].metadata,\n",
        "        indent=2,\n",
        "        ensure_ascii=False,\n",
        "    ))\n",
        "\n",
        "asyncio.run(transform_documents())"
      ],
      "metadata": {
        "id": "QYdF5kOl2CIV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Embedding与Vector Store"
      ],
      "metadata": {
        "id": "FDQccDlk2BwV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Embedding"
      ],
      "metadata": {
        "id": "bLCy1r8Q2cwM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "\n",
        "embeddings_model = OpenAIEmbeddings()\n",
        "\n",
        "# 将文本转换成向量\n",
        "embeddings = embeddings_model.embed_documents(\n",
        "    [\n",
        "        'Hi there!',\n",
        "        'Oh, hello!',\n",
        "        'What is your name?',\n",
        "        'My friends call me World',\n",
        "        'Hello World!'\n",
        "    ]\n",
        ")\n",
        "\n",
        "# 检索相似向量\n",
        "embedded_query = embeddings_model.embed_query(\n",
        "    'What was the name mentioned in the conversation?')\n"
      ],
      "metadata": {
        "id": "JvQmvz6r2duj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import time\n",
        "from langchain.storage import InMemoryStore, LocalFileStore\n",
        "from langchain.embeddings import (\n",
        "    OpenAIEmbeddings,\n",
        "    CacheBackedEmbeddings\n",
        ")\n",
        "\n",
        "\n",
        "def calculate_embed_time(embedder):\n",
        "    start = time.time()\n",
        "    embedder.embed_documents(['hello', 'goodbye'])\n",
        "    return time.time() - start\n",
        "\n",
        "\n",
        "underlying_embeddings = OpenAIEmbeddings()\n",
        "\n",
        "# 使用内存缓存\n",
        "store = InMemoryStore()\n",
        "embedder = CacheBackedEmbeddings.from_bytes_store(\n",
        "    underlying_embeddings, store,\n",
        "    namespace=underlying_embeddings.model\n",
        ")\n",
        "print(f'memory - no cache: {calculate_embed_time(embedder)}')\n",
        "print(f'memory - cached: {calculate_embed_time(embedder)}')\n",
        "\n",
        "\n",
        "# 使用文件缓存\n",
        "store = LocalFileStore('./test_cache/')\n",
        "embedder = CacheBackedEmbeddings.from_bytes_store(\n",
        "    underlying_embeddings, store,\n",
        "    namespace=underlying_embeddings.model\n",
        ")\n",
        "print(f'memory - no cache: {calculate_embed_time(embedder)}')\n",
        "print(f'memory - cached: {calculate_embed_time(embedder)}')"
      ],
      "metadata": {
        "id": "DHVH2Hgq2mC1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 本地向量存储"
      ],
      "metadata": {
        "id": "Fi1gnp6d2rgk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install chroma"
      ],
      "metadata": {
        "id": "7i4HzNmpGQn1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5074d1f0-cde9-4dae-e4ae-956f04418828"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting chroma\n",
            "  Downloading Chroma-0.2.0.tar.gz (5.8 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Building wheels for collected packages: chroma\n",
            "  Building wheel for chroma (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for chroma: filename=Chroma-0.2.0-py3-none-any.whl size=7097 sha256=913fc6f0ebd5e9aacd484de5d00f5a57a3fd419be99c99aac6ad7dec3fc91010\n",
            "  Stored in directory: /root/.cache/pip/wheels/58/74/75/a6ab7999ae473ecbe819bc5cae9ccb902429dd6c60795f5112\n",
            "Successfully built chroma\n",
            "Installing collected packages: chroma\n",
            "Successfully installed chroma-0.2.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.vectorstores import Chroma\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "\n",
        "texts = [\n",
        "    '在这里！',\n",
        "    '你好啊！',\n",
        "    '你叫什么名字？',\n",
        "    '我的朋友们都称呼我为小明',\n",
        "    '哦，那太棒了！'\n",
        "]\n",
        "\n",
        "# 存储数据\n",
        "db = Chroma.from_texts(\n",
        "    texts, OpenAIEmbeddings(),\n",
        "    persist_directory='./chroma_test/'\n",
        ")\n",
        "\n",
        "# 相关性搜索\n",
        "query = '谈话中和名字相关的有哪些？'\n",
        "docs = db.similarity_search(query, k=2)\n",
        "for doc in docs:\n",
        "    print(doc.page_content)\n"
      ],
      "metadata": {
        "id": "c7sj6LvlGTn9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import uuid\n",
        "import chromadb\n",
        "from chromadb.config import Settings\n",
        "from langchain.vectorstores import Chroma\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "\n",
        "client = chromadb.HttpClient(\n",
        "    host='127.0.0.1',\n",
        "    port=8000,\n",
        "    settings=Settings(allow_reset=True)\n",
        ")\n",
        "client.reset()\n",
        "collection = client.create_collection('my_collection')\n",
        "\n",
        "db = Chroma(\n",
        "    client=client,\n",
        "    collection_name='my_collection',\n",
        "    embedding_function=OpenAIEmbeddings()\n",
        ")\n",
        "\n",
        "texts = [\n",
        "    '在这里！',\n",
        "    '你好啊！',\n",
        "    '你叫什么名字？',\n",
        "    '我的朋友们都称呼我为小明',\n",
        "    '哦，那太棒了！'\n",
        "]\n",
        "\n",
        "# 存储数据\n",
        "db = Chroma.add_texts(\n",
        "    texts=texts,\n",
        "  \tids=[str(uuid.uuid1()) for _ in len(texts)]\n",
        ")\n",
        "\n",
        "query = '谈话中和名字相关的有哪些？'\n",
        "docs = db4.similarity_search(query)\n",
        "print(docs[0].page_content)"
      ],
      "metadata": {
        "id": "pd3Cfui_2y7j"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install faiss-cpu tiktoken"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-zPFJFOQGXZ1",
        "outputId": "bdc2ae5f-bf92-478a-fbde-f33ea789c0e9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: faiss-cpu in /usr/local/lib/python3.10/dist-packages (1.7.4)\n",
            "Collecting tiktoken\n",
            "  Downloading tiktoken-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m19.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2023.6.3)\n",
            "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2.31.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.3.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2023.7.22)\n",
            "Installing collected packages: tiktoken\n",
            "Successfully installed tiktoken-0.5.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.vectorstores import FAISS\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "\n",
        "texts = [\n",
        "    '在这里！',\n",
        "    '你好啊！',\n",
        "    '你叫什么名字？',\n",
        "    '我的朋友们都称呼我为小明',\n",
        "    '哦，那太棒了！'\n",
        "]\n",
        "\n",
        "# 存储数据\n",
        "db = FAISS.from_texts(\n",
        "    texts, OpenAIEmbeddings()\n",
        ")\n",
        "\n",
        "# 相关性搜索\n",
        "query = '谈话中和名字相关的有哪些？'\n",
        "embedding_vector = OpenAIEmbeddings().embed_query(query)\n",
        "docs = db.similarity_search_by_vector(\n",
        "    embedding_vector, k=2)\n",
        "for doc in docs:\n",
        "    print(doc.page_content)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gmn4XqX8GeG0",
        "outputId": "f7cebb09-1c9d-4b98-ba8c-6cc026840012"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "你叫什么名字？\n",
            "我的朋友们都称呼我为小明\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 云端向量存储"
      ],
      "metadata": {
        "id": "R7HvcjDu2963"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pinecone-client tiktoken"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kOJfNsDxOv6V",
        "outputId": "8d260a2d-44b7-4085-c2bd-7ac1adb2034b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pinecone-client in /usr/local/lib/python3.10/dist-packages (2.2.4)\n",
            "Collecting tiktoken\n",
            "  Downloading tiktoken-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from pinecone-client) (2.31.0)\n",
            "Requirement already satisfied: pyyaml>=5.4 in /usr/local/lib/python3.10/dist-packages (from pinecone-client) (6.0.1)\n",
            "Requirement already satisfied: loguru>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from pinecone-client) (0.7.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.10/dist-packages (from pinecone-client) (4.5.0)\n",
            "Requirement already satisfied: dnspython>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from pinecone-client) (2.4.2)\n",
            "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.10/dist-packages (from pinecone-client) (2.8.2)\n",
            "Requirement already satisfied: urllib3>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from pinecone-client) (2.0.7)\n",
            "Requirement already satisfied: tqdm>=4.64.1 in /usr/local/lib/python3.10/dist-packages (from pinecone-client) (4.66.1)\n",
            "Requirement already satisfied: numpy>=1.22.0 in /usr/local/lib/python3.10/dist-packages (from pinecone-client) (1.22.0)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2023.6.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.5.3->pinecone-client) (1.16.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pinecone-client) (3.3.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pinecone-client) (3.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pinecone-client) (2023.7.22)\n",
            "Installing collected packages: tiktoken\n",
            "Successfully installed tiktoken-0.5.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pinecone\n",
        "from langchain.vectorstores import Pinecone\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "\n",
        "\n",
        "pinecone.init(\n",
        "    api_key='9d3d2e65-c501-4e03-a2d2-6e55b0334638',\n",
        "    environment='us-west4-gcp',\n",
        ")\n",
        "\n",
        "texts = [\n",
        "    '在这里！',\n",
        "    '你好啊！',\n",
        "    '你叫什么名字？',\n",
        "    '我的朋友们都称呼我为小明',\n",
        "    '哦，那太棒了！'\n",
        "]\n",
        "\n",
        "embedding = OpenAIEmbeddings()\n",
        "docsearch = Pinecone.from_texts(\n",
        "    texts,\n",
        "    index_name='my-index',\n",
        "    embedding=embedding\n",
        ")\n"
      ],
      "metadata": {
        "id": "k2UBPzA8OsIy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pinecone\n",
        "from langchain.vectorstores import Pinecone\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "\n",
        "\n",
        "pinecone.init(\n",
        "    api_key='YOU_API_KEY',\n",
        "    environment='YOU_ENVIRONMENT',\n",
        ")\n",
        "\n",
        "docsearch = Pinecone.from_existing_index(\n",
        "    'my-index',OpenAIEmbeddings()\n",
        ")\n",
        "\n",
        "docs = docsearch.similarity_search_with_score(\n",
        "    '谈话中和名字相关的有哪些？',\n",
        "    k=2\n",
        ")\n",
        "for doc_info in docs:\n",
        "    print(repr(doc_info[0]))\n",
        "    print(f'score: {doc_info[1]}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D8bKeoCXTKOd",
        "outputId": "c3bce999-0466-4e41-b62d-f4d0661595b4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Document(page_content='你叫什么名字？')\n",
            "score:0.852765441\n",
            "Document(page_content='我的朋友们都称呼我为小明')\n",
            "score:0.796148896\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Retriever（检索器）"
      ],
      "metadata": {
        "id": "4DTex-s63OV8"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 基础检索器"
      ],
      "metadata": {
        "id": "iPMjzWz13SYc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.vectorstores import Chroma\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "\n",
        "texts = [\n",
        "    '在这里！',\n",
        "    '你好啊！',\n",
        "    '你叫什么名字？',\n",
        "    '我的朋友们都称呼我为小明',\n",
        "    '哦，那太棒了！'\n",
        "]\n",
        "\n",
        "# 存储数据\n",
        "db = Chroma.from_texts(\n",
        "    texts, OpenAIEmbeddings(),\n",
        "    persist_directory='./chroma_test/'\n",
        ")\n",
        "\n",
        "retriever = db.as_retriever(\n",
        "    search_kwargs={'k': 2}\n",
        ")\n",
        "\n",
        "# 相关性搜索\n",
        "query = '谈话中和名字相关的有哪些？'\n",
        "docs = retriever.invoke(query)\n",
        "for doc in docs:\n",
        "    print(doc.page_content)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C7mou3PlcIeV",
        "outputId": "432a83bc-8c2c-4c5b-bedb-f93a37f93b19"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "你叫什么名字？\n",
            "我的朋友们都称呼我为小明\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 多重提问检索器"
      ],
      "metadata": {
        "id": "s9heXrfJ3ZJb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install chromadb tiktoken"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JCJuE1xWa6kt",
        "outputId": "a52f0168-a9d9-4214-d3b2-883dc840e984"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: chromadb in /usr/local/lib/python3.10/dist-packages (0.4.14)\n",
            "Collecting tiktoken\n",
            "  Downloading tiktoken-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m21.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.28 in /usr/local/lib/python3.10/dist-packages (from chromadb) (2.31.0)\n",
            "Requirement already satisfied: pydantic>=1.9 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.10.13)\n",
            "Requirement already satisfied: chroma-hnswlib==0.7.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.7.3)\n",
            "Requirement already satisfied: fastapi>=0.95.2 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.104.0)\n",
            "Requirement already satisfied: uvicorn[standard]>=0.18.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.23.2)\n",
            "Requirement already satisfied: posthog>=2.4.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (3.0.2)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.8.0)\n",
            "Requirement already satisfied: pulsar-client>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (3.3.0)\n",
            "Requirement already satisfied: onnxruntime>=1.14.1 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.16.1)\n",
            "Requirement already satisfied: tokenizers>=0.13.2 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.14.1)\n",
            "Requirement already satisfied: pypika>=0.48.9 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.48.9)\n",
            "Requirement already satisfied: tqdm>=4.65.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.66.1)\n",
            "Requirement already satisfied: overrides>=7.3.1 in /usr/local/lib/python3.10/dist-packages (from chromadb) (7.4.0)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.10/dist-packages (from chromadb) (6.1.0)\n",
            "Requirement already satisfied: grpcio>=1.58.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.59.0)\n",
            "Requirement already satisfied: bcrypt>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.0.1)\n",
            "Requirement already satisfied: typer>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.9.0)\n",
            "Requirement already satisfied: numpy>=1.22.5 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.26.1)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2023.6.3)\n",
            "Requirement already satisfied: anyio<4.0.0,>=3.7.1 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb) (3.7.1)\n",
            "Requirement already satisfied: starlette<0.28.0,>=0.27.0 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb) (0.27.0)\n",
            "Requirement already satisfied: coloredlogs in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (15.0.1)\n",
            "Requirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (23.5.26)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (23.2)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (3.20.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (1.12)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb) (1.16.0)\n",
            "Requirement already satisfied: monotonic>=1.5 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb) (1.6)\n",
            "Requirement already satisfied: backoff>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb) (2.2.1)\n",
            "Requirement already satisfied: python-dateutil>2.1 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb) (2.8.2)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from pulsar-client>=3.1.0->chromadb) (2023.7.22)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.3.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (2.0.7)\n",
            "Requirement already satisfied: huggingface_hub<0.18,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers>=0.13.2->chromadb) (0.17.3)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb) (8.1.7)\n",
            "Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.14.0)\n",
            "Requirement already satisfied: httptools>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.6.1)\n",
            "Requirement already satisfied: python-dotenv>=0.13 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.0.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (6.0.1)\n",
            "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.19.0)\n",
            "Requirement already satisfied: watchfiles>=0.13 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.21.0)\n",
            "Requirement already satisfied: websockets>=10.4 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (12.0)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<4.0.0,>=3.7.1->fastapi>=0.95.2->chromadb) (1.3.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4.0.0,>=3.7.1->fastapi>=0.95.2->chromadb) (1.1.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<0.18,>=0.16.4->tokenizers>=0.13.2->chromadb) (3.12.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<0.18,>=0.16.4->tokenizers>=0.13.2->chromadb) (2023.6.0)\n",
            "Requirement already satisfied: humanfriendly>=9.1 in /usr/local/lib/python3.10/dist-packages (from coloredlogs->onnxruntime>=1.14.1->chromadb) (10.0)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n",
            "Installing collected packages: tiktoken\n",
            "Successfully installed tiktoken-0.5.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.llms import OpenAI\n",
        "from langchain.vectorstores import Chroma\n",
        "from langchain.document_loaders import WebBaseLoader\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain.retrievers import MultiQueryRetriever\n",
        "\n",
        "# 加载网页数据\n",
        "loader = WebBaseLoader(\n",
        "    'https://blog.langchain.dev/introducing-langserve/'\n",
        ")\n",
        "data = loader.load()\n",
        "\n",
        "# 分割数据\n",
        "splitter = RecursiveCharacterTextSplitter(\n",
        "    chunk_size=500, chunk_overlap=0)\n",
        "splits = splitter.split_documents(data)\n",
        "\n",
        "# 存入向量数据\n",
        "vectordb = Chroma.from_documents(\n",
        "    documents=splits,\n",
        "    embedding=OpenAIEmbeddings()\n",
        ")\n",
        "\n",
        "query = 'Why use LangServe?'\n",
        "llm = OpenAI(temperature=0)\n",
        "retriever_from_llm = MultiQueryRetriever.from_llm(\n",
        "    retriever=vectordb.as_retriever(), llm=llm\n",
        ")\n",
        "\n",
        "import logging\n",
        "\n",
        "logging.basicConfig()\n",
        "logging.getLogger(\"langchain.retrievers.multi_query\").setLevel(logging.INFO)\n",
        "\n",
        "unique_docs = retriever_from_llm.get_relevant_documents(\n",
        "    query=query\n",
        ")\n",
        "print(len(unique_docs))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A3aDmZ6JZw6O",
        "outputId": "9cf11610-0d4d-4767-a452-e4b526c41197"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:langchain.retrievers.multi_query:Generated queries: ['What are the benefits of using LangServe?', 'What is the purpose of LangServe?', 'What advantages does LangServe offer?']\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import logging\n",
        "from langchain.llms import OpenAI\n",
        "from langchain.vectorstores import Chroma\n",
        "from langchain.prompts import PromptTemplate\n",
        "from langchain.document_loaders import WebBaseLoader\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "\n",
        "from langchain.retrievers import MultiQueryRetriever\n",
        "\n",
        "# 加载网页数据\n",
        "loader = WebBaseLoader(\n",
        "    'https://blog.langchain.dev/introducing-langserve/'\n",
        ")\n",
        "data = loader.load()\n",
        "\n",
        "# 分割数据\n",
        "splitter = RecursiveCharacterTextSplitter(\n",
        "    chunk_size=500, chunk_overlap=0)\n",
        "splits = splitter.split_documents(data)\n",
        "\n",
        "# 存入向量数据\n",
        "vectordb = Chroma.from_documents(\n",
        "    documents=splits,\n",
        "    embedding=OpenAIEmbeddings()\n",
        ")\n",
        "\n",
        "prompt_tpl = PromptTemplate(\n",
        "    input_variables=['question'],\n",
        "    template=\"\"\"You are an AI language model assistant. Your task is\n",
        "    to generate 6 different versions of the given user\n",
        "    question to retrieve relevant documents from a vector  database.\n",
        "    By generating multiple perspectives on the user question,\n",
        "    your goal is to help the user overcome some of the limitations\n",
        "    of distance-based similarity search. Provide these alternative\n",
        "    questions separated by newlines. Original question: {question}\"\"\"\n",
        ")\n",
        "\n",
        "\n",
        "# 初始化 MultiQueryRetriever\n",
        "llm = OpenAI(temperature=0)\n",
        "retriever_from_llm = MultiQueryRetriever.from_llm(\n",
        "    retriever=vectordb.as_retriever(), llm=llm,\n",
        "    prompt=prompt_tpl\n",
        ")\n",
        "\n",
        "# 设置 langchain.retrievers.multi_query 日志级别\n",
        "logging.basicConfig()\n",
        "logging.getLogger(\n",
        "    'langchain.retrievers.multi_query'\n",
        ").setLevel(logging.INFO)\n",
        "\n",
        "# 相似性搜索\n",
        "docs = retriever_from_llm.get_relevant_documents(\n",
        "    query='Why use LangServe?'\n",
        ")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1G4aRxHBmYWq",
        "outputId": "c3f67fc3-5e63-4410-9ccb-6475d68a9426"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:langchain.retrievers.multi_query:Generated queries: ['1. What are the benefits of using LangServe?', '2. What makes LangServe a better choice than other language models?', '3. What advantages does LangServe offer?', '4. How does LangServe improve language processing?', '5. What features does LangServe provide?', '6. What makes LangServe a unique language model?']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 上下文压缩检索器"
      ],
      "metadata": {
        "id": "a8Ca49z83zjz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install chromadb tiktoken"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "t-R85hNU3VDf",
        "outputId": "60859fbf-786f-4388-8682-028e7e92de7e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting chromadb\n",
            "  Downloading chromadb-0.4.15-py3-none-any.whl (479 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m479.8/479.8 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tiktoken\n",
            "  Downloading tiktoken-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m46.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.28 in /usr/local/lib/python3.10/dist-packages (from chromadb) (2.31.0)\n",
            "Requirement already satisfied: pydantic>=1.9 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.10.13)\n",
            "Collecting chroma-hnswlib==0.7.3 (from chromadb)\n",
            "  Downloading chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m82.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting fastapi>=0.95.2 (from chromadb)\n",
            "  Downloading fastapi-0.104.0-py3-none-any.whl (92 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.9/92.9 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting uvicorn[standard]>=0.18.3 (from chromadb)\n",
            "  Downloading uvicorn-0.23.2-py3-none-any.whl (59 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.5/59.5 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting posthog>=2.4.0 (from chromadb)\n",
            "  Downloading posthog-3.0.2-py2.py3-none-any.whl (37 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.5.0)\n",
            "Collecting pulsar-client>=3.1.0 (from chromadb)\n",
            "  Downloading pulsar_client-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m64.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting onnxruntime>=1.14.1 (from chromadb)\n",
            "  Downloading onnxruntime-1.16.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.2/6.2 MB\u001b[0m \u001b[31m70.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-api>=1.2.0 (from chromadb)\n",
            "  Downloading opentelemetry_api-1.20.0-py3-none-any.whl (57 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.5/57.5 kB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-exporter-otlp-proto-grpc>=1.2.0 (from chromadb)\n",
            "  Downloading opentelemetry_exporter_otlp_proto_grpc-1.20.0-py3-none-any.whl (18 kB)\n",
            "Collecting opentelemetry-sdk>=1.2.0 (from chromadb)\n",
            "  Downloading opentelemetry_sdk-1.20.0-py3-none-any.whl (103 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m103.9/103.9 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tokenizers>=0.13.2 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.14.1)\n",
            "Collecting pypika>=0.48.9 (from chromadb)\n",
            "  Downloading PyPika-0.48.9.tar.gz (67 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tqdm>=4.65.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.66.1)\n",
            "Collecting overrides>=7.3.1 (from chromadb)\n",
            "  Downloading overrides-7.4.0-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.10/dist-packages (from chromadb) (6.1.0)\n",
            "Requirement already satisfied: grpcio>=1.58.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.59.0)\n",
            "Collecting bcrypt>=4.0.1 (from chromadb)\n",
            "  Downloading bcrypt-4.0.1-cp36-abi3-manylinux_2_28_x86_64.whl (593 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m593.7/593.7 kB\u001b[0m \u001b[31m36.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typer>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.9.0)\n",
            "Collecting kubernetes>=28.1.0 (from chromadb)\n",
            "  Downloading kubernetes-28.1.0-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m60.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tenacity>=8.2.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (8.2.3)\n",
            "Collecting numpy>=1.22.5 (from chromadb)\n",
            "  Downloading numpy-1.26.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.2/18.2 MB\u001b[0m \u001b[31m54.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2023.6.3)\n",
            "Requirement already satisfied: anyio<4.0.0,>=3.7.1 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb) (3.7.1)\n",
            "Collecting starlette<0.28.0,>=0.27.0 (from fastapi>=0.95.2->chromadb)\n",
            "  Downloading starlette-0.27.0-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.0/67.0 kB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-extensions>=4.5.0 (from chromadb)\n",
            "  Downloading typing_extensions-4.8.0-py3-none-any.whl (31 kB)\n",
            "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2023.7.22)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.16.0)\n",
            "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.8.2)\n",
            "Requirement already satisfied: pyyaml>=5.4.1 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (6.0.1)\n",
            "Requirement already satisfied: google-auth>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.17.3)\n",
            "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.6.4)\n",
            "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.3.1)\n",
            "Requirement already satisfied: oauthlib>=3.2.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\n",
            "Collecting urllib3<2.0,>=1.24.2 (from kubernetes>=28.1.0->chromadb)\n",
            "  Downloading urllib3-1.26.18-py2.py3-none-any.whl (143 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.8/143.8 kB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting coloredlogs (from onnxruntime>=1.14.1->chromadb)\n",
            "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (23.5.26)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (23.2)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (3.20.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (1.12)\n",
            "Collecting deprecated>=1.2.6 (from opentelemetry-api>=1.2.0->chromadb)\n",
            "  Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n",
            "Requirement already satisfied: importlib-metadata<7.0,>=6.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-api>=1.2.0->chromadb) (6.8.0)\n",
            "Requirement already satisfied: backoff<3.0.0,>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (2.2.1)\n",
            "Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.61.0)\n",
            "Collecting opentelemetry-exporter-otlp-proto-common==1.20.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb)\n",
            "  Downloading opentelemetry_exporter_otlp_proto_common-1.20.0-py3-none-any.whl (17 kB)\n",
            "Collecting opentelemetry-proto==1.20.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb)\n",
            "  Downloading opentelemetry_proto-1.20.0-py3-none-any.whl (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.8/50.8 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-semantic-conventions==0.41b0 (from opentelemetry-sdk>=1.2.0->chromadb)\n",
            "  Downloading opentelemetry_semantic_conventions-0.41b0-py3-none-any.whl (26 kB)\n",
            "Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb)\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.3.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.4)\n",
            "Requirement already satisfied: huggingface_hub<0.18,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers>=0.13.2->chromadb) (0.17.3)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb) (8.1.7)\n",
            "Collecting h11>=0.8 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httptools>=0.5.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (341 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m35.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting python-dotenv>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n",
            "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m92.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting watchfiles>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading watchfiles-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m71.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websockets>=10.4 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (130 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<4.0.0,>=3.7.1->fastapi>=0.95.2->chromadb) (1.3.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4.0.0,>=3.7.1->fastapi>=0.95.2->chromadb) (1.1.3)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated>=1.2.6->opentelemetry-api>=1.2.0->chromadb) (1.14.1)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (5.3.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<0.18,>=0.16.4->tokenizers>=0.13.2->chromadb) (3.12.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<0.18,>=0.16.4->tokenizers>=0.13.2->chromadb) (2023.6.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.17.0)\n",
            "Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime>=1.14.1->chromadb)\n",
            "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.5.0)\n",
            "Building wheels for collected packages: pypika\n",
            "  Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pypika: filename=PyPika-0.48.9-py2.py3-none-any.whl size=53723 sha256=3030b66196945d34c5df8722c956451bcab82377ea7412c99245bbbd1cc53185\n",
            "  Stored in directory: /root/.cache/pip/wheels/e1/26/51/d0bffb3d2fd82256676d7ad3003faea3bd6dddc9577af665f4\n",
            "Successfully built pypika\n",
            "Installing collected packages: pypika, monotonic, websockets, uvloop, urllib3, typing-extensions, python-dotenv, pulsar-client, overrides, opentelemetry-semantic-conventions, opentelemetry-proto, numpy, humanfriendly, httptools, h11, deprecated, bcrypt, watchfiles, uvicorn, starlette, opentelemetry-exporter-otlp-proto-common, opentelemetry-api, coloredlogs, chroma-hnswlib, tiktoken, posthog, opentelemetry-sdk, onnxruntime, fastapi, opentelemetry-exporter-otlp-proto-grpc, kubernetes, chromadb\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 2.0.7\n",
            "    Uninstalling urllib3-2.0.7:\n",
            "      Successfully uninstalled urllib3-2.0.7\n",
            "  Attempting uninstall: typing-extensions\n",
            "    Found existing installation: typing_extensions 4.5.0\n",
            "    Uninstalling typing_extensions-4.5.0:\n",
            "      Successfully uninstalled typing_extensions-4.5.0\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 1.22.0\n",
            "    Uninstalling numpy-1.22.0:\n",
            "      Successfully uninstalled numpy-1.22.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "lida 0.0.10 requires kaleido, which is not installed.\n",
            "lida 0.0.10 requires python-multipart, which is not installed.\n",
            "bigframes 0.10.0 requires pandas>=1.5.0, but you have pandas 1.3.5 which is incompatible.\n",
            "clarifai 9.9.3 requires numpy==1.22.0, but you have numpy 1.26.1 which is incompatible.\n",
            "cupy-cuda11x 11.0.0 requires numpy<1.26,>=1.20, but you have numpy 1.26.1 which is incompatible.\n",
            "numba 0.56.4 requires numpy<1.24,>=1.18, but you have numpy 1.26.1 which is incompatible.\n",
            "plotnine 0.12.3 requires pandas>=1.5.0, but you have pandas 1.3.5 which is incompatible.\n",
            "tensorflow-probability 0.22.0 requires typing-extensions<4.6.0, but you have typing-extensions 4.8.0 which is incompatible.\n",
            "xarray 2023.7.0 requires pandas>=1.4, but you have pandas 1.3.5 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed bcrypt-4.0.1 chroma-hnswlib-0.7.3 chromadb-0.4.15 coloredlogs-15.0.1 deprecated-1.2.14 fastapi-0.104.0 h11-0.14.0 httptools-0.6.1 humanfriendly-10.0 kubernetes-28.1.0 monotonic-1.6 numpy-1.26.1 onnxruntime-1.16.1 opentelemetry-api-1.20.0 opentelemetry-exporter-otlp-proto-common-1.20.0 opentelemetry-exporter-otlp-proto-grpc-1.20.0 opentelemetry-proto-1.20.0 opentelemetry-sdk-1.20.0 opentelemetry-semantic-conventions-0.41b0 overrides-7.4.0 posthog-3.0.2 pulsar-client-3.3.0 pypika-0.48.9 python-dotenv-1.0.0 starlette-0.27.0 tiktoken-0.5.1 typing-extensions-4.8.0 urllib3-1.26.18 uvicorn-0.23.2 uvloop-0.19.0 watchfiles-0.21.0 websockets-12.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "urllib3"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.vectorstores import Chroma\n",
        "from langchain.document_loaders import WebBaseLoader\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "\n",
        "# 加载网页数据\n",
        "loader = WebBaseLoader(\n",
        "    'https://zh.wikipedia.org/wiki/%E6%A1%82%E8%8A%B1'\n",
        ")\n",
        "data = loader.load()\n",
        "\n",
        "# 分割数据\n",
        "splitter = RecursiveCharacterTextSplitter(\n",
        "    chunk_size=300, chunk_overlap=0)\n",
        "splits = splitter.split_documents(data)\n",
        "\n",
        "# 存入向量数据\n",
        "vectordb = Chroma.from_documents(\n",
        "    documents=splits,\n",
        "    embedding=OpenAIEmbeddings()\n",
        ")\n",
        "\n",
        "retriever = vectordb.as_retriever()\n",
        "docs = retriever.get_relevant_documents(\n",
        "    '桂花的食用'\n",
        ")\n",
        "\n",
        "\n",
        "def pretty_print_docs(docs):\n",
        "    doc_str_list = [\n",
        "        f'Document {i + 1}:\\n\\n{d.page_content}'\n",
        "        for i, d in enumerate(docs)\n",
        "    ]\n",
        "    print(f\"\\n{'-' * 100}\\n\".join(doc_str_list))\n",
        "\n",
        "\n",
        "pretty_print_docs(docs)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S8JXM18e25GY",
        "outputId": "513bb30e-9cfc-4e0d-96ba-edffd2d9cb60"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Document 1:\n",
            "\n",
            "食用[编辑]\n",
            "桂花醪糟\n",
            "桂花糕\n",
            "桂花味辛，可入药。有化痰、止咳、生津、止牙痛等功效。\n",
            "桂花味香，持久，可制糕点（桂花糕）、糖果，并可酿酒。此外，亦常制成桂花糖、桂花湯圓、桂花釀（與蜂蜜泡制而成）、桂花酱、桂花滷等。在中国南京，加工盐水鸭的过程中，加入桂花，增加香味，也称为桂花鸭。过去台湾桂花多半种植在茶园旁，作为提升茶香气陪衬，也因此易染病需施农药，不能食用居多。\n",
            "中國國家林業局有訂定林業行業標準LY/T 1910-2010食用桂花栽培技術規程，其附表有列出可用的主要食用桂花品種。[4][5]\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2:\n",
            "\n",
            "取自“https://zh.wikipedia.org/w/index.php?title=桂花&oldid=76269857”\n",
            "分类：​可食用植物木樨属1790年描述的植物隐藏分类：​CS1英语来源 (en)物种微格式条目含有拉丁語的條目\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3:\n",
            "\n",
            "桂花（学名：Osmanthus fragrans），又名梫、白桂、銀桂、銀木樨，常绿灌木或小乔木；叶子对生，多呈椭圆或长椭圆形，叶面光滑，革质，叶缘有锯齿；秋季开花，花簇生于叶腋，花冠分裂至基乳有乳白、黄、橙红等色，极芳香；核果成熟后为紫黑色。常见于南方。\n",
            "其种加词“Fragrans”意为芳香的、有着甜美气味的。\n",
            "桂花有兩種，一種就是本條目所講的白色桂花“銀桂”，為其正種；另一種是黃色桂花“丹桂”，為銀桂的亞種。在中國，白色的銀桂主要用於觀賞和提升香氣，黃色的丹桂主要用於甜點食用。另外桂花的中文名易與“肉桂、月桂”兩者混淆，在辨識時需小心。\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 4:\n",
            "\n",
            "延伸阅读[编辑]\n",
            "[在维基数据编辑]\n",
            "\n",
            " 《欽定古今圖書集成·博物彙編·草木典·桂部》，出自陈梦雷《古今圖書集成》\n",
            " 《植物名實圖考·桂》，出自吳其濬《植物名實圖考》\n",
            "外部链接[编辑]\n",
            "\n",
            "\n",
            "\n",
            "维基共享资源中相关的多媒体资源：桂花\n",
            "桂花 Guihua （页面存档备份，存于互联网档案馆） 藥用植物圖像資料庫 (香港浸會大學中醫藥學院) （中文）（英文）\n",
            "連翹苷元 Phillygenin （页面存档备份，存于互联网档案馆） 中草藥化學圖像數據庫 (香港浸會大學中醫藥學院) （中文）（英文）\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.llms import OpenAI\n",
        "from langchain.retrievers import ContextualCompressionRetriever\n",
        "from langchain.retrievers.document_compressors import LLMChainExtractor\n",
        "\n",
        "llm = OpenAI(temperature=0)\n",
        "compressor = LLMChainExtractor.from_llm(llm)\n",
        "compression_retriever = ContextualCompressionRetriever(\n",
        "    base_compressor=compressor,\n",
        "    base_retriever=retriever\n",
        ")\n",
        "\n",
        "compressed_docs = compression_retriever.get_relevant_documents(\n",
        "    '桂花的食用'\n",
        ")\n",
        "pretty_print_docs(compressed_docs)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NkxhQZS033b_",
        "outputId": "d5b47114-1a1c-46b2-c0aa-e953fd9dcbd0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/langchain/chains/llm.py:280: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/langchain/chains/llm.py:280: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/langchain/chains/llm.py:280: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/langchain/chains/llm.py:280: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Document 1:\n",
            "\n",
            "桂花醪糟、桂花糕、桂花糖、桂花湯圓、桂花釀（與蜂蜜泡制而成）、桂花酱、桂花滷等、桂花鸭、LY/T 1910-2010食用桂花栽培技術規程\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2:\n",
            "\n",
            "銀桂（Osmanthus fragrans），又名梫、白桂、銀桂、銀木樨，常绿灌木或小乔木；叶子对生，多呈椭圆或长椭圆形，叶面光滑，革质，叶缘有锯齿；秋季开花，花簇生于叶腋，花冠分裂至基乳有乳白、黄、橙红等色，极芳香；核果成熟后为紫黑色。常见于南\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.llms import OpenAI\n",
        "from langchain.retrievers import ContextualCompressionRetriever\n",
        "from langchain.retrievers.document_compressors import LLMChainFilter\n",
        "\n",
        "llm = OpenAI(temperature=0)\n",
        "_filter = LLMChainFilter.from_llm(llm)\n",
        "compression_retriever = ContextualCompressionRetriever(\n",
        "    base_compressor=_filter,\n",
        "    base_retriever=retriever\n",
        ")\n",
        "\n",
        "compressed_docs = compression_retriever.get_relevant_documents(\n",
        "    '桂花的食用'\n",
        ")\n",
        "pretty_print_docs(compressed_docs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pQfXB9g26Oxq",
        "outputId": "9141a3a8-b758-4240-9ae3-727dfc797cc4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/langchain/chains/llm.py:280: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/langchain/chains/llm.py:280: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/langchain/chains/llm.py:280: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/langchain/chains/llm.py:280: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Document 1:\n",
            "\n",
            "食用[编辑]\n",
            "桂花醪糟\n",
            "桂花糕\n",
            "桂花味辛，可入药。有化痰、止咳、生津、止牙痛等功效。\n",
            "桂花味香，持久，可制糕点（桂花糕）、糖果，并可酿酒。此外，亦常制成桂花糖、桂花湯圓、桂花釀（與蜂蜜泡制而成）、桂花酱、桂花滷等。在中国南京，加工盐水鸭的过程中，加入桂花，增加香味，也称为桂花鸭。过去台湾桂花多半种植在茶园旁，作为提升茶香气陪衬，也因此易染病需施农药，不能食用居多。\n",
            "中國國家林業局有訂定林業行業標準LY/T 1910-2010食用桂花栽培技術規程，其附表有列出可用的主要食用桂花品種。[4][5]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "from langchain.retrievers import ContextualCompressionRetriever\n",
        "from langchain.retrievers.document_compressors import EmbeddingsFilter\n",
        "\n",
        "embeddings = OpenAIEmbeddings()\n",
        "embeddings_filter = EmbeddingsFilter(\n",
        "    embeddings=embeddings,\n",
        "    similarity_threshold=0.87\n",
        ")\n",
        "compression_retriever = ContextualCompressionRetriever(\n",
        "    base_compressor=embeddings_filter,\n",
        "    base_retriever=retriever\n",
        ")\n",
        "\n",
        "compressed_docs = compression_retriever.get_relevant_documents(\n",
        "    '桂花的食用'\n",
        ")\n",
        "pretty_print_docs(compressed_docs)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1wJgrKLj-TEm",
        "outputId": "ecff5fd0-3a17-451c-90fc-20779cf274f0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Document 1:\n",
            "\n",
            "食用[编辑]\n",
            "桂花醪糟\n",
            "桂花糕\n",
            "桂花味辛，可入药。有化痰、止咳、生津、止牙痛等功效。\n",
            "桂花味香，持久，可制糕点（桂花糕）、糖果，并可酿酒。此外，亦常制成桂花糖、桂花湯圓、桂花釀（與蜂蜜泡制而成）、桂花酱、桂花滷等。在中国南京，加工盐水鸭的过程中，加入桂花，增加香味，也称为桂花鸭。过去台湾桂花多半种植在茶园旁，作为提升茶香气陪衬，也因此易染病需施农药，不能食用居多。\n",
            "中國國家林業局有訂定林業行業標準LY/T 1910-2010食用桂花栽培技術規程，其附表有列出可用的主要食用桂花品種。[4][5]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_transformers import EmbeddingsRedundantFilter\n",
        "from langchain.retrievers.document_compressors import (\n",
        "    DocumentCompressorPipeline\n",
        ")\n",
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "\n",
        "splitter = CharacterTextSplitter(\n",
        "    chunk_size=180,\n",
        "    chunk_overlap=0,\n",
        "    separator=\"。\"\n",
        ")\n",
        "redundant_filter = EmbeddingsRedundantFilter(embeddings=embeddings)\n",
        "relevant_filter = EmbeddingsFilter(\n",
        "    embeddings=embeddings,\n",
        "    similarity_threshold=0.865\n",
        ")\n",
        "pipeline_compressor = DocumentCompressorPipeline(\n",
        "    transformers=[splitter, redundant_filter, relevant_filter]\n",
        ")\n",
        "\n",
        "compression_retriever = ContextualCompressionRetriever(\n",
        "    base_compressor=pipeline_compressor,\n",
        "    base_retriever=retriever\n",
        ")\n",
        "\n",
        "compressed_docs = compression_retriever.get_relevant_documents(\n",
        "    '桂花的食用'\n",
        ")\n",
        "pretty_print_docs(compressed_docs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yofWJ4u3IV8d",
        "outputId": "5412f411-17b9-459a-9a69-c09d75f94ca8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Document 1:\n",
            "\n",
            "过去台湾桂花多半种植在茶园旁，作为提升茶香气陪衬，也因此易染病需施农药，不能食用居多。\n",
            "中國國家林業局有訂定林業行業標準LY/T 1910-2010食用桂花栽培技術規程，其附表有列出可用的主要食用桂花品種。[4][5]\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 2:\n",
            "\n",
            "食用[编辑]\n",
            "桂花醪糟\n",
            "桂花糕\n",
            "桂花味辛，可入药。有化痰、止咳、生津、止牙痛等功效。\n",
            "桂花味香，持久，可制糕点（桂花糕）、糖果，并可酿酒。此外，亦常制成桂花糖、桂花湯圓、桂花釀（與蜂蜜泡制而成）、桂花酱、桂花滷等。在中国南京，加工盐水鸭的过程中，加入桂花，增加香味，也称为桂花鸭\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Document 3:\n",
            "\n",
            "桂花有兩種，一種就是本條目所講的白色桂花“銀桂”，為其正種；另一種是黃色桂花“丹桂”，為銀桂的亞種。在中國，白色的銀桂主要用於觀賞和提升香氣，黃色的丹桂主要用於甜點食用。另外桂花的中文名易與“肉桂、月桂”兩者混淆，在辨識時需小心\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 集成检索器"
      ],
      "metadata": {
        "id": "K9N9_P_t4GtK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install chromadb tiktoken rank_bm25"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rXzJxyt7oM4M",
        "outputId": "1ba3ac5e-c091-4a20-cd87-d9e291216f27"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting chromadb\n",
            "  Downloading chromadb-0.4.15-py3-none-any.whl (479 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m479.8/479.8 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tiktoken\n",
            "  Downloading tiktoken-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m44.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting rank_bm25\n",
            "  Downloading rank_bm25-0.2.2-py3-none-any.whl (8.6 kB)\n",
            "Requirement already satisfied: requests>=2.28 in /usr/local/lib/python3.10/dist-packages (from chromadb) (2.31.0)\n",
            "Requirement already satisfied: pydantic>=1.9 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.10.13)\n",
            "Collecting chroma-hnswlib==0.7.3 (from chromadb)\n",
            "  Downloading chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m76.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting fastapi>=0.95.2 (from chromadb)\n",
            "  Downloading fastapi-0.104.0-py3-none-any.whl (92 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.9/92.9 kB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting uvicorn[standard]>=0.18.3 (from chromadb)\n",
            "  Downloading uvicorn-0.23.2-py3-none-any.whl (59 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.5/59.5 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting posthog>=2.4.0 (from chromadb)\n",
            "  Downloading posthog-3.0.2-py2.py3-none-any.whl (37 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.5.0)\n",
            "Collecting pulsar-client>=3.1.0 (from chromadb)\n",
            "  Downloading pulsar_client-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m69.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting onnxruntime>=1.14.1 (from chromadb)\n",
            "  Downloading onnxruntime-1.16.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.2/6.2 MB\u001b[0m \u001b[31m67.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-api>=1.2.0 (from chromadb)\n",
            "  Downloading opentelemetry_api-1.20.0-py3-none-any.whl (57 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.5/57.5 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-exporter-otlp-proto-grpc>=1.2.0 (from chromadb)\n",
            "  Downloading opentelemetry_exporter_otlp_proto_grpc-1.20.0-py3-none-any.whl (18 kB)\n",
            "Collecting opentelemetry-sdk>=1.2.0 (from chromadb)\n",
            "  Downloading opentelemetry_sdk-1.20.0-py3-none-any.whl (103 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m103.9/103.9 kB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tokenizers>=0.13.2 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.14.1)\n",
            "Collecting pypika>=0.48.9 (from chromadb)\n",
            "  Downloading PyPika-0.48.9.tar.gz (67 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tqdm>=4.65.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.66.1)\n",
            "Collecting overrides>=7.3.1 (from chromadb)\n",
            "  Downloading overrides-7.4.0-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.10/dist-packages (from chromadb) (6.1.0)\n",
            "Requirement already satisfied: grpcio>=1.58.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.59.0)\n",
            "Collecting bcrypt>=4.0.1 (from chromadb)\n",
            "  Downloading bcrypt-4.0.1-cp36-abi3-manylinux_2_28_x86_64.whl (593 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m593.7/593.7 kB\u001b[0m \u001b[31m27.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typer>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.9.0)\n",
            "Collecting kubernetes>=28.1.0 (from chromadb)\n",
            "  Downloading kubernetes-28.1.0-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m40.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tenacity>=8.2.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (8.2.3)\n",
            "Collecting numpy>=1.22.5 (from chromadb)\n",
            "  Downloading numpy-1.26.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.2/18.2 MB\u001b[0m \u001b[31m32.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2023.6.3)\n",
            "Requirement already satisfied: anyio<4.0.0,>=3.7.1 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb) (3.7.1)\n",
            "Collecting starlette<0.28.0,>=0.27.0 (from fastapi>=0.95.2->chromadb)\n",
            "  Downloading starlette-0.27.0-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.0/67.0 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-extensions>=4.5.0 (from chromadb)\n",
            "  Downloading typing_extensions-4.8.0-py3-none-any.whl (31 kB)\n",
            "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2023.7.22)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.16.0)\n",
            "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.8.2)\n",
            "Requirement already satisfied: pyyaml>=5.4.1 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (6.0.1)\n",
            "Requirement already satisfied: google-auth>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.17.3)\n",
            "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.6.4)\n",
            "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.3.1)\n",
            "Requirement already satisfied: oauthlib>=3.2.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\n",
            "Collecting urllib3<2.0,>=1.24.2 (from kubernetes>=28.1.0->chromadb)\n",
            "  Downloading urllib3-1.26.18-py2.py3-none-any.whl (143 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.8/143.8 kB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting coloredlogs (from onnxruntime>=1.14.1->chromadb)\n",
            "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (23.5.26)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (23.2)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (3.20.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (1.12)\n",
            "Collecting deprecated>=1.2.6 (from opentelemetry-api>=1.2.0->chromadb)\n",
            "  Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n",
            "Requirement already satisfied: importlib-metadata<7.0,>=6.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-api>=1.2.0->chromadb) (6.8.0)\n",
            "Requirement already satisfied: backoff<3.0.0,>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (2.2.1)\n",
            "Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.61.0)\n",
            "Collecting opentelemetry-exporter-otlp-proto-common==1.20.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb)\n",
            "  Downloading opentelemetry_exporter_otlp_proto_common-1.20.0-py3-none-any.whl (17 kB)\n",
            "Collecting opentelemetry-proto==1.20.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb)\n",
            "  Downloading opentelemetry_proto-1.20.0-py3-none-any.whl (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.8/50.8 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-semantic-conventions==0.41b0 (from opentelemetry-sdk>=1.2.0->chromadb)\n",
            "  Downloading opentelemetry_semantic_conventions-0.41b0-py3-none-any.whl (26 kB)\n",
            "Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb)\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.3.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.4)\n",
            "Requirement already satisfied: huggingface_hub<0.18,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers>=0.13.2->chromadb) (0.17.3)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb) (8.1.7)\n",
            "Collecting h11>=0.8 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httptools>=0.5.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (341 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m21.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting python-dotenv>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n",
            "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m45.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting watchfiles>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading watchfiles-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m45.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websockets>=10.4 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (130 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<4.0.0,>=3.7.1->fastapi>=0.95.2->chromadb) (1.3.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4.0.0,>=3.7.1->fastapi>=0.95.2->chromadb) (1.1.3)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated>=1.2.6->opentelemetry-api>=1.2.0->chromadb) (1.14.1)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (5.3.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<0.18,>=0.16.4->tokenizers>=0.13.2->chromadb) (3.12.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<0.18,>=0.16.4->tokenizers>=0.13.2->chromadb) (2023.6.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.17.0)\n",
            "Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime>=1.14.1->chromadb)\n",
            "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.5.0)\n",
            "Building wheels for collected packages: pypika\n",
            "  Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pypika: filename=PyPika-0.48.9-py2.py3-none-any.whl size=53723 sha256=0750b3078e6a974adb6a1950e42934bc03bd126362430e6e8b5ec21c5f63ae4d\n",
            "  Stored in directory: /root/.cache/pip/wheels/e1/26/51/d0bffb3d2fd82256676d7ad3003faea3bd6dddc9577af665f4\n",
            "Successfully built pypika\n",
            "Installing collected packages: pypika, monotonic, websockets, uvloop, urllib3, typing-extensions, python-dotenv, pulsar-client, overrides, opentelemetry-semantic-conventions, opentelemetry-proto, numpy, humanfriendly, httptools, h11, deprecated, bcrypt, watchfiles, uvicorn, starlette, rank_bm25, opentelemetry-exporter-otlp-proto-common, opentelemetry-api, coloredlogs, chroma-hnswlib, tiktoken, posthog, opentelemetry-sdk, onnxruntime, fastapi, opentelemetry-exporter-otlp-proto-grpc, kubernetes, chromadb\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 2.0.7\n",
            "    Uninstalling urllib3-2.0.7:\n",
            "      Successfully uninstalled urllib3-2.0.7\n",
            "  Attempting uninstall: typing-extensions\n",
            "    Found existing installation: typing_extensions 4.5.0\n",
            "    Uninstalling typing_extensions-4.5.0:\n",
            "      Successfully uninstalled typing_extensions-4.5.0\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 1.22.0\n",
            "    Uninstalling numpy-1.22.0:\n",
            "      Successfully uninstalled numpy-1.22.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "lida 0.0.10 requires kaleido, which is not installed.\n",
            "lida 0.0.10 requires python-multipart, which is not installed.\n",
            "bigframes 0.10.0 requires pandas>=1.5.0, but you have pandas 1.3.5 which is incompatible.\n",
            "clarifai 9.9.3 requires numpy==1.22.0, but you have numpy 1.26.1 which is incompatible.\n",
            "cupy-cuda11x 11.0.0 requires numpy<1.26,>=1.20, but you have numpy 1.26.1 which is incompatible.\n",
            "numba 0.56.4 requires numpy<1.24,>=1.18, but you have numpy 1.26.1 which is incompatible.\n",
            "plotnine 0.12.3 requires pandas>=1.5.0, but you have pandas 1.3.5 which is incompatible.\n",
            "tensorflow-probability 0.22.0 requires typing-extensions<4.6.0, but you have typing-extensions 4.8.0 which is incompatible.\n",
            "xarray 2023.7.0 requires pandas>=1.4, but you have pandas 1.3.5 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed bcrypt-4.0.1 chroma-hnswlib-0.7.3 chromadb-0.4.15 coloredlogs-15.0.1 deprecated-1.2.14 fastapi-0.104.0 h11-0.14.0 httptools-0.6.1 humanfriendly-10.0 kubernetes-28.1.0 monotonic-1.6 numpy-1.26.1 onnxruntime-1.16.1 opentelemetry-api-1.20.0 opentelemetry-exporter-otlp-proto-common-1.20.0 opentelemetry-exporter-otlp-proto-grpc-1.20.0 opentelemetry-proto-1.20.0 opentelemetry-sdk-1.20.0 opentelemetry-semantic-conventions-0.41b0 overrides-7.4.0 posthog-3.0.2 pulsar-client-3.3.0 pypika-0.48.9 python-dotenv-1.0.0 rank_bm25-0.2.2 starlette-0.27.0 tiktoken-0.5.1 typing-extensions-4.8.0 urllib3-1.26.18 uvicorn-0.23.2 uvloop-0.19.0 watchfiles-0.21.0 websockets-12.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.vectorstores import Chroma\n",
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "from langchain.retrievers import BM25Retriever, EnsembleRetriever\n",
        "\n",
        "\n",
        "doc_list = [\n",
        "    'I like apples',\n",
        "    'I like oranges',\n",
        "    'Apples and oranges are fruits',\n",
        "]\n",
        "\n",
        "bm25_retriever = BM25Retriever.from_texts(doc_list)\n",
        "bm25_retriever.k = 2\n",
        "\n",
        "embedding = OpenAIEmbeddings()\n",
        "vectorstore = Chroma.from_texts(doc_list, embedding)\n",
        "retriever = vectorstore.as_retriever(search_kwargs={'k': 2})\n",
        "\n",
        "ensemble_retriever = EnsembleRetriever(\n",
        "    retrievers=[bm25_retriever, retriever],\n",
        "    weights=[0.5, 0.5]\n",
        ")\n",
        "\n",
        "docs = ensemble_retriever.get_relevant_documents('apples')\n",
        "for doc in docs:\n",
        "    print(repr(doc))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ey1-9fOQnE5k",
        "outputId": "8fe9affb-8123-4cdf-8b70-22896a9a2813"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Document(page_content='I like apples')\n",
            "Document(page_content='Apples and oranges are fruits')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 父文档检索器"
      ],
      "metadata": {
        "id": "0Z_P43Yw4JeD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.vectorstores import Chroma\n",
        "from langchain.storage import InMemoryStore\n",
        "from langchain.document_loaders import WebBaseLoader\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "from langchain.retrievers import ParentDocumentRetriever\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "\n",
        "# 加载网页数据\n",
        "loader = WebBaseLoader(\n",
        "    'https://raw.githubusercontent.com/run-llama/llama_index/'\n",
        "    'main/examples/paul_graham_essay/data/paul_graham_essay.txt'\n",
        ")\n",
        "data = loader.load()\n",
        "parent_splitter = RecursiveCharacterTextSplitter(chunk_size=2000)\n",
        "\n",
        "# 创建拆分小块分割器\n",
        "child_splitter = RecursiveCharacterTextSplitter(chunk_size=400)\n",
        "# 创建用于存储小块的矢量存储\n",
        "vectorstore = Chroma(\n",
        "    embedding_function=OpenAIEmbeddings()\n",
        ")\n",
        "\n",
        "# 创建用于存储大块的本地存储\n",
        "store = InMemoryStore()\n",
        "\n",
        "retriever = ParentDocumentRetriever(\n",
        "    vectorstore=vectorstore,\n",
        "    docstore=store,\n",
        "    child_splitter=child_splitter,\n",
        "    parent_splitter=parent_splitter,\n",
        ")\n",
        "\n",
        "# 将原始文档和 ParentDocumentRetriever关联，\n",
        "# 用 parent_splitter 切分原始文档生成大文本块存入 InMemoryStore，并生成大文本块ID\n",
        "# 用 child_splitter 分割大文本块，\n",
        "# 将分割好的小文本块存入 Chroma 中，同时每个小文本块也会记录大文本块的 ID\n",
        "retriever.add_documents(data)\n",
        "\n",
        "retrieved_docs = retriever.get_relevant_documents('code related')\n",
        "print(retrieved_docs)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w0l6Vs462zKd",
        "outputId": "cc6a3df8-1d33-43b6-ae19-dfe3c521cb1c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Document(page_content='Now they are, though. Now you could continue using McCarthy\\'s axiomatic approach till you\\'d defined a complete programming language. And as long as every change you made to McCarthy\\'s Lisp was a discoveredness-preserving transformation, you could, in principle, end up with a complete language that had this quality. Harder to do than to talk about, of course, but if it was possible in principle, why not try? So I decided to take a shot at it. It took 4 years, from March 26, 2015 to October 12, 2019. It was fortunate that I had a precisely defined goal, or it would have been hard to keep at it for so long.\\n\\nI wrote this new Lisp, called Bel, in itself in Arc. That may sound like a contradiction, but it\\'s an indication of the sort of trickery I had to engage in to make this work. By means of an egregious collection of hacks I managed to make something close enough to an interpreter written in itself that could actually run. Not fast, but fast enough to test.\\n\\nI had to ban myself from writing essays during most of this time, or I\\'d never have finished. In late 2015 I spent 3 months writing essays, and when I went back to working on Bel I could barely understand the code. Not so much because it was badly written as because the problem is so convoluted. When you\\'re working on an interpreter written in itself, it\\'s hard to keep track of what\\'s happening at what level, and errors can be practically encrypted by the time you get them.\\n\\nSo I said no more essays till Bel was done. But I told few people about Bel while I was working on it. So for years it must have seemed that I was doing nothing, when in fact I was working harder than I\\'d ever worked on anything. Occasionally after wrestling for hours with some gruesome bug I\\'d check Twitter or HN and see someone asking \"Does Paul Graham still code?\"', metadata={'source': 'https://raw.githubusercontent.com/run-llama/llama_index/main/examples/paul_graham_essay/data/paul_graham_essay.txt'}), Document(page_content=\"[8] Most software you can launch as soon as it's done. But when the software is an online store builder and you're hosting the stores, if you don't have any users yet, that fact will be painfully obvious. So before we could launch publicly we had to launch privately, in the sense of recruiting an initial set of users and making sure they had decent-looking stores.\\n\\n[9] We'd had a code editor in Viaweb for users to define their own page styles. They didn't know it, but they were editing Lisp expressions underneath. But this wasn't an app editor, because the code ran when the merchants' sites were generated, not when shoppers visited them.\\n\\n[10] This was the first instance of what is now a familiar experience, and so was what happened next, when I read the comments and found they were full of angry people. How could I claim that Lisp was better than other languages? Weren't they all Turing complete? People who see the responses to essays I write sometimes tell me how sorry they feel for me, but I'm not exaggerating when I reply that it has always been like this, since the very beginning. It comes with the territory. An essay must tell readers things they don't already know, and some people dislike being told such things.\\n\\n[11] People put plenty of stuff on the internet in the 90s of course, but putting something online is not the same as publishing it online. Publishing online means you treat the online version as the (or at least a) primary version.\", metadata={'source': 'https://raw.githubusercontent.com/run-llama/llama_index/main/examples/paul_graham_essay/data/paul_graham_essay.txt'})]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 多向量检索器"
      ],
      "metadata": {
        "id": "3ogNss1T4Pq8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import uuid\n",
        "\n",
        "from langchain.vectorstores import Chroma\n",
        "from langchain.storage import InMemoryStore\n",
        "from langchain.chat_models import ChatOpenAI\n",
        "from langchain.schema.document import Document\n",
        "from langchain.prompts import ChatPromptTemplate\n",
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "from langchain.document_loaders import WebBaseLoader\n",
        "from langchain.schema.output_parser import StrOutputParser\n",
        "from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "\n",
        "# 加载网页数据\n",
        "loader = WebBaseLoader(\n",
        "    'https://raw.githubusercontent.com/run-llama/llama_index/'\n",
        "    'main/examples/paul_graham_essay/data/paul_graham_essay.txt'\n",
        ")\n",
        "data = loader.load()\n",
        "\n",
        "# 切分文档块\n",
        "text_splitter = RecursiveCharacterTextSplitter(\n",
        "    chunk_size=10000\n",
        ")\n",
        "docs = text_splitter.split_documents(data)\n",
        "\n",
        "# 通过 chain 批量对文本块生成摘要\n",
        "# 关于 chain 相关内容，在下一个章节会进行详细讲解\n",
        "summary_tpl = ChatPromptTemplate.from_template(\n",
        "    'Summarize the following document:\\n\\n{doc}'\n",
        ")\n",
        "\n",
        "chain = (\n",
        "    {'doc': lambda x: x.page_content}\n",
        "    | summary_tpl\n",
        "    | ChatOpenAI(max_retries=0)\n",
        "    | StrOutputParser()\n",
        ")\n",
        "summaries = chain.batch(docs, {'max_concurrency': 5})\n",
        "\n",
        "# 创建用于存储总结的向量存储\n",
        "vectorstore = Chroma(\n",
        "    collection_name=\"summaries\",\n",
        "    embedding_function=OpenAIEmbeddings()\n",
        ")\n",
        "\n",
        "# 创建用于存储文本块的文档存储\n",
        "store = InMemoryStore()\n",
        "\n",
        "# 创建多向量检索器\n",
        "id_key = 'doc_id'\n",
        "retriever = MultiVectorRetriever(\n",
        "    vectorstore=vectorstore,\n",
        "    docstore=store,\n",
        "    id_key=id_key,\n",
        "    search_kwargs={'k': 4}\n",
        ")\n",
        "\n",
        "# 批量生成文档对应的 ID\n",
        "doc_ids1 = [str(uuid.uuid4()) for _ in docs]\n",
        "print(doc_ids1)\n",
        "\n",
        "# 创建带有摘要和文档 ID 的 Document 列表\n",
        "summary_docs = [\n",
        "    Document(page_content=s, metadata={id_key: doc_ids1[i]})\n",
        "    for i, s in enumerate(summaries)\n",
        "]\n",
        "for s in summary_docs:\n",
        "    print(s)\n",
        "\n",
        "# 将摘要 Document 列表 存入向量存储\n",
        "retriever.vectorstore.add_documents(summary_docs)\n",
        "# 将文本块和对应的 ID 存入本地存储\n",
        "retriever.docstore.mset(list(zip(doc_ids1, docs)))\n",
        "\n",
        "\n",
        "# 进行检索\n",
        "retrieved_docs = retriever.get_relevant_documents('code related')\n",
        "print(retrieved_docs)\n",
        "\n",
        "\n",
        "sub_docs = vectorstore.similarity_search('code related')\n",
        "print(sub_docs)\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VM5zWRFjF8oF",
        "outputId": "9c0052c3-59b9-4fab-8ce9-3e7f331d6776"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['8d3d01bc-aa10-4eb8-b15b-84871b3822e4', '395580a2-3887-461e-9c7b-5f608a39d87f', '18997bd2-5958-4d17-8c90-690f80696039', 'd7ae5d41-1187-458e-a11d-8c13669e418d', '8514dae8-1aed-481b-9e24-1b0482d45a3b', '51f51546-37db-44e5-81f8-fb6159b22dbe', 'e194f23c-1f00-427d-875d-fa46282c1192', '0dcd1290-3f89-4bd9-b46b-22ed0c81ce44']\n",
            "page_content='The author discusses their early experiences with writing and programming before college. They recall writing short stories and experimenting with programming on an IBM 1401 computer in 9th grade. They later transitioned to microcomputers and started programming games and other applications. Despite initially planning to study philosophy in college, they became interested in artificial intelligence (AI) and taught themselves Lisp, the language associated with AI at the time. They pursued AI in graduate school but eventually realized its limitations and shifted their focus to Lisp. They decided to write a book about Lisp hacking and became interested in building things that would last. During a visit to the Carnegie Institute, they realized the longevity of paintings and considered pursuing art as a career. They believed that as an artist, they could be independent and make a living.' metadata={'doc_id': '8d3d01bc-aa10-4eb8-b15b-84871b3822e4'}\n",
            "page_content='The author discusses their journey from being interested in art to pursuing a PhD in computer science. They took art classes while in grad school and decided to write their dissertation in a short amount of time to be able to graduate. They applied to art schools and got accepted to RISD. They describe their experience at the Accademia di Belli Arti in Florence, where they learned to paint still lifes. They eventually returned to the US, got a job at a software company, and saved enough money to go back to RISD and pay off their college loans.' metadata={'doc_id': '395580a2-3887-461e-9c7b-5f608a39d87f'}\n",
            "page_content='The document discusses the author\\'s experiences and lessons learned at Interleaf, an organization they worked for. They learned that it is better for technology companies to be run by product people rather than sales people. They also learned about the dangers of editing code by too many people, the importance of having a positive office space, the value of corridor conversations over planned meetings, and the risks associated with big bureaucratic customers. The author also emphasizes the importance of being the \"entry level\" option in the market, as the low end tends to dominate the high end. The document then transitions to the author\\'s time at art school and their decision to drop out and move to New York. They discuss their interest in the World Wide Web and their attempt to start a company for online art galleries, which eventually evolved into building online stores. The author reflects on the challenges they faced and the early stages of their company, Viaweb, which later became a model for Y Combinator\\'s funding structure.' metadata={'doc_id': '18997bd2-5958-4d17-8c90-690f80696039'}\n",
            "page_content='The document discusses the founding and early years of the company Viaweb, an online store builder, which later got acquired by Yahoo. The author mentions the initial ambition to launch the software in September but became more ambitious as they worked on it. They managed to build a WYSIWYG site builder that made online stores look professional. They recruited more programmers and opened for business in January 1996. The author reflects on the challenges and successes of the company, including their low pricing strategy and their focus on user needs. The document also mentions the stress and fatigue experienced by the author during this time. The author eventually leaves the company to pursue his passion for painting.' metadata={'doc_id': 'd7ae5d41-1187-458e-a11d-8c13669e418d'}\n",
            "page_content=\"The document describes the author's experiences after becoming rich in New York. They resumed their old patterns but now had access to new opportunities, such as taking taxis and eating at restaurants. The author then had an idea to build a web app for making web apps and decided to start a company in Cambridge. However, their partner refused to work on the project, so the author recruited others and started building the software. Eventually, they decided to create a new dialect of Lisp and started working on it. The author also realized the power of publishing essays online and began writing and publishing essays. They found that working on less prestigious projects often led to meaningful discoveries. The author continued writing essays on various topics, worked on spam filters, and had a dinner group. They met a woman named Jessica Livingston and started dating. Jessica compiled a book of interviews with startup founders, and the author began discussing the flaws in venture capital. The author was then asked to give a talk on how to start a startup.\" metadata={'doc_id': '8514dae8-1aed-481b-9e24-1b0482d45a3b'}\n",
            "page_content=\"The document describes the founding and growth of Y Combinator (YC), an angel investment firm and startup accelerator. The author explains how they stumbled upon the idea of funding startups in batches, which allowed them to provide more support and create a community among founders. The document also mentions the creation of Hacker News, a news aggregator for startup founders, and the challenges and stress that came with running it. The author reflects on their work at YC, including both the engaging aspects and the parts they didn't enjoy.\" metadata={'doc_id': '51f51546-37db-44e5-81f8-fb6159b22dbe'}\n",
            "page_content=\"The document discusses the author's journey after receiving unsolicited advice from Robert Morris. The author reflects on the suggestion to not let Y Combinator be the last thing they do and eventually decides to hand over control of YC to Sam Altman. The author then goes on to pursue painting but eventually loses interest and starts writing essays again. They then delve into the study of Lisp and spend several years working on a new Lisp called Bel. The author shares their experience and satisfaction in working on Bel and eventually finishes it in 2019. They then resume writing essays and start considering new projects to work on. The document ends with a mention of an essay the author wrote about how they choose what to work on.\" metadata={'doc_id': 'e194f23c-1f00-427d-875d-fa46282c1192'}\n",
            "page_content='The document consists of several excerpts that touch on various topics.\\n\\nExcerpt [3] describes the author\\'s experience living in Florence and walking through the city on a daily basis, observing its different conditions.\\n\\nExcerpt [4] discusses the possibility of painting people as still life subjects, highlighting the challenges of capturing natural expressions.\\n\\nExcerpt [5] mentions a company called Interleaf that was impacted by Moore\\'s Law and the exponential growth of commodity processors.\\n\\nExcerpt [6] explores the relationship between money and coolness in the art world, where expensive things are often seen as cool.\\n\\nExcerpt [7] mentions the author\\'s apartment in New York City, which was rent-stabilized and significantly cheaper than the market price.\\n\\nExcerpt [8] talks about the need to recruit an initial set of users and ensure their online stores look decent before publicly launching an online store builder software.\\n\\nExcerpt [9] refers to a code editor in Viaweb that allowed users to define their own page styles, unknowingly editing Lisp expressions.\\n\\nExcerpt [10] discusses the author\\'s experience of encountering backlash and criticism when claiming that Lisp was better than other programming languages.\\n\\nExcerpt [11] discusses the distinction between putting something online and publishing it online, emphasizing the importance of treating the online version as the primary version.\\n\\nExcerpt [12] explores how customs and traditions can continue to influence and constrain industries, even after the original constraints have disappeared.\\n\\nExcerpt [13] explains the name change of Y Combinator and the reasoning behind the choice of color for its logo.\\n\\nExcerpt [14] mentions that Y Combinator became a fund for a couple of years before returning to being self-funded.\\n\\nExcerpt [15] criticizes the term \"deal flow\" and explains how Y Combinator aims to create startups that would not have existed otherwise.\\n\\nExcerpt [16] describes the experience of a person struggling to carry heavy air conditioners due to a shortage.\\n\\nExcerpt [17] discusses the challenges faced when running a forum and writing essays, including dealing with misinterpretations and the need to respond to them.\\n\\nExcerpt [18] expresses the author\\'s sadness about leaving Y Combinator and no longer working with Jessica, their long-time collaborator.\\n\\nExcerpt [19] introduces the concept of invented vs discovered by discussing the potential knowledge of advanced alien civilizations and their understanding of concepts like the Pythagorean theorem and Lisp.' metadata={'doc_id': '0dcd1290-3f89-4bd9-b46b-22ed0c81ce44'}\n",
            "[Document(page_content='[3] I lived at Piazza San Felice 4, so my walk to the Accademia went straight down the spine of old Florence: past the Pitti, across the bridge, past Orsanmichele, between the Duomo and the Baptistery, and then up Via Ricasoli to Piazza San Marco. I saw Florence at street level in every possible condition, from empty dark winter evenings to sweltering summer days when the streets were packed with tourists.\\n\\n[4] You can of course paint people like still lives if you want to, and they\\'re willing. That sort of portrait is arguably the apex of still life painting, though the long sitting does tend to produce pained expressions in the sitters.\\n\\n[5] Interleaf was one of many companies that had smart people and built impressive technology, and yet got crushed by Moore\\'s Law. In the 1990s the exponential growth in the power of commodity (i.e. Intel) processors rolled up high-end, special-purpose hardware and software companies like a bulldozer.\\n\\n[6] The signature style seekers at RISD weren\\'t specifically mercenary. In the art world, money and coolness are tightly coupled. Anything expensive comes to be seen as cool, and anything seen as cool will soon become equally expensive.\\n\\n[7] Technically the apartment wasn\\'t rent-controlled but rent-stabilized, but this is a refinement only New Yorkers would know or care about. The point is that it was really cheap, less than half market price.\\n\\n[8] Most software you can launch as soon as it\\'s done. But when the software is an online store builder and you\\'re hosting the stores, if you don\\'t have any users yet, that fact will be painfully obvious. So before we could launch publicly we had to launch privately, in the sense of recruiting an initial set of users and making sure they had decent-looking stores.\\n\\n[9] We\\'d had a code editor in Viaweb for users to define their own page styles. They didn\\'t know it, but they were editing Lisp expressions underneath. But this wasn\\'t an app editor, because the code ran when the merchants\\' sites were generated, not when shoppers visited them.\\n\\n[10] This was the first instance of what is now a familiar experience, and so was what happened next, when I read the comments and found they were full of angry people. How could I claim that Lisp was better than other languages? Weren\\'t they all Turing complete? People who see the responses to essays I write sometimes tell me how sorry they feel for me, but I\\'m not exaggerating when I reply that it has always been like this, since the very beginning. It comes with the territory. An essay must tell readers things they don\\'t already know, and some people dislike being told such things.\\n\\n[11] People put plenty of stuff on the internet in the 90s of course, but putting something online is not the same as publishing it online. Publishing online means you treat the online version as the (or at least a) primary version.\\n\\n[12] There is a general lesson here that our experience with Y Combinator also teaches: Customs continue to constrain you long after the restrictions that caused them have disappeared. Customary VC practice had once, like the customs about publishing essays, been based on real constraints. Startups had once been much more expensive to start, and proportionally rare. Now they could be cheap and common, but the VCs\\' customs still reflected the old world, just as customs about writing essays still reflected the constraints of the print era.\\n\\nWhich in turn implies that people who are independent-minded (i.e. less influenced by custom) will have an advantage in fields affected by rapid change (where customs are more likely to be obsolete).\\n\\nHere\\'s an interesting point, though: you can\\'t always predict which fields will be affected by rapid change. Obviously software and venture capital will be, but who would have predicted that essay writing would be?\\n\\n[13] Y Combinator was not the original name. At first we were called Cambridge Seed. But we didn\\'t want a regional name, in case someone copied us in Silicon Valley, so we renamed ourselves after one of the coolest tricks in the lambda calculus, the Y combinator.\\n\\nI picked orange as our color partly because it\\'s the warmest, and partly because no VC used it. In 2005 all the VCs used staid colors like maroon, navy blue, and forest green, because they were trying to appeal to LPs, not founders. The YC logo itself is an inside joke: the Viaweb logo had been a white V on a red circle, so I made the YC logo a white Y on an orange square.\\n\\n[14] YC did become a fund for a couple years starting in 2009, because it was getting so big I could no longer afford to fund it personally. But after Heroku got bought we had enough money to go back to being self-funded.\\n\\n[15] I\\'ve never liked the term \"deal flow,\" because it implies that the number of new startups at any given time is fixed. This is not only false, but it\\'s the purpose of YC to falsify it, by causing startups to be founded that would not otherwise have existed.\\n\\n[16] She reports that they were all different shapes and sizes, because there was a run on air conditioners and she had to get whatever she could, but that they were all heavier than she could carry now.\\n\\n[17] Another problem with HN was a bizarre edge case that occurs when you both write essays and run a forum. When you run a forum, you\\'re assumed to see if not every conversation, at least every conversation involving you. And when you write essays, people post highly imaginative misinterpretations of them on forums. Individually these two phenomena are tedious but bearable, but the combination is disastrous. You actually have to respond to the misinterpretations, because the assumption that you\\'re present in the conversation means that not responding to any sufficiently upvoted misinterpretation reads as a tacit admission that it\\'s correct. But that in turn encourages more; anyone who wants to pick a fight with you senses that now is their chance.\\n\\n[18] The worst thing about leaving YC was not working with Jessica anymore. We\\'d been working on YC almost the whole time we\\'d known each other, and we\\'d neither tried nor wanted to separate it from our personal lives, so leaving was like pulling up a deeply rooted tree.\\n\\n[19] One way to get more precise about the concept of invented vs discovered is to talk about space aliens. Any sufficiently advanced alien civilization would certainly know about the Pythagorean theorem, for example. I believe, though with less certainty, that they would also know about the Lisp in McCarthy\\'s 1960 paper.\\n\\nBut if so there\\'s no reason to suppose that this is the limit of the language that might be known to them. Presumably aliens need numbers and errors and I/O too. So it seems likely there exists at least one path out of McCarthy\\'s Lisp along which discoveredness is preserved.\\n\\n\\n\\nThanks to Trevor Blackwell, John Collison, Patrick Collison, Daniel Gackle, Ralph Hazell, Jessica Livingston, Robert Morris, and Harj Taggar for reading drafts of this.', metadata={'source': 'https://raw.githubusercontent.com/run-llama/llama_index/main/examples/paul_graham_essay/data/paul_graham_essay.txt'})]\n",
            "[Document(page_content=\"The document is a collection of various snippets or excerpts that cover different topics. Some of the topics discussed include the author's experience living in Florence, the impact of Moore's Law on technology companies, the relationship between money and coolness in the art world, the difference between rent-controlled and rent-stabilized apartments, the process of launching a software product, the use of Lisp in programming, the challenges of publishing essays online, the influence of customs and traditions on industries, the history and naming of Y Combinator, the purpose of Y Combinator in creating new startups, personal experiences and relationships, the concept of invented vs discovered knowledge, and acknowledgments to people who provided feedback on drafts of the document.\", metadata={'doc_id': '19e96007-89c9-444b-b051-c48d56a5db97'}), Document(page_content=\"The document discusses the author's experiences with writing and programming before and during college. They began writing short stories and later started programming on an IBM 1401 in high school. They then obtained a microcomputer and began programming games and other projects. Initially, the author planned to study philosophy in college but switched to studying artificial intelligence (AI) due to their interest in the subject. However, during graduate school, the author realized that the current approach to AI was not effective and shifted their focus to Lisp programming. They decided to write a book on Lisp hacking and became interested in building things that would last. The author also considered pursuing a career in art, as they saw paintings as something that could withstand the test of time.\", metadata={'doc_id': '67ca62ec-0c4b-4e8a-8906-0f2cc7056aa5'}), Document(page_content='The document consists of several excerpts that touch on various topics.\\n\\nExcerpt [3] describes the author\\'s experience living in Florence and walking through the city on a daily basis, observing its different conditions.\\n\\nExcerpt [4] discusses the possibility of painting people as still life subjects, highlighting the challenges of capturing natural expressions.\\n\\nExcerpt [5] mentions a company called Interleaf that was impacted by Moore\\'s Law and the exponential growth of commodity processors.\\n\\nExcerpt [6] explores the relationship between money and coolness in the art world, where expensive things are often seen as cool.\\n\\nExcerpt [7] mentions the author\\'s apartment in New York City, which was rent-stabilized and significantly cheaper than the market price.\\n\\nExcerpt [8] talks about the need to recruit an initial set of users and ensure their online stores look decent before publicly launching an online store builder software.\\n\\nExcerpt [9] refers to a code editor in Viaweb that allowed users to define their own page styles, unknowingly editing Lisp expressions.\\n\\nExcerpt [10] discusses the author\\'s experience of encountering backlash and criticism when claiming that Lisp was better than other programming languages.\\n\\nExcerpt [11] discusses the distinction between putting something online and publishing it online, emphasizing the importance of treating the online version as the primary version.\\n\\nExcerpt [12] explores how customs and traditions can continue to influence and constrain industries, even after the original constraints have disappeared.\\n\\nExcerpt [13] explains the name change of Y Combinator and the reasoning behind the choice of color for its logo.\\n\\nExcerpt [14] mentions that Y Combinator became a fund for a couple of years before returning to being self-funded.\\n\\nExcerpt [15] criticizes the term \"deal flow\" and explains how Y Combinator aims to create startups that would not have existed otherwise.\\n\\nExcerpt [16] describes the experience of a person struggling to carry heavy air conditioners due to a shortage.\\n\\nExcerpt [17] discusses the challenges faced when running a forum and writing essays, including dealing with misinterpretations and the need to respond to them.\\n\\nExcerpt [18] expresses the author\\'s sadness about leaving Y Combinator and no longer working with Jessica, their long-time collaborator.\\n\\nExcerpt [19] introduces the concept of invented vs discovered by discussing the potential knowledge of advanced alien civilizations and their understanding of concepts like the Pythagorean theorem and Lisp.', metadata={'doc_id': '0dcd1290-3f89-4bd9-b46b-22ed0c81ce44'}), Document(page_content=\"The document is a personal account of the author's experiences and decisions regarding their involvement with Y Combinator (YC), a startup accelerator. The author recalls receiving advice from Robert Morris to not let YC be the last significant thing they do. The author reflects on the idea and realizes that they need to eventually leave YC to pursue other endeavors. Eventually, the author decides to step back from YC and focuses on helping their mother, who had cancer. After their mother's death, the author explores different activities, such as painting and writing essays. They then delve into the development of a programming language called Bel, which took them four years to complete. The author reflects on their journey and contemplates their future endeavors.\", metadata={'doc_id': 'e17e8f43-8f77-4ada-8a6b-b7a2539f53c7'})]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install chromadb tiktoken"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1-jmCwk6IOBF",
        "outputId": "07d54c77-1b6a-447e-8477-103a3e685b60"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting chromadb\n",
            "  Downloading chromadb-0.4.15-py3-none-any.whl (479 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m479.8/479.8 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tiktoken\n",
            "  Downloading tiktoken-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m52.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.28 in /usr/local/lib/python3.10/dist-packages (from chromadb) (2.31.0)\n",
            "Requirement already satisfied: pydantic>=1.9 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.10.13)\n",
            "Collecting chroma-hnswlib==0.7.3 (from chromadb)\n",
            "  Downloading chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m62.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting fastapi>=0.95.2 (from chromadb)\n",
            "  Downloading fastapi-0.104.1-py3-none-any.whl (92 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.9/92.9 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting uvicorn[standard]>=0.18.3 (from chromadb)\n",
            "  Downloading uvicorn-0.23.2-py3-none-any.whl (59 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.5/59.5 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting posthog>=2.4.0 (from chromadb)\n",
            "  Downloading posthog-3.0.2-py2.py3-none-any.whl (37 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.5.0)\n",
            "Collecting pulsar-client>=3.1.0 (from chromadb)\n",
            "  Downloading pulsar_client-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m83.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting onnxruntime>=1.14.1 (from chromadb)\n",
            "  Downloading onnxruntime-1.16.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.2/6.2 MB\u001b[0m \u001b[31m91.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-api>=1.2.0 (from chromadb)\n",
            "  Downloading opentelemetry_api-1.20.0-py3-none-any.whl (57 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.5/57.5 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-exporter-otlp-proto-grpc>=1.2.0 (from chromadb)\n",
            "  Downloading opentelemetry_exporter_otlp_proto_grpc-1.20.0-py3-none-any.whl (18 kB)\n",
            "Collecting opentelemetry-sdk>=1.2.0 (from chromadb)\n",
            "  Downloading opentelemetry_sdk-1.20.0-py3-none-any.whl (103 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m103.9/103.9 kB\u001b[0m \u001b[31m14.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tokenizers>=0.13.2 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.14.1)\n",
            "Collecting pypika>=0.48.9 (from chromadb)\n",
            "  Downloading PyPika-0.48.9.tar.gz (67 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tqdm>=4.65.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.66.1)\n",
            "Collecting overrides>=7.3.1 (from chromadb)\n",
            "  Downloading overrides-7.4.0-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.10/dist-packages (from chromadb) (6.1.0)\n",
            "Requirement already satisfied: grpcio>=1.58.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.59.0)\n",
            "Collecting bcrypt>=4.0.1 (from chromadb)\n",
            "  Downloading bcrypt-4.0.1-cp36-abi3-manylinux_2_28_x86_64.whl (593 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m593.7/593.7 kB\u001b[0m \u001b[31m49.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typer>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.9.0)\n",
            "Collecting kubernetes>=28.1.0 (from chromadb)\n",
            "  Downloading kubernetes-28.1.0-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m82.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tenacity>=8.2.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (8.2.3)\n",
            "Collecting numpy>=1.22.5 (from chromadb)\n",
            "  Downloading numpy-1.26.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.2/18.2 MB\u001b[0m \u001b[31m54.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2023.6.3)\n",
            "Requirement already satisfied: anyio<4.0.0,>=3.7.1 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb) (3.7.1)\n",
            "Collecting starlette<0.28.0,>=0.27.0 (from fastapi>=0.95.2->chromadb)\n",
            "  Downloading starlette-0.27.0-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.0/67.0 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-extensions>=4.5.0 (from chromadb)\n",
            "  Downloading typing_extensions-4.8.0-py3-none-any.whl (31 kB)\n",
            "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2023.7.22)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.16.0)\n",
            "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.8.2)\n",
            "Requirement already satisfied: pyyaml>=5.4.1 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (6.0.1)\n",
            "Requirement already satisfied: google-auth>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.17.3)\n",
            "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.6.4)\n",
            "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.3.1)\n",
            "Requirement already satisfied: oauthlib>=3.2.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\n",
            "Collecting urllib3<2.0,>=1.24.2 (from kubernetes>=28.1.0->chromadb)\n",
            "  Downloading urllib3-1.26.18-py2.py3-none-any.whl (143 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.8/143.8 kB\u001b[0m \u001b[31m16.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting coloredlogs (from onnxruntime>=1.14.1->chromadb)\n",
            "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (23.5.26)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (23.2)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (3.20.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (1.12)\n",
            "Collecting deprecated>=1.2.6 (from opentelemetry-api>=1.2.0->chromadb)\n",
            "  Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n",
            "Requirement already satisfied: importlib-metadata<7.0,>=6.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-api>=1.2.0->chromadb) (6.8.0)\n",
            "Requirement already satisfied: backoff<3.0.0,>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (2.2.1)\n",
            "Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.61.0)\n",
            "Collecting opentelemetry-exporter-otlp-proto-common==1.20.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb)\n",
            "  Downloading opentelemetry_exporter_otlp_proto_common-1.20.0-py3-none-any.whl (17 kB)\n",
            "Collecting opentelemetry-proto==1.20.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb)\n",
            "  Downloading opentelemetry_proto-1.20.0-py3-none-any.whl (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.8/50.8 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-semantic-conventions==0.41b0 (from opentelemetry-sdk>=1.2.0->chromadb)\n",
            "  Downloading opentelemetry_semantic_conventions-0.41b0-py3-none-any.whl (26 kB)\n",
            "Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb)\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.3.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.4)\n",
            "Requirement already satisfied: huggingface_hub<0.18,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers>=0.13.2->chromadb) (0.17.3)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb) (8.1.7)\n",
            "Collecting h11>=0.8 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httptools>=0.5.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (341 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m33.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting python-dotenv>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n",
            "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m60.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting watchfiles>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading watchfiles-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m56.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websockets>=10.4 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (130 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m14.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<4.0.0,>=3.7.1->fastapi>=0.95.2->chromadb) (1.3.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4.0.0,>=3.7.1->fastapi>=0.95.2->chromadb) (1.1.3)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated>=1.2.6->opentelemetry-api>=1.2.0->chromadb) (1.14.1)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (5.3.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<0.18,>=0.16.4->tokenizers>=0.13.2->chromadb) (3.12.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<0.18,>=0.16.4->tokenizers>=0.13.2->chromadb) (2023.6.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.17.0)\n",
            "Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime>=1.14.1->chromadb)\n",
            "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.5.0)\n",
            "Building wheels for collected packages: pypika\n",
            "  Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pypika: filename=PyPika-0.48.9-py2.py3-none-any.whl size=53723 sha256=b1deed1b9f1a33b19df672604b74943a3c053ed735dd651e850d05fdc6686b84\n",
            "  Stored in directory: /root/.cache/pip/wheels/e1/26/51/d0bffb3d2fd82256676d7ad3003faea3bd6dddc9577af665f4\n",
            "Successfully built pypika\n",
            "Installing collected packages: pypika, monotonic, websockets, uvloop, urllib3, typing-extensions, python-dotenv, pulsar-client, overrides, opentelemetry-semantic-conventions, opentelemetry-proto, numpy, humanfriendly, httptools, h11, deprecated, bcrypt, watchfiles, uvicorn, starlette, opentelemetry-exporter-otlp-proto-common, opentelemetry-api, coloredlogs, chroma-hnswlib, tiktoken, posthog, opentelemetry-sdk, onnxruntime, fastapi, opentelemetry-exporter-otlp-proto-grpc, kubernetes, chromadb\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 2.0.7\n",
            "    Uninstalling urllib3-2.0.7:\n",
            "      Successfully uninstalled urllib3-2.0.7\n",
            "  Attempting uninstall: typing-extensions\n",
            "    Found existing installation: typing_extensions 4.5.0\n",
            "    Uninstalling typing_extensions-4.5.0:\n",
            "      Successfully uninstalled typing_extensions-4.5.0\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 1.22.0\n",
            "    Uninstalling numpy-1.22.0:\n",
            "      Successfully uninstalled numpy-1.22.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "lida 0.0.10 requires kaleido, which is not installed.\n",
            "lida 0.0.10 requires python-multipart, which is not installed.\n",
            "bigframes 0.10.0 requires pandas>=1.5.0, but you have pandas 1.3.5 which is incompatible.\n",
            "clarifai 9.9.3 requires numpy==1.22.0, but you have numpy 1.26.1 which is incompatible.\n",
            "cupy-cuda11x 11.0.0 requires numpy<1.26,>=1.20, but you have numpy 1.26.1 which is incompatible.\n",
            "numba 0.56.4 requires numpy<1.24,>=1.18, but you have numpy 1.26.1 which is incompatible.\n",
            "plotnine 0.12.3 requires pandas>=1.5.0, but you have pandas 1.3.5 which is incompatible.\n",
            "tensorflow-probability 0.22.0 requires typing-extensions<4.6.0, but you have typing-extensions 4.8.0 which is incompatible.\n",
            "xarray 2023.7.0 requires pandas>=1.4, but you have pandas 1.3.5 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed bcrypt-4.0.1 chroma-hnswlib-0.7.3 chromadb-0.4.15 coloredlogs-15.0.1 deprecated-1.2.14 fastapi-0.104.1 h11-0.14.0 httptools-0.6.1 humanfriendly-10.0 kubernetes-28.1.0 monotonic-1.6 numpy-1.26.1 onnxruntime-1.16.1 opentelemetry-api-1.20.0 opentelemetry-exporter-otlp-proto-common-1.20.0 opentelemetry-exporter-otlp-proto-grpc-1.20.0 opentelemetry-proto-1.20.0 opentelemetry-sdk-1.20.0 opentelemetry-semantic-conventions-0.41b0 overrides-7.4.0 posthog-3.0.2 pulsar-client-3.3.0 pypika-0.48.9 python-dotenv-1.0.0 starlette-0.27.0 tiktoken-0.5.1 typing-extensions-4.8.0 urllib3-1.26.18 uvicorn-0.23.2 uvloop-0.19.0 watchfiles-0.21.0 websockets-12.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import uuid\n",
        "\n",
        "from langchain.vectorstores import Chroma\n",
        "from langchain.storage import InMemoryStore\n",
        "from langchain.chat_models import ChatOpenAI\n",
        "from langchain.schema.document import Document\n",
        "from langchain.prompts import ChatPromptTemplate\n",
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "from langchain.document_loaders import WebBaseLoader\n",
        "from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain.output_parsers.openai_functions import (\n",
        "    JsonKeyOutputFunctionsParser\n",
        ")\n",
        "\n",
        "# 加载网页数据\n",
        "loader = WebBaseLoader(\n",
        "    'https://raw.githubusercontent.com/run-llama/llama_index/'\n",
        "    'main/examples/paul_graham_essay/data/paul_graham_essay.txt'\n",
        ")\n",
        "data = loader.load()\n",
        "\n",
        "# 切分文档块\n",
        "text_splitter = RecursiveCharacterTextSplitter(\n",
        "    chunk_size=10000\n",
        ")\n",
        "docs = text_splitter.split_documents(data)\n",
        "\n",
        "# 创建一个 function call 函数描述\n",
        "functions = [\n",
        "    {\n",
        "        'name': 'hypothetical_questions',\n",
        "        'description': 'Generate hypothetical questions',\n",
        "        'parameters': {\n",
        "            'type': 'object',\n",
        "            'properties': {\n",
        "                'questions': {\n",
        "                    'type': 'array',\n",
        "                    'items': {'type': 'string'},\n",
        "                },\n",
        "            },\n",
        "            'required': ['questions'],\n",
        "        },\n",
        "    }\n",
        "]\n",
        "chain = (\n",
        "    {'doc': lambda x: x.page_content}\n",
        "    | ChatPromptTemplate.from_template(\n",
        "        'Generate a list of 3 hypothetical questions that the '\n",
        "        'below document could be used to answer:\\n\\n{doc}'\n",
        "    )\n",
        "    | ChatOpenAI(max_retries=0).bind(\n",
        "        functions=functions,\n",
        "        function_call={'name': 'hypothetical_questions'}\n",
        "    )\n",
        "    | JsonKeyOutputFunctionsParser(key_name='questions')\n",
        ")\n",
        "\n",
        "questions = chain.batch(docs, {'max_concurrency': 5})\n",
        "\n",
        "# 创建用于存储总结的向量存储\n",
        "vectorstore = Chroma(\n",
        "    collection_name='hypo-questions',\n",
        "    embedding_function=OpenAIEmbeddings()\n",
        ")\n",
        "\n",
        "# 创建用于存储文本块的文档存储\n",
        "store = InMemoryStore()\n",
        "\n",
        "# 创建多向量检索器\n",
        "id_key = 'doc_id'\n",
        "retriever = MultiVectorRetriever(\n",
        "    vectorstore=vectorstore,\n",
        "    docstore=store,\n",
        "    id_key=id_key,\n",
        ")\n",
        "\n",
        "# 批量生成文档对应的 ID\n",
        "doc_ids = [str(uuid.uuid4()) for _ in docs]\n",
        "\n",
        "# 创建带有摘要和文档 ID 的 Document 列表\n",
        "question_docs = []\n",
        "for i, question_list in enumerate(questions):\n",
        "    question_docs.extend(\n",
        "        [Document(page_content=s, metadata={id_key: doc_ids[i]})\n",
        "        for s in question_list]\n",
        "    )\n",
        "\n",
        "# 将摘要 Document 列表 存入向量存储\n",
        "retriever.vectorstore.add_documents(question_docs)\n",
        "# 将文本块和对应的 ID 存入本地存储\n",
        "retriever.docstore.mset(list(zip(doc_ids, docs)))\n",
        "\n",
        "# 进行检索\n",
        "retrieved_docs = retriever.get_relevant_documents('code related')\n",
        "print(retrieved_docs)\n",
        "\n",
        "# 也可以通过 vectorstore.similarity_search 方法查看到对应搜索到的摘要\n",
        "sub_docs = vectorstore.similarity_search('code related')\n",
        "print(sub_docs)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iv7thA4YIEAt",
        "outputId": "c446bac5-3125-41a1-f6b8-2f6f7cd3d443"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Document(page_content='I learned some useful things at Interleaf, though they were mostly about what not to do. I learned that it\\'s better for technology companies to be run by product people than sales people (though sales is a real skill and people who are good at it are really good at it), that it leads to bugs when code is edited by too many people, that cheap office space is no bargain if it\\'s depressing, that planned meetings are inferior to corridor conversations, that big, bureaucratic customers are a dangerous source of money, and that there\\'s not much overlap between conventional office hours and the optimal time for hacking, or conventional offices and the optimal place for it.\\n\\nBut the most important thing I learned, and which I used in both Viaweb and Y Combinator, is that the low end eats the high end: that it\\'s good to be the \"entry level\" option, even though that will be less prestigious, because if you\\'re not, someone else will be, and will squash you against the ceiling. Which in turn means that prestige is a danger sign.\\n\\nWhen I left to go back to RISD the next fall, I arranged to do freelance work for the group that did projects for customers, and this was how I survived for the next several years. When I came back to visit for a project later on, someone told me about a new thing called HTML, which was, as he described it, a derivative of SGML. Markup language enthusiasts were an occupational hazard at Interleaf and I ignored him, but this HTML thing later became a big part of my life.\\n\\nIn the fall of 1992 I moved back to Providence to continue at RISD. The foundation had merely been intro stuff, and the Accademia had been a (very civilized) joke. Now I was going to see what real art school was like. But alas it was more like the Accademia than not. Better organized, certainly, and a lot more expensive, but it was now becoming clear that art school did not bear the same relationship to art that medical school bore to medicine. At least not the painting department. The textile department, which my next door neighbor belonged to, seemed to be pretty rigorous. No doubt illustration and architecture were too. But painting was post-rigorous. Painting students were supposed to express themselves, which to the more worldly ones meant to try to cook up some sort of distinctive signature style.\\n\\nA signature style is the visual equivalent of what in show business is known as a \"schtick\": something that immediately identifies the work as yours and no one else\\'s. For example, when you see a painting that looks like a certain kind of cartoon, you know it\\'s by Roy Lichtenstein. So if you see a big painting of this type hanging in the apartment of a hedge fund manager, you know he paid millions of dollars for it. That\\'s not always why artists have a signature style, but it\\'s usually why buyers pay a lot for such work. [6]\\n\\nThere were plenty of earnest students too: kids who \"could draw\" in high school, and now had come to what was supposed to be the best art school in the country, to learn to draw even better. They tended to be confused and demoralized by what they found at RISD, but they kept going, because painting was what they did. I was not one of the kids who could draw in high school, but at RISD I was definitely closer to their tribe than the tribe of signature style seekers.\\n\\nI learned a lot in the color class I took at RISD, but otherwise I was basically teaching myself to paint, and I could do that for free. So in 1993 I dropped out. I hung around Providence for a bit, and then my college friend Nancy Parmet did me a big favor. A rent-controlled apartment in a building her mother owned in New York was becoming vacant. Did I want it? It wasn\\'t much more than my current place, and New York was supposed to be where the artists were. So yes, I wanted it! [7]\\n\\nAsterix comics begin by zooming in on a tiny corner of Roman Gaul that turns out not to be controlled by the Romans. You can do something similar on a map of New York City: if you zoom in on the Upper East Side, there\\'s a tiny corner that\\'s not rich, or at least wasn\\'t in 1993. It\\'s called Yorkville, and that was my new home. Now I was a New York artist — in the strictly technical sense of making paintings and living in New York.\\n\\nI was nervous about money, because I could sense that Interleaf was on the way down. Freelance Lisp hacking work was very rare, and I didn\\'t want to have to program in another language, which in those days would have meant C++ if I was lucky. So with my unerring nose for financial opportunity, I decided to write another book on Lisp. This would be a popular book, the sort of book that could be used as a textbook. I imagined myself living frugally off the royalties and spending all my time painting. (The painting on the cover of this book, ANSI Common Lisp, is one that I painted around this time.)\\n\\nThe best thing about New York for me was the presence of Idelle and Julian Weber. Idelle Weber was a painter, one of the early photorealists, and I\\'d taken her painting class at Harvard. I\\'ve never known a teacher more beloved by her students. Large numbers of former students kept in touch with her, including me. After I moved to New York I became her de facto studio assistant.\\n\\nShe liked to paint on big, square canvases, 4 to 5 feet on a side. One day in late 1994 as I was stretching one of these monsters there was something on the radio about a famous fund manager. He wasn\\'t that much older than me, and was super rich. The thought suddenly occurred to me: why don\\'t I become rich? Then I\\'ll be able to work on whatever I want.\\n\\nMeanwhile I\\'d been hearing more and more about this new thing called the World Wide Web. Robert Morris showed it to me when I visited him in Cambridge, where he was now in grad school at Harvard. It seemed to me that the web would be a big deal. I\\'d seen what graphical user interfaces had done for the popularity of microcomputers. It seemed like the web would do the same for the internet.\\n\\nIf I wanted to get rich, here was the next train leaving the station. I was right about that part. What I got wrong was the idea. I decided we should start a company to put art galleries online. I can\\'t honestly say, after reading so many Y Combinator applications, that this was the worst startup idea ever, but it was up there. Art galleries didn\\'t want to be online, and still don\\'t, not the fancy ones. That\\'s not how they sell. I wrote some software to generate web sites for galleries, and Robert wrote some to resize images and set up an http server to serve the pages. Then we tried to sign up galleries. To call this a difficult sale would be an understatement. It was difficult to give away. A few galleries let us make sites for them for free, but none paid us.\\n\\nThen some online stores started to appear, and I realized that except for the order buttons they were identical to the sites we\\'d been generating for galleries. This impressive-sounding thing called an \"internet storefront\" was something we already knew how to build.\\n\\nSo in the summer of 1995, after I submitted the camera-ready copy of ANSI Common Lisp to the publishers, we started trying to write software to build online stores. At first this was going to be normal desktop software, which in those days meant Windows software. That was an alarming prospect, because neither of us knew how to write Windows software or wanted to learn. We lived in the Unix world. But we decided we\\'d at least try writing a prototype store builder on Unix. Robert wrote a shopping cart, and I wrote a new site generator for stores — in Lisp, of course.\\n\\nWe were working out of Robert\\'s apartment in Cambridge. His roommate was away for big chunks of time, during which I got to sleep in his room. For some reason there was no bed frame or sheets, just a mattress on the floor. One morning as I was lying on this mattress I had an idea that made me sit up like a capital L. What if we ran the software on the server, and let users control it by clicking on links? Then we\\'d never have to write anything to run on users\\' computers. We could generate the sites on the same server we\\'d serve them from. Users wouldn\\'t need anything more than a browser.\\n\\nThis kind of software, known as a web app, is common now, but at the time it wasn\\'t clear that it was even possible. To find out, we decided to try making a version of our store builder that you could control through the browser. A couple days later, on August 12, we had one that worked. The UI was horrible, but it proved you could build a whole store through the browser, without any client software or typing anything into the command line on the server.\\n\\nNow we felt like we were really onto something. I had visions of a whole new generation of software working this way. You wouldn\\'t need versions, or ports, or any of that crap. At Interleaf there had been a whole group called Release Engineering that seemed to be at least as big as the group that actually wrote the software. Now you could just update the software right on the server.\\n\\nWe started a new company we called Viaweb, after the fact that our software worked via the web, and we got $10,000 in seed funding from Idelle\\'s husband Julian. In return for that and doing the initial legal work and giving us business advice, we gave him 10% of the company. Ten years later this deal became the model for Y Combinator\\'s. We knew founders needed something like this, because we\\'d needed it ourselves.\\n\\nAt this stage I had a negative net worth, because the thousand dollars or so I had in the bank was more than counterbalanced by what I owed the government in taxes. (Had I diligently set aside the proper proportion of the money I\\'d made consulting for Interleaf? No, I had not.) So although Robert had his graduate student stipend, I needed that seed funding to live on.', metadata={'source': 'https://raw.githubusercontent.com/run-llama/llama_index/main/examples/paul_graham_essay/data/paul_graham_essay.txt'}), Document(page_content='What I Worked On\\n\\nFebruary 2021\\n\\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn\\'t write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\\n\\nThe first programs I tried writing were on the IBM 1401 that our school district used for what was then called \"data processing.\" This was in 9th grade, so I was 13 or 14. The school district\\'s 1401 happened to be in the basement of our junior high school, and my friend Rich Draves and I got permission to use it. It was like a mini Bond villain\\'s lair down there, with all these alien-looking machines — CPU, disk drives, printer, card reader — sitting up on a raised floor under bright fluorescent lights.\\n\\nThe language we used was an early version of Fortran. You had to type programs on punch cards, then stack them in the card reader and press a button to load the program into memory and run it. The result would ordinarily be to print something on the spectacularly loud printer.\\n\\nI was puzzled by the 1401. I couldn\\'t figure out what to do with it. And in retrospect there\\'s not much I could have done with it. The only form of input to programs was data stored on punched cards, and I didn\\'t have any data stored on punched cards. The only other option was to do things that didn\\'t rely on any input, like calculate approximations of pi, but I didn\\'t know enough math to do anything interesting of that type. So I\\'m not surprised I can\\'t remember any programs I wrote, because they can\\'t have done much. My clearest memory is of the moment I learned it was possible for programs not to terminate, when one of mine didn\\'t. On a machine without time-sharing, this was a social as well as a technical error, as the data center manager\\'s expression made clear.\\n\\nWith microcomputers, everything changed. Now you could have a computer sitting right in front of you, on a desk, that could respond to your keystrokes as it was running instead of just churning through a stack of punch cards and then stopping. [1]\\n\\nThe first of my friends to get a microcomputer built it himself. It was sold as a kit by Heathkit. I remember vividly how impressed and envious I felt watching him sitting in front of it, typing programs right into the computer.\\n\\nComputers were expensive in those days and it took me years of nagging before I convinced my father to buy one, a TRS-80, in about 1980. The gold standard then was the Apple II, but a TRS-80 was good enough. This was when I really started programming. I wrote simple games, a program to predict how high my model rockets would fly, and a word processor that my father used to write at least one book. There was only room in memory for about 2 pages of text, so he\\'d write 2 pages at a time and then print them out, but it was a lot better than a typewriter.\\n\\nThough I liked programming, I didn\\'t plan to study it in college. In college I was going to study philosophy, which sounded much more powerful. It seemed, to my naive high school self, to be the study of the ultimate truths, compared to which the things studied in other fields would be mere domain knowledge. What I discovered when I got to college was that the other fields took up so much of the space of ideas that there wasn\\'t much left for these supposed ultimate truths. All that seemed left for philosophy were edge cases that people in other fields felt could safely be ignored.\\n\\nI couldn\\'t have put this into words when I was 18. All I knew at the time was that I kept taking philosophy courses and they kept being boring. So I decided to switch to AI.\\n\\nAI was in the air in the mid 1980s, but there were two things especially that made me want to work on it: a novel by Heinlein called The Moon is a Harsh Mistress, which featured an intelligent computer called Mike, and a PBS documentary that showed Terry Winograd using SHRDLU. I haven\\'t tried rereading The Moon is a Harsh Mistress, so I don\\'t know how well it has aged, but when I read it I was drawn entirely into its world. It seemed only a matter of time before we\\'d have Mike, and when I saw Winograd using SHRDLU, it seemed like that time would be a few years at most. All you had to do was teach SHRDLU more words.\\n\\nThere weren\\'t any classes in AI at Cornell then, not even graduate classes, so I started trying to teach myself. Which meant learning Lisp, since in those days Lisp was regarded as the language of AI. The commonly used programming languages then were pretty primitive, and programmers\\' ideas correspondingly so. The default language at Cornell was a Pascal-like language called PL/I, and the situation was similar elsewhere. Learning Lisp expanded my concept of a program so fast that it was years before I started to have a sense of where the new limits were. This was more like it; this was what I had expected college to do. It wasn\\'t happening in a class, like it was supposed to, but that was ok. For the next couple years I was on a roll. I knew what I was going to do.\\n\\nFor my undergraduate thesis, I reverse-engineered SHRDLU. My God did I love working on that program. It was a pleasing bit of code, but what made it even more exciting was my belief — hard to imagine now, but not unique in 1985 — that it was already climbing the lower slopes of intelligence.\\n\\nI had gotten into a program at Cornell that didn\\'t make you choose a major. You could take whatever classes you liked, and choose whatever you liked to put on your degree. I of course chose \"Artificial Intelligence.\" When I got the actual physical diploma, I was dismayed to find that the quotes had been included, which made them read as scare-quotes. At the time this bothered me, but now it seems amusingly accurate, for reasons I was about to discover.\\n\\nI applied to 3 grad schools: MIT and Yale, which were renowned for AI at the time, and Harvard, which I\\'d visited because Rich Draves went there, and was also home to Bill Woods, who\\'d invented the type of parser I used in my SHRDLU clone. Only Harvard accepted me, so that was where I went.\\n\\nI don\\'t remember the moment it happened, or if there even was a specific moment, but during the first year of grad school I realized that AI, as practiced at the time, was a hoax. By which I mean the sort of AI in which a program that\\'s told \"the dog is sitting on the chair\" translates this into some formal representation and adds it to the list of things it knows.\\n\\nWhat these programs really showed was that there\\'s a subset of natural language that\\'s a formal language. But a very proper subset. It was clear that there was an unbridgeable gap between what they could do and actually understanding natural language. It was not, in fact, simply a matter of teaching SHRDLU more words. That whole way of doing AI, with explicit data structures representing concepts, was not going to work. Its brokenness did, as so often happens, generate a lot of opportunities to write papers about various band-aids that could be applied to it, but it was never going to get us Mike.\\n\\nSo I looked around to see what I could salvage from the wreckage of my plans, and there was Lisp. I knew from experience that Lisp was interesting for its own sake and not just for its association with AI, even though that was the main reason people cared about it at the time. So I decided to focus on Lisp. In fact, I decided to write a book about Lisp hacking. It\\'s scary to think how little I knew about Lisp hacking when I started writing that book. But there\\'s nothing like writing a book about something to help you learn it. The book, On Lisp, wasn\\'t published till 1993, but I wrote much of it in grad school.\\n\\nComputer Science is an uneasy alliance between two halves, theory and systems. The theory people prove things, and the systems people build things. I wanted to build things. I had plenty of respect for theory — indeed, a sneaking suspicion that it was the more admirable of the two halves — but building things seemed so much more exciting.\\n\\nThe problem with systems work, though, was that it didn\\'t last. Any program you wrote today, no matter how good, would be obsolete in a couple decades at best. People might mention your software in footnotes, but no one would actually use it. And indeed, it would seem very feeble work. Only people with a sense of the history of the field would even realize that, in its time, it had been good.\\n\\nThere were some surplus Xerox Dandelions floating around the computer lab at one point. Anyone who wanted one to play around with could have one. I was briefly tempted, but they were so slow by present standards; what was the point? No one else wanted one either, so off they went. That was what happened to systems work.\\n\\nI wanted not just to build things, but to build things that would last.\\n\\nIn this dissatisfied state I went in 1988 to visit Rich Draves at CMU, where he was in grad school. One day I went to visit the Carnegie Institute, where I\\'d spent a lot of time as a kid. While looking at a painting there I realized something that might seem obvious, but was a big surprise to me. There, right on the wall, was something you could make that would last. Paintings didn\\'t become obsolete. Some of the best ones were hundreds of years old.\\n\\nAnd moreover this was something you could make a living doing. Not as easily as you could by writing software, of course, but I thought if you were really industrious and lived really cheaply, it had to be possible to make enough to survive. And as an artist you could be truly independent. You wouldn\\'t have a boss, or even need to get research funding.', metadata={'source': 'https://raw.githubusercontent.com/run-llama/llama_index/main/examples/paul_graham_essay/data/paul_graham_essay.txt'}), Document(page_content='When I got back to New York I resumed my old life, except now I was rich. It was as weird as it sounds. I resumed all my old patterns, except now there were doors where there hadn\\'t been. Now when I was tired of walking, all I had to do was raise my hand, and (unless it was raining) a taxi would stop to pick me up. Now when I walked past charming little restaurants I could go in and order lunch. It was exciting for a while. Painting started to go better. I experimented with a new kind of still life where I\\'d paint one painting in the old way, then photograph it and print it, blown up, on canvas, and then use that as the underpainting for a second still life, painted from the same objects (which hopefully hadn\\'t rotted yet).\\n\\nMeanwhile I looked for an apartment to buy. Now I could actually choose what neighborhood to live in. Where, I asked myself and various real estate agents, is the Cambridge of New York? Aided by occasional visits to actual Cambridge, I gradually realized there wasn\\'t one. Huh.\\n\\nAround this time, in the spring of 2000, I had an idea. It was clear from our experience with Viaweb that web apps were the future. Why not build a web app for making web apps? Why not let people edit code on our server through the browser, and then host the resulting applications for them? [9] You could run all sorts of services on the servers that these applications could use just by making an API call: making and receiving phone calls, manipulating images, taking credit card payments, etc.\\n\\nI got so excited about this idea that I couldn\\'t think about anything else. It seemed obvious that this was the future. I didn\\'t particularly want to start another company, but it was clear that this idea would have to be embodied as one, so I decided to move to Cambridge and start it. I hoped to lure Robert into working on it with me, but there I ran into a hitch. Robert was now a postdoc at MIT, and though he\\'d made a lot of money the last time I\\'d lured him into working on one of my schemes, it had also been a huge time sink. So while he agreed that it sounded like a plausible idea, he firmly refused to work on it.\\n\\nHmph. Well, I\\'d do it myself then. I recruited Dan Giffin, who had worked for Viaweb, and two undergrads who wanted summer jobs, and we got to work trying to build what it\\'s now clear is about twenty companies and several open source projects worth of software. The language for defining applications would of course be a dialect of Lisp. But I wasn\\'t so naive as to assume I could spring an overt Lisp on a general audience; we\\'d hide the parentheses, like Dylan did.\\n\\nBy then there was a name for the kind of company Viaweb was, an \"application service provider,\" or ASP. This name didn\\'t last long before it was replaced by \"software as a service,\" but it was current for long enough that I named this new company after it: it was going to be called Aspra.\\n\\nI started working on the application builder, Dan worked on network infrastructure, and the two undergrads worked on the first two services (images and phone calls). But about halfway through the summer I realized I really didn\\'t want to run a company — especially not a big one, which it was looking like this would have to be. I\\'d only started Viaweb because I needed the money. Now that I didn\\'t need money anymore, why was I doing this? If this vision had to be realized as a company, then screw the vision. I\\'d build a subset that could be done as an open source project.\\n\\nMuch to my surprise, the time I spent working on this stuff was not wasted after all. After we started Y Combinator, I would often encounter startups working on parts of this new architecture, and it was very useful to have spent so much time thinking about it and even trying to write some of it.\\n\\nThe subset I would build as an open source project was the new Lisp, whose parentheses I now wouldn\\'t even have to hide. A lot of Lisp hackers dream of building a new Lisp, partly because one of the distinctive features of the language is that it has dialects, and partly, I think, because we have in our minds a Platonic form of Lisp that all existing dialects fall short of. I certainly did. So at the end of the summer Dan and I switched to working on this new dialect of Lisp, which I called Arc, in a house I bought in Cambridge.\\n\\nThe following spring, lightning struck. I was invited to give a talk at a Lisp conference, so I gave one about how we\\'d used Lisp at Viaweb. Afterward I put a postscript file of this talk online, on paulgraham.com, which I\\'d created years before using Viaweb but had never used for anything. In one day it got 30,000 page views. What on earth had happened? The referring urls showed that someone had posted it on Slashdot. [10]\\n\\nWow, I thought, there\\'s an audience. If I write something and put it on the web, anyone can read it. That may seem obvious now, but it was surprising then. In the print era there was a narrow channel to readers, guarded by fierce monsters known as editors. The only way to get an audience for anything you wrote was to get it published as a book, or in a newspaper or magazine. Now anyone could publish anything.\\n\\nThis had been possible in principle since 1993, but not many people had realized it yet. I had been intimately involved with building the infrastructure of the web for most of that time, and a writer as well, and it had taken me 8 years to realize it. Even then it took me several years to understand the implications. It meant there would be a whole new generation of essays. [11]\\n\\nIn the print era, the channel for publishing essays had been vanishingly small. Except for a few officially anointed thinkers who went to the right parties in New York, the only people allowed to publish essays were specialists writing about their specialties. There were so many essays that had never been written, because there had been no way to publish them. Now they could be, and I was going to write them. [12]\\n\\nI\\'ve worked on several different things, but to the extent there was a turning point where I figured out what to work on, it was when I started publishing essays online. From then on I knew that whatever else I did, I\\'d always write essays too.\\n\\nI knew that online essays would be a marginal medium at first. Socially they\\'d seem more like rants posted by nutjobs on their GeoCities sites than the genteel and beautifully typeset compositions published in The New Yorker. But by this point I knew enough to find that encouraging instead of discouraging.\\n\\nOne of the most conspicuous patterns I\\'ve noticed in my life is how well it has worked, for me at least, to work on things that weren\\'t prestigious. Still life has always been the least prestigious form of painting. Viaweb and Y Combinator both seemed lame when we started them. I still get the glassy eye from strangers when they ask what I\\'m writing, and I explain that it\\'s an essay I\\'m going to publish on my web site. Even Lisp, though prestigious intellectually in something like the way Latin is, also seems about as hip.\\n\\nIt\\'s not that unprestigious types of work are good per se. But when you find yourself drawn to some kind of work despite its current lack of prestige, it\\'s a sign both that there\\'s something real to be discovered there, and that you have the right kind of motives. Impure motives are a big danger for the ambitious. If anything is going to lead you astray, it will be the desire to impress people. So while working on things that aren\\'t prestigious doesn\\'t guarantee you\\'re on the right track, it at least guarantees you\\'re not on the most common type of wrong one.\\n\\nOver the next several years I wrote lots of essays about all kinds of different topics. O\\'Reilly reprinted a collection of them as a book, called Hackers & Painters after one of the essays in it. I also worked on spam filters, and did some more painting. I used to have dinners for a group of friends every thursday night, which taught me how to cook for groups. And I bought another building in Cambridge, a former candy factory (and later, twas said, porn studio), to use as an office.\\n\\nOne night in October 2003 there was a big party at my house. It was a clever idea of my friend Maria Daniels, who was one of the thursday diners. Three separate hosts would all invite their friends to one party. So for every guest, two thirds of the other guests would be people they didn\\'t know but would probably like. One of the guests was someone I didn\\'t know but would turn out to like a lot: a woman called Jessica Livingston. A couple days later I asked her out.\\n\\nJessica was in charge of marketing at a Boston investment bank. This bank thought it understood startups, but over the next year, as she met friends of mine from the startup world, she was surprised how different reality was. And how colorful their stories were. So she decided to compile a book of interviews with startup founders.\\n\\nWhen the bank had financial problems and she had to fire half her staff, she started looking for a new job. In early 2005 she interviewed for a marketing job at a Boston VC firm. It took them weeks to make up their minds, and during this time I started telling her about all the things that needed to be fixed about venture capital. They should make a larger number of smaller investments instead of a handful of giant ones, they should be funding younger, more technical founders instead of MBAs, they should let the founders remain as CEO, and so on.\\n\\nOne of my tricks for writing essays had always been to give talks. The prospect of having to stand up in front of a group of people and tell them something that won\\'t waste their time is a great spur to the imagination. When the Harvard Computer Society, the undergrad computer club, asked me to give a talk, I decided I would tell them how to start a startup. Maybe they\\'d be able to avoid the worst of the mistakes we\\'d made.', metadata={'source': 'https://raw.githubusercontent.com/run-llama/llama_index/main/examples/paul_graham_essay/data/paul_graham_essay.txt'})]\n",
            "[Document(page_content='How does code being edited by too many people lead to bugs?', metadata={'doc_id': 'ad5cb6e0-ac2f-4383-846c-b016fd31dd8c'}), Document(page_content='What programming language did the author use when working on the IBM 1401?', metadata={'doc_id': '8738bfa7-d26a-4195-800f-3be2666cc8b1'}), Document(page_content='What motivated the narrator to start a new company in the web app industry?', metadata={'doc_id': 'baaf3405-04d6-4a06-863e-719eec70567a'}), Document(page_content='Why did the author decide to focus on Lisp?', metadata={'doc_id': '8738bfa7-d26a-4195-800f-3be2666cc8b1'})]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import uuid\n",
        "\n",
        "from langchain.vectorstores import Chroma\n",
        "from langchain.storage import InMemoryStore\n",
        "from langchain.chat_models import ChatOpenAI\n",
        "from langchain.schema.document import Document\n",
        "from langchain.prompts import ChatPromptTemplate\n",
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "from langchain.document_loaders import WebBaseLoader\n",
        "from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain.output_parsers.openai_functions import (\n",
        "    JsonKeyOutputFunctionsParser\n",
        ")\n",
        "\n",
        "# 加载网页数据\n",
        "loader = WebBaseLoader(\n",
        "    'https://raw.githubusercontent.com/run-llama/llama_index/'\n",
        "    'main/examples/paul_graham_essay/data/paul_graham_essay.txt'\n",
        ")\n",
        "data = loader.load()\n",
        "\n",
        "# 切分文档块\n",
        "text_splitter = RecursiveCharacterTextSplitter(\n",
        "    chunk_size=10000\n",
        ")\n",
        "docs = text_splitter.split_documents(data)\n",
        "\n",
        "# 创建一个 function call 函数描述\n",
        "functions = [\n",
        "    {\n",
        "        'name': 'hypothetical_questions',\n",
        "        'description': 'Generate hypothetical questions',\n",
        "        'parameters': {\n",
        "            'type': 'object',\n",
        "            'properties': {\n",
        "                'questions': {\n",
        "                    'type': 'array',\n",
        "                    'items': {'type': 'string'},\n",
        "                },\n",
        "            },\n",
        "            'required': ['questions'],\n",
        "        },\n",
        "    }\n",
        "]\n",
        "chain = (\n",
        "    {'doc': lambda x: x.page_content}\n",
        "    | ChatPromptTemplate.from_template(\n",
        "        'Generate a list of 3 hypothetical questions that the '\n",
        "        'below document could be used to answer:\\n\\n{doc}'\n",
        "    )\n",
        "    | ChatOpenAI(max_retries=0).bind(\n",
        "        functions=functions,\n",
        "        function_call={'name': 'hypothetical_questions'}\n",
        "    )\n",
        "\n",
        ")\n",
        "\n",
        "questions = chain.batch(docs, {'max_concurrency': 5})\n",
        "print(questions)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R-o3hGJtJxk7",
        "outputId": "21078c0b-153b-4bfb-c6cf-3c5ad1529ab2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[AIMessage(content='', additional_kwargs={'function_call': {'name': 'hypothetical_questions', 'arguments': '{\\n  \"questions\": [\\n    \"What were some of the early programming experiences of the author?\",\\n    \"Why did the author decide to focus on Lisp?\",\\n    \"What made the author realize that AI, as practiced at the time, was a hoax?\"\\n  ]\\n}'}}), AIMessage(content='', additional_kwargs={'function_call': {'name': 'hypothetical_questions', 'arguments': '{\\n  \"questions\": [\\n    \"What motivated the author to take art classes at Harvard?\",\\n    \"How did the author manage to write their dissertation in such a short amount of time?\",\\n    \"What factors influenced the author\\'s decision to apply to art schools?\"\\n  ]\\n}'}}), AIMessage(content='', additional_kwargs={'function_call': {'name': 'hypothetical_questions', 'arguments': '{\\n  \"questions\": [\\n    \"What are the advantages and disadvantages of having technology companies run by product people rather than sales people?\",\\n    \"How does editing code by too many people lead to bugs?\",\\n    \"What are the implications of having cheap and depressing office space?\",\\n    \"What is the difference between planned meetings and corridor conversations?\",\\n    \"Why are big, bureaucratic customers considered a dangerous source of money?\",\\n    \"What are the optimal hours and places for hacking?\",\\n    \"What is the significance of being the \\'entry level\\' option in a market?\",\\n    \"How does having a signature style affect the value of an artist\\'s work?\",\\n    \"What is the relationship between art school and art?\",\\n    \"Why did the author decide to drop out of art school and move to New York?\",\\n    \"What opportunities did the author see in the World Wide Web?\",\\n    \"Why did the author\\'s initial idea of putting art galleries online not work?\",\\n    \"How did the author come up with the idea for a web app?\",\\n    \"What were the advantages of building a web app instead of desktop software?\",\\n    \"What were the implications of being able to update software directly on the server?\"\\n  ]\\n}'}}), AIMessage(content='', additional_kwargs={'function_call': {'name': 'hypothetical_questions', 'arguments': '{\\n  \"questions\": [\\n    \"What were the main challenges faced while developing the WYSIWYG site builder?\",\\n    \"How did studying art contribute to the success of the online store builder?\",\\n    \"What were the key factors that led to the early success of the online store builder?\"\\n  ]\\n}'}}), AIMessage(content='', additional_kwargs={'function_call': {'name': 'hypothetical_questions', 'arguments': '{\\n  \"questions\": [\\n    \"What are the advantages and disadvantages of living a wealthy lifestyle?\",\\n    \"How can web apps revolutionize the way businesses operate?\",\\n    \"What are the challenges and rewards of starting a company?\"\\n  ]\\n}'}}), AIMessage(content='', additional_kwargs={'function_call': {'name': 'hypothetical_questions', 'arguments': '{\\n  \"questions\": [\\n    \"What were the challenges faced by the speaker in starting their own investment firm?\",\\n    \"How did the batch model of funding startups help solve the problem of isolation for founders?\",\\n    \"What were the advantages of scale that Y Combinator noticed as it grew?\"\\n  ]\\n}'}}), AIMessage(content='', additional_kwargs={'function_call': {'name': 'hypothetical_questions', 'arguments': '{\\n  \"questions\": [\\n    \"What was the significance of Robert Morris offering unsolicited advice to the author?\",\\n    \"Why did the author decide to hand over Y Combinator to someone else?\",\\n    \"What motivated the author to start painting after leaving Y Combinator?\"\\n  ]\\n}'}}), AIMessage(content='', additional_kwargs={'function_call': {'name': 'hypothetical_questions', 'arguments': '{\\n  \"questions\": [\\n    \"How does living in Florence at street level in different conditions impact one\\'s perception of the city?\",\\n    \"What are the advantages of launching a software privately before launching it publicly?\",\\n    \"How does the customs and practices in a field affect the ability to adapt to rapid change?\"\\n  ]\\n}'}})]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 自查询检索器"
      ],
      "metadata": {
        "id": "ynD_rZkT4bCy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install lark chromadb tiktoken"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-SQ374sGVo2P",
        "outputId": "a4d01500-f204-4dc4-a639-844eafcefaa0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: lark in /usr/local/lib/python3.10/dist-packages (1.1.8)\n",
            "Requirement already satisfied: chromadb in /usr/local/lib/python3.10/dist-packages (0.4.15)\n",
            "Collecting tiktoken\n",
            "  Downloading tiktoken-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m24.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.28 in /usr/local/lib/python3.10/dist-packages (from chromadb) (2.31.0)\n",
            "Requirement already satisfied: pydantic>=1.9 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.10.13)\n",
            "Requirement already satisfied: chroma-hnswlib==0.7.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.7.3)\n",
            "Requirement already satisfied: fastapi>=0.95.2 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.104.1)\n",
            "Requirement already satisfied: uvicorn[standard]>=0.18.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.23.2)\n",
            "Requirement already satisfied: posthog>=2.4.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (3.0.2)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.8.0)\n",
            "Requirement already satisfied: pulsar-client>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (3.3.0)\n",
            "Requirement already satisfied: onnxruntime>=1.14.1 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.16.1)\n",
            "Requirement already satisfied: opentelemetry-api>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.20.0)\n",
            "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.20.0)\n",
            "Requirement already satisfied: opentelemetry-sdk>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.20.0)\n",
            "Requirement already satisfied: tokenizers>=0.13.2 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.14.1)\n",
            "Requirement already satisfied: pypika>=0.48.9 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.48.9)\n",
            "Requirement already satisfied: tqdm>=4.65.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.66.1)\n",
            "Requirement already satisfied: overrides>=7.3.1 in /usr/local/lib/python3.10/dist-packages (from chromadb) (7.4.0)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.10/dist-packages (from chromadb) (6.1.0)\n",
            "Requirement already satisfied: grpcio>=1.58.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.59.0)\n",
            "Requirement already satisfied: bcrypt>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.0.1)\n",
            "Requirement already satisfied: typer>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.9.0)\n",
            "Requirement already satisfied: kubernetes>=28.1.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (28.1.0)\n",
            "Requirement already satisfied: tenacity>=8.2.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (8.2.3)\n",
            "Requirement already satisfied: numpy>=1.22.5 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.26.1)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2023.6.3)\n",
            "Requirement already satisfied: anyio<4.0.0,>=3.7.1 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb) (3.7.1)\n",
            "Requirement already satisfied: starlette<0.28.0,>=0.27.0 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb) (0.27.0)\n",
            "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2023.7.22)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.16.0)\n",
            "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.8.2)\n",
            "Requirement already satisfied: pyyaml>=5.4.1 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (6.0.1)\n",
            "Requirement already satisfied: google-auth>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.17.3)\n",
            "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.6.4)\n",
            "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.3.1)\n",
            "Requirement already satisfied: oauthlib>=3.2.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\n",
            "Requirement already satisfied: urllib3<2.0,>=1.24.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.26.18)\n",
            "Requirement already satisfied: coloredlogs in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (15.0.1)\n",
            "Requirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (23.5.26)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (23.2)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (3.20.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (1.12)\n",
            "Requirement already satisfied: deprecated>=1.2.6 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-api>=1.2.0->chromadb) (1.2.14)\n",
            "Requirement already satisfied: importlib-metadata<7.0,>=6.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-api>=1.2.0->chromadb) (6.8.0)\n",
            "Requirement already satisfied: backoff<3.0.0,>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (2.2.1)\n",
            "Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.61.0)\n",
            "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.20.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.20.0)\n",
            "Requirement already satisfied: opentelemetry-proto==1.20.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.20.0)\n",
            "Requirement already satisfied: opentelemetry-semantic-conventions==0.41b0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-sdk>=1.2.0->chromadb) (0.41b0)\n",
            "Requirement already satisfied: monotonic>=1.5 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb) (1.6)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.3.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.4)\n",
            "Requirement already satisfied: huggingface_hub<0.18,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers>=0.13.2->chromadb) (0.17.3)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb) (8.1.7)\n",
            "Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.14.0)\n",
            "Requirement already satisfied: httptools>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.6.1)\n",
            "Requirement already satisfied: python-dotenv>=0.13 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.0.0)\n",
            "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.19.0)\n",
            "Requirement already satisfied: watchfiles>=0.13 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.21.0)\n",
            "Requirement already satisfied: websockets>=10.4 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (12.0)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<4.0.0,>=3.7.1->fastapi>=0.95.2->chromadb) (1.3.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4.0.0,>=3.7.1->fastapi>=0.95.2->chromadb) (1.1.3)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated>=1.2.6->opentelemetry-api>=1.2.0->chromadb) (1.14.1)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (5.3.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<0.18,>=0.16.4->tokenizers>=0.13.2->chromadb) (3.12.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface_hub<0.18,>=0.16.4->tokenizers>=0.13.2->chromadb) (2023.6.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.17.0)\n",
            "Requirement already satisfied: humanfriendly>=9.1 in /usr/local/lib/python3.10/dist-packages (from coloredlogs->onnxruntime>=1.14.1->chromadb) (10.0)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.5.0)\n",
            "Installing collected packages: tiktoken\n",
            "Successfully installed tiktoken-0.5.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.schema import Document\n",
        "from langchain.vectorstores import Chroma\n",
        "from langchain.chat_models import ChatOpenAI\n",
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "from langchain.chains.query_constructor.base import AttributeInfo\n",
        "from langchain.retrievers.self_query.base import SelfQueryRetriever\n",
        "\n",
        "# 创建文档块\n",
        "docs = [\n",
        "    Document(\n",
        "        page_content=\"A bunch of scientists bring back dinosaurs \"\n",
        "                     \"and mayhem breaks loose\",\n",
        "        metadata={\"year\": 1993, \"rating\": 7.7, \"genre\": \"science fiction\"},\n",
        "    ),\n",
        "    Document(\n",
        "        page_content=\"Leo DiCaprio gets lost in a dream within a dream \"\n",
        "                     \"within a dream within a ...\",\n",
        "        metadata={\"year\": 2010, \"director\": \"Christopher Nolan\",\n",
        "                  \"rating\": 8.2},\n",
        "    ),\n",
        "    Document(\n",
        "        page_content=\"A psychologist / detective gets lost in a series \"\n",
        "                     \"of dreams within dreams within dreams and \"\n",
        "                     \"Inception reused the idea\",\n",
        "        metadata={\"year\": 2006, \"director\": \"Satoshi Kon\", \"rating\": 8.6},\n",
        "    ),\n",
        "    Document(\n",
        "        page_content=\"A bunch of normal-sized women are supremely \"\n",
        "                     \"wholesome and some men pine after them\",\n",
        "        metadata={\"year\": 2019, \"director\": \"Greta Gerwig\", \"rating\": 8.3},\n",
        "    ),\n",
        "    Document(\n",
        "        page_content=\"Toys come alive and have a blast doing so\",\n",
        "        metadata={\"year\": 1995, \"genre\": \"animated\"},\n",
        "    ),\n",
        "    Document(\n",
        "        page_content=\"Three men walk into the Zone, three men \"\n",
        "                     \"walk out of the Zone\",\n",
        "        metadata={\n",
        "            \"year\": 1979,\n",
        "            \"rating\": 9.9,\n",
        "            \"director\": \"Andrei Tarkovsky\",\n",
        "            \"genre\": \"thriller\",\n",
        "        },\n",
        "    ),\n",
        "]\n",
        "# 将文档块存入 Chroma 中\n",
        "vectorstore = Chroma.from_documents(docs, OpenAIEmbeddings())\n",
        "\n",
        "# 构建用于描述 metadata 中各字段属性的 AttributeInfo\n",
        "metadata_field_info = [\n",
        "    AttributeInfo(\n",
        "        name=\"genre\",\n",
        "        description=\"The genre of the movie. One of \"\n",
        "                    \"['science fiction', 'comedy', 'drama', \"\n",
        "                    \"'thriller', 'romance', 'action', 'animated']\",\n",
        "        type=\"string\",\n",
        "    ),\n",
        "    AttributeInfo(\n",
        "        name=\"year\",\n",
        "        description=\"The year the movie was released\",\n",
        "        type=\"integer\",\n",
        "    ),\n",
        "    AttributeInfo(\n",
        "        name=\"director\",\n",
        "        description=\"The name of the movie director\",\n",
        "        type=\"string\",\n",
        "    ),\n",
        "    AttributeInfo(\n",
        "        name=\"rating\", description=\"A 1-10 rating for the movie\",\n",
        "        type=\"float\"\n",
        "    ),\n",
        "]\n",
        "\n",
        "# 构建自查询检索器\n",
        "document_content_description = \"Brief summary of a movie\"\n",
        "llm = ChatOpenAI(temperature=0)\n",
        "retriever = SelfQueryRetriever.from_llm(\n",
        "    llm,\n",
        "    vectorstore,\n",
        "    document_content_description,\n",
        "    metadata_field_info,\n",
        ")\n",
        "\n",
        "# 然后我们就可以使用 get_relevant_documents 进行查询\n",
        "docs = retriever.get_relevant_documents(\n",
        "    'What are movies about dinosaurs')\n",
        "print(docs)\n",
        "\n",
        "# 这个查询会结合 metadata 中的值进行查询\n",
        "docs = retriever.get_relevant_documents(\n",
        "    \"I want to watch a movie rated higher than 8.5\")\n",
        "print(docs)\n",
        "\n",
        "# 当 SelfQueryRetriever.from_llm 的 enable_limit=True 时，\n",
        "# 我们可以在查询语句中设置要返回的条数\n",
        "retriever = SelfQueryRetriever.from_llm(\n",
        "    llm,\n",
        "    vectorstore,\n",
        "    document_content_description,\n",
        "    metadata_field_info,\n",
        "    enable_limit=True,\n",
        ")\n",
        "docs = retriever.get_relevant_documents(\n",
        "    'What are two movies about dinosaurs')\n",
        "print(docs)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KTxLjCxTVnsg",
        "outputId": "6946b40a-7df6-4519-84ed-3576900b15a9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Document(page_content='A bunch of scientists bring back dinosaurs and mayhem breaks loose', metadata={'genre': 'science fiction', 'rating': 7.7, 'year': 1993}), Document(page_content='A bunch of scientists bring back dinosaurs and mayhem breaks loose', metadata={'genre': 'science fiction', 'rating': 7.7, 'year': 1993}), Document(page_content='A bunch of scientists bring back dinosaurs and mayhem breaks loose', metadata={'genre': 'science fiction', 'rating': 7.7, 'year': 1993}), Document(page_content='Toys come alive and have a blast doing so', metadata={'genre': 'animated', 'year': 1995})]\n",
            "[Document(page_content='Three men walk into the Zone, three men walk out of the Zone', metadata={'director': 'Andrei Tarkovsky', 'genre': 'thriller', 'rating': 9.9, 'year': 1979}), Document(page_content='Three men walk into the Zone, three men walk out of the Zone', metadata={'director': 'Andrei Tarkovsky', 'genre': 'thriller', 'rating': 9.9, 'year': 1979}), Document(page_content='Three men walk into the Zone, three men walk out of the Zone', metadata={'director': 'Andrei Tarkovsky', 'genre': 'thriller', 'rating': 9.9, 'year': 1979}), Document(page_content='A psychologist / detective gets lost in a series of dreams within dreams within dreams and Inception reused the idea', metadata={'director': 'Satoshi Kon', 'rating': 8.6, 'year': 2006})]\n",
            "[Document(page_content='A bunch of scientists bring back dinosaurs and mayhem breaks loose', metadata={'genre': 'science fiction', 'rating': 7.7, 'year': 1993}), Document(page_content='A bunch of scientists bring back dinosaurs and mayhem breaks loose', metadata={'genre': 'science fiction', 'rating': 7.7, 'year': 1993})]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 检索内容重排"
      ],
      "metadata": {
        "id": "1RnwPuE34gMh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.vectorstores import Chroma\n",
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "from langchain.document_transformers import LongContextReorder\n",
        "\n",
        "texts = [\n",
        "    \"Basquetball is a great sport.\",\n",
        "    \"Fly me to the moon is one of my favourite songs.\",\n",
        "    \"The Celtics are my favourite team.\",\n",
        "    \"This is a document about the Boston Celtics\",\n",
        "    \"I simply love going to the movies\",\n",
        "    \"L. Kornet is one of the best Celtics players.\",\n",
        "    \"Elden Ring is one of the best games in the last 15 years.\",\n",
        "    \"The Boston Celtics won the game by 20 points\",\n",
        "]\n",
        "\n",
        "# 创建检索器\n",
        "retriever = Chroma.from_texts(\n",
        "    texts,\n",
        "    embedding=OpenAIEmbeddings()\n",
        ").as_retriever(\n",
        "    search_kwargs={\"k\": 6}\n",
        ")\n",
        "query = \"What can you tell me about the Celtics?\"\n",
        "\n",
        "# 查询相关性\n",
        "docs = retriever.get_relevant_documents(query)\n",
        "print(f'before:\\n')\n",
        "for doc in docs:\n",
        "    print(doc)\n",
        "\n",
        "# 初始化 LongContextReorder 加载搜索到的文档，并重新排序\n",
        "reordering = LongContextReorder()\n",
        "reordered_docs = reordering.transform_documents(docs)\n",
        "\n",
        "print(f'after:\\n')\n",
        "for doc in reordered_docs:\n",
        "    print(doc)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IiQP04wpNWP-",
        "outputId": "044ef37c-46c3-4d16-ac25-c6794963aae3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "before:\n",
            "\n",
            "page_content='The Celtics are my favourite team.'\n",
            "page_content='This is a document about the Boston Celtics'\n",
            "page_content='The Boston Celtics won the game by 20 points'\n",
            "page_content='L. Kornet is one of the best Celtics players.'\n",
            "page_content='Basquetball is a great sport.'\n",
            "page_content='I simply love going to the movies'\n",
            "after:\n",
            "\n",
            "page_content='This is a document about the Boston Celtics'\n",
            "page_content='L. Kornet is one of the best Celtics players.'\n",
            "page_content='I simply love going to the movies'\n",
            "page_content='Basquetball is a great sport.'\n",
            "page_content='The Boston Celtics won the game by 20 points'\n",
            "page_content='The Celtics are my favourite team.'\n"
          ]
        }
      ]
    }
  ]
}