From 08ce9d894e32c9993dd44189f3f9af73927a45a0 Mon Sep 17 00:00:00 2001 From: seref Date: Wed, 19 Jul 2023 00:13:51 +0300 Subject: [PATCH 01/37] Initial commit --- LICENSE | 21 +++++++++++++++++++++ README.md | 1 + 2 files changed, 22 insertions(+) create mode 100644 LICENSE create mode 100644 README.md diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..e56e28bc --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 indexas + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000..d9e9f531 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# llm-indexer \ No newline at end of file From 4429a21245d7213d2504dc97b2464d082b2d4a3b Mon Sep 17 00:00:00 2001 From: serafettin Date: Tue, 18 Jul 2023 23:52:33 +0200 Subject: [PATCH 02/37] initial commit --- .github/workflows/build.yaml | 64 ++++++++++++ .gitignore | 6 ++ Dockerfile | 6 ++ TODO.md | 17 ++++ app/__init__.py | 0 app/main.py | 182 +++++++++++++++++++++++++++++++++++ requirements.txt | 126 ++++++++++++++++++++++++ 7 files changed, 401 insertions(+) create mode 100644 .github/workflows/build.yaml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 TODO.md create mode 100644 app/__init__.py create mode 100644 app/main.py create mode 100644 requirements.txt diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 00000000..6515ab85 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,64 @@ +name: deploy +on: + push: + branches: + - dev + - testnet +jobs: + main: + runs-on: ubuntu-latest + steps: + + - name: Install kubectl + uses: azure/setup-kubectl@v2.0 + with: + version: 'v1.23.6' # default is latest stable + id: install + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - run: |- + aws eks --region us-east-1 update-kubeconfig --name indexas + + - name: Check k8s connection + run: kubectl get pods + + - name: Store build time + id: build-time + shell: bash + run: >- + echo "::set-output name=time::$(date +%s)" + + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Build, tag, and push image to Amazon ECR + env: + DOCKER_TAG: indexas/llm-indexer:${{ steps.build-time.outputs.time }} + DOCKER_REGISTRY: 534970752686.dkr.ecr.us-east-1.amazonaws.com + run: | + docker build -t $DOCKER_TAG . + docker tag $DOCKER_TAG $DOCKER_REGISTRY/$DOCKER_TAG + docker push $DOCKER_REGISTRY/$DOCKER_TAG + docker tag $DOCKER_TAG $DOCKER_REGISTRY/indexas/llm-indexer:latest-${GITHUB_REF#refs/heads/} + docker push $DOCKER_REGISTRY/indexas/llm-indexer:latest-${GITHUB_REF#refs/heads/} + + - name: Deploy + run: |- + kubectl set image deployment/api api=534970752686.dkr.ecr.us-east-1.amazonaws.com/indexas/llm-indexer:${{ steps.build-time.outputs.time }} --namespace web3-${GITHUB_REF#refs/heads/} + kubectl set image deployment/consumer consumer=534970752686.dkr.ecr.us-east-1.amazonaws.com/indexas/llm-indexer:${{ steps.build-time.outputs.time }} --namespace web3-${GITHUB_REF#refs/heads/} diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..7478b2ca --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.chroma/* +chroma-indexes/* +.idea/* +__pycache__/* +.DS_Store +.env \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..c42fc7f5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.11.3 +WORKDIR /code +COPY ./requirements.txt /code/requirements.txt +RUN pip3 install numpy==1.23.5 fastapi pydantic llama_index chromadb==0.3.26 openai redis --no-cache +COPY ./app /code/app +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"] diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..cf6c94d4 --- /dev/null +++ b/TODO.md @@ -0,0 +1,17 @@ +[x] Add link +[x] Get sources +[x] Get answer +[] Frontend Integration +- [x] Chat +- [x] LLamaIndex & NextJS Chat Integration +- [x] Conversation history +- [] per session hash +- [] chat ui with index_id +- [x] chat api +- [] indexer deploy +[] Bonus +- [x] Composability +- [] Huggingface Models +[] Others +- [] Remove link +- [] Update link \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..4089225a --- /dev/null +++ b/app/main.py @@ -0,0 +1,182 @@ +import os +import json + +from dotenv import load_dotenv + +from fastapi import FastAPI +from fastapi import Request +from fastapi.responses import JSONResponse +from fastapi.responses import StreamingResponse +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel + + +from llama_index import download_loader +from llama_index.llms import ChatMessage +from llama_index.vector_stores import ChromaVectorStore +from llama_index.embeddings.openai import OpenAIEmbedding +from llama_index import VectorStoreIndex, ListIndex, LLMPredictor, ServiceContext +from llama_index.chat_engine.types import ChatMode +from llama_index.indices.composability import ComposableGraph +from langchain.chat_models import ChatOpenAI + +import chromadb +from chromadb.config import Settings + +import openai # + +import redis + +redisClient = redis.Redis(host='localhost', port=6379, db=0) + +load_dotenv() + +origins = [ + "http://localhost", + "http://localhost:3000", + "http://localhost:8000", +] + +app = FastAPI() +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +openai.api_key = os.environ["OPENAI_API_KEY"] + +local_directory = "chroma-indexes" +persist_directory = os.path.join(os.getcwd(), local_directory) + +chroma_client = chromadb.Client(Settings( + chroma_db_impl="duckdb+parquet", + persist_directory=persist_directory +)) + +llm = ChatOpenAI(temperature=0, model_name="gpt-3.5-turbo", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) +llm_predictor = LLMPredictor(llm=llm) +service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor) + + +def get_index(index_id: str): + collection = chroma_client.get_or_create_collection(name="index-" + index_id) + vector_store = ChromaVectorStore(chroma_collection=collection) + index = VectorStoreIndex.from_vector_store(vector_store=vector_store, service_context=service_context) + return index + +class ChatHistory(BaseModel): + messages: list[ChatMessage] + +class Prompt(BaseModel): + prompt: str + +class Link(BaseModel): + url: str + +class Composition(BaseModel): + did: str + prompt: str + +@app.post("/index/{index_id}/links") +def add(index_id, link: Link): + + index = get_index(index_id=index_id) + + UnstructuredURLLoader = download_loader("UnstructuredURLLoader") + loader = UnstructuredURLLoader(urls=[link.url]) + kb_data = loader.load() + + index.insert(kb_data[0]) + chroma_client.persist() + + summary = index.as_query_engine().query("summarize").response + x = redisClient.hset("summaries", index_id, summary) + print(x) + + return JSONResponse(content={'message': 'Document added successfully'}) + +@app.delete("/index/{index_id}/links") +def remove(index_id: str, link: Link): + return JSONResponse(content={"message": "Documents deleted successfully"}) + + +@app.post("/index/{index_id}/prompt") +async def query(index_id, prompt: Prompt): + index = get_index(index_id=index_id) + response = index.as_query_engine(streaming=True).query(prompt.prompt) + # return JSONResponse(content={ + # "response": response.response, + # "sources": [{"id": s.node.id_, "url": s.node.metadata.get("source")} for s in response.source_nodes] + # }) + return StreamingResponse(response.response_gen, media_type='text/event-stream') + +@app.post("/index/{index_id}/chat_stream") +async def chat_stream(index_id, chat_history: ChatHistory): + + index = get_index(index_id=index_id) + chat_engine = index.as_chat_engine(streaming=True, verbose=True) + + messages = chat_history.messages + last_message = messages[-1] + print(last_message.content, messages) + # TODO Pop last "user" message from chat history. + streaming_response = chat_engine.stream_chat(message=last_message.content, chat_history=messages) + + def response_generator(): + yield json.dumps({ + "sources": [{"id": s.node.id_, "url": s.node.metadata.get("source")} for s in streaming_response.source_nodes] + }) + yield "\n ###endjson### \n\n" + for text in streaming_response.response_gen: + yield text + return StreamingResponse(response_generator(), media_type='text/event-stream') + + +@app.post("/index/{index_id}/chat") +async def chat(index_id, chat_history: ChatHistory): + + index = get_index(index_id=index_id) + chat_engine = index.as_chat_engine() + + messages = chat_history.messages + last_message = messages[-1] + + response = chat_engine.chat(message=last_message.content, chat_history=messages) + + return JSONResponse(content={ + "response": response.response, + "sources": [{"id": s.node.id_, "url": s.node.metadata.get("source"), "index_id": s.node.metadata.get("index_id")} for s in response.source_nodes] + }) + + +@app.post("/compose") +async def compose(c: Composition): + + + id_resp = redisClient.hkeys("user_indexes:by_did:" + c.did.lower()) + index_ids = [item.rstrip(":my_indexes") for item in id_resp] + + indexes = list(map(lambda index_id: get_index(index_id=index_id), index_ids)) + summaries = redisClient.hmget("summaries", index_ids) + + graph = ComposableGraph.from_indices( + ListIndex, + indexes, + index_summaries=summaries, + max_keywords_per_chunk=2000, + ) + query_engine = graph.as_query_engine() + response = query_engine.query(c.prompt) + return JSONResponse(content={ + "response": response.response, + "sources": [{"id": s.node.id_, "url": s.node.metadata.get("source"), "index_id": s.node.metadata.get("index_id")} for s in response.source_nodes] + }) + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..8f9d66e5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,126 @@ +aiohttp==3.8.4 +aiosignal==1.3.1 +annotated-types==0.5.0 +anyio==3.7.0 +argilla==1.12.0 +async-timeout==4.0.2 +attrs==23.1.0 +backoff==2.2.1 +beautifulsoup4==4.12.2 +blinker==1.6.2 +certifi==2023.5.7 +cffi==1.15.1 +chardet==5.1.0 +charset-normalizer==3.2.0 +chromadb==0.3.26 +click==8.1.3 +clickhouse-connect==0.6.4 +colorama==0.4.6 +coloredlogs==15.0.1 +commonmark==0.9.1 +cryptography==41.0.1 +dataclasses-json==0.5.8 +Deprecated==1.2.14 +dnspython==2.3.0 +docutils==0.19 +duckdb==0.8.1 +et-xmlfile==1.1.0 +fastapi==0.98.0 +filetype==1.2.0 +Flask==2.3.2 +flatbuffers==23.5.26 +frozenlist==1.3.3 +fsspec==2023.6.0 +graphlib-backport==1.0.3 +greenlet==2.0.2 +h11==0.14.0 +hnswlib==0.7.0 +httpcore==0.16.3 +httptools==0.5.0 +httpx==0.23.3 +humanfriendly==10.0 +idna==3.4 +itsdangerous==2.1.2 +Jinja2==3.1.2 +joblib==1.3.1 +jsonify==0.5 +langchain==0.0.219 +langchainplus-sdk==0.0.17 +llama-index==0.7.4 +loguru==0.7.0 +lxml==4.9.2 +lz4==4.3.2 +make-response==1 +Markdown==3.4.3 +MarkupSafe==2.1.3 +marshmallow==3.19.0 +marshmallow-enum==1.5.1 +monotonic==1.6 +mpmath==1.3.0 +msg-parser==1.2.0 +multidict==6.0.4 +mypy-extensions==1.0.0 +nest-asyncio==1.5.6 +nltk==3.8.1 +numexpr==2.8.4 +numpy==1.23.5 +olefile==0.46 +onnxruntime==1.15.1 +openai==0.27.8 +openapi-schema-pydantic==1.2.4 +openpyxl==3.1.2 +overrides==7.3.1 +packaging==23.1 +pandas==1.3.5 +pdf2image==1.16.3 +pdfminer.six==20221105 +Pillow==9.5.0 +pinecone-client==2.2.2 +posthog==3.0.1 +protobuf==4.23.4 +pulsar-client==3.2.0 +pycparser==2.21 +pydantic==1.10.10 +pydantic_core==2.1.2 +Pygments==2.15.1 +pypandoc==1.11 +pyreadline3==3.4.1 +python-dateutil==2.8.2 +python-docx==0.8.11 +python-dotenv==1.0.0 +python-magic==0.4.27 +python-pptx==0.6.21 +pytz==2023.3 +pywatchman==1.4.1 +PyYAML==6.0 +redis==4.6.0 +regex==2023.6.3 +requests==2.28.2 +rfc3986==1.5.0 +rich==13.0.1 +six==1.16.0 +sniffio==1.3.0 +soupsieve==2.4.1 +SQLAlchemy==2.0.17 +starlette==0.27.0 +sympy==1.12 +tabulate==0.9.0 +tenacity==8.2.2 +tiktoken==0.4.0 +tokenizers==0.13.3 +tqdm==4.65.0 +typer==0.7.0 +typing-inspect==0.8.0 +typing_extensions==4.5.0 +unstructured==0.7.10 +urllib3==1.26.16 +uvicorn==0.22.0 +uvloop==0.17.0 +watchfiles==0.19.0 +websockets==11.0.3 +Werkzeug==2.3.6 +wrapt==1.14.1 +xlrd==2.0.1 +XlsxWriter==3.1.2 +yarl==1.9.2 +zstandard==0.21.0 From e731a4b0fc436329d5add2576b2caf53abeca92c Mon Sep 17 00:00:00 2001 From: serafettin Date: Wed, 19 Jul 2023 00:30:35 +0200 Subject: [PATCH 03/37] iterate --- Dockerfile | 3 +- app/main.py | 13 +++-- requirements.txt | 126 ----------------------------------------------- 3 files changed, 10 insertions(+), 132 deletions(-) delete mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index c42fc7f5..9e48e409 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ FROM python:3.11.3 WORKDIR /code -COPY ./requirements.txt /code/requirements.txt -RUN pip3 install numpy==1.23.5 fastapi pydantic llama_index chromadb==0.3.26 openai redis --no-cache +RUN pip3 install numpy==1.24.2 fastapi==0.85.1 pydantic==1.10.11 llama_index==0.7.10 chromadb==0.3.29 openai redis --no-cache-dir COPY ./app /code/app CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"] diff --git a/app/main.py b/app/main.py index 4089225a..75c2ff7d 100644 --- a/app/main.py +++ b/app/main.py @@ -27,10 +27,11 @@ import redis -redisClient = redis.Redis(host='localhost', port=6379, db=0) - load_dotenv() +redisClient = redis.Redis.from_url(os.environ["REDIS_CONNECTION_STRING"]); + + origins = [ "http://localhost", "http://localhost:3000", @@ -65,6 +66,7 @@ def get_index(index_id: str): collection = chroma_client.get_or_create_collection(name="index-" + index_id) vector_store = ChromaVectorStore(chroma_collection=collection) index = VectorStoreIndex.from_vector_store(vector_store=vector_store, service_context=service_context) + return index class ChatHistory(BaseModel): @@ -84,7 +86,7 @@ class Composition(BaseModel): def add(index_id, link: Link): index = get_index(index_id=index_id) - + UnstructuredURLLoader = download_loader("UnstructuredURLLoader") loader = UnstructuredURLLoader(urls=[link.url]) kb_data = loader.load() @@ -117,6 +119,7 @@ async def query(index_id, prompt: Prompt): async def chat_stream(index_id, chat_history: ChatHistory): index = get_index(index_id=index_id) + chat_engine = index.as_chat_engine(streaming=True, verbose=True) messages = chat_history.messages @@ -160,8 +163,10 @@ async def compose(c: Composition): index_ids = [item.rstrip(":my_indexes") for item in id_resp] indexes = list(map(lambda index_id: get_index(index_id=index_id), index_ids)) - summaries = redisClient.hmget("summaries", index_ids) + indexes = [get_index(index_id=index_id) for index_id in index_ids if get_index(index_id=index_id)] + summaries = redisClient.hmget("summaries", index_ids) + graph = ComposableGraph.from_indices( ListIndex, indexes, diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 8f9d66e5..00000000 --- a/requirements.txt +++ /dev/null @@ -1,126 +0,0 @@ -aiohttp==3.8.4 -aiosignal==1.3.1 -annotated-types==0.5.0 -anyio==3.7.0 -argilla==1.12.0 -async-timeout==4.0.2 -attrs==23.1.0 -backoff==2.2.1 -beautifulsoup4==4.12.2 -blinker==1.6.2 -certifi==2023.5.7 -cffi==1.15.1 -chardet==5.1.0 -charset-normalizer==3.2.0 -chromadb==0.3.26 -click==8.1.3 -clickhouse-connect==0.6.4 -colorama==0.4.6 -coloredlogs==15.0.1 -commonmark==0.9.1 -cryptography==41.0.1 -dataclasses-json==0.5.8 -Deprecated==1.2.14 -dnspython==2.3.0 -docutils==0.19 -duckdb==0.8.1 -et-xmlfile==1.1.0 -fastapi==0.98.0 -filetype==1.2.0 -Flask==2.3.2 -flatbuffers==23.5.26 -frozenlist==1.3.3 -fsspec==2023.6.0 -graphlib-backport==1.0.3 -greenlet==2.0.2 -h11==0.14.0 -hnswlib==0.7.0 -httpcore==0.16.3 -httptools==0.5.0 -httpx==0.23.3 -humanfriendly==10.0 -idna==3.4 -itsdangerous==2.1.2 -Jinja2==3.1.2 -joblib==1.3.1 -jsonify==0.5 -langchain==0.0.219 -langchainplus-sdk==0.0.17 -llama-index==0.7.4 -loguru==0.7.0 -lxml==4.9.2 -lz4==4.3.2 -make-response==1 -Markdown==3.4.3 -MarkupSafe==2.1.3 -marshmallow==3.19.0 -marshmallow-enum==1.5.1 -monotonic==1.6 -mpmath==1.3.0 -msg-parser==1.2.0 -multidict==6.0.4 -mypy-extensions==1.0.0 -nest-asyncio==1.5.6 -nltk==3.8.1 -numexpr==2.8.4 -numpy==1.23.5 -olefile==0.46 -onnxruntime==1.15.1 -openai==0.27.8 -openapi-schema-pydantic==1.2.4 -openpyxl==3.1.2 -overrides==7.3.1 -packaging==23.1 -pandas==1.3.5 -pdf2image==1.16.3 -pdfminer.six==20221105 -Pillow==9.5.0 -pinecone-client==2.2.2 -posthog==3.0.1 -protobuf==4.23.4 -pulsar-client==3.2.0 -pycparser==2.21 -pydantic==1.10.10 -pydantic_core==2.1.2 -Pygments==2.15.1 -pypandoc==1.11 -pyreadline3==3.4.1 -python-dateutil==2.8.2 -python-docx==0.8.11 -python-dotenv==1.0.0 -python-magic==0.4.27 -python-pptx==0.6.21 -pytz==2023.3 -pywatchman==1.4.1 -PyYAML==6.0 -redis==4.6.0 -regex==2023.6.3 -requests==2.28.2 -rfc3986==1.5.0 -rich==13.0.1 -six==1.16.0 -sniffio==1.3.0 -soupsieve==2.4.1 -SQLAlchemy==2.0.17 -starlette==0.27.0 -sympy==1.12 -tabulate==0.9.0 -tenacity==8.2.2 -tiktoken==0.4.0 -tokenizers==0.13.3 -tqdm==4.65.0 -typer==0.7.0 -typing-inspect==0.8.0 -typing_extensions==4.5.0 -unstructured==0.7.10 -urllib3==1.26.16 -uvicorn==0.22.0 -uvloop==0.17.0 -watchfiles==0.19.0 -websockets==11.0.3 -Werkzeug==2.3.6 -wrapt==1.14.1 -xlrd==2.0.1 -XlsxWriter==3.1.2 -yarl==1.9.2 -zstandard==0.21.0 From 7bc9e7cd4e2d2dde6e116c5153afea488e1f07e7 Mon Sep 17 00:00:00 2001 From: serafettin Date: Wed, 19 Jul 2023 01:18:52 +0200 Subject: [PATCH 04/37] iterate --- .github/workflows/build.yaml | 3 +-- app/main.py | 27 +++++++++++++-------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6515ab85..80272962 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -60,5 +60,4 @@ jobs: - name: Deploy run: |- - kubectl set image deployment/api api=534970752686.dkr.ecr.us-east-1.amazonaws.com/indexas/llm-indexer:${{ steps.build-time.outputs.time }} --namespace web3-${GITHUB_REF#refs/heads/} - kubectl set image deployment/consumer consumer=534970752686.dkr.ecr.us-east-1.amazonaws.com/indexas/llm-indexer:${{ steps.build-time.outputs.time }} --namespace web3-${GITHUB_REF#refs/heads/} + kubectl set image deployment/llm-indexer api=534970752686.dkr.ecr.us-east-1.amazonaws.com/indexas/llm-indexer:${{ steps.build-time.outputs.time }} --namespace web3-${GITHUB_REF#refs/heads/} diff --git a/app/main.py b/app/main.py index 75c2ff7d..ae66ce31 100644 --- a/app/main.py +++ b/app/main.py @@ -106,17 +106,17 @@ def remove(index_id: str, link: Link): @app.post("/index/{index_id}/prompt") -async def query(index_id, prompt: Prompt): +def query(index_id, prompt: Prompt): index = get_index(index_id=index_id) - response = index.as_query_engine(streaming=True).query(prompt.prompt) - # return JSONResponse(content={ - # "response": response.response, - # "sources": [{"id": s.node.id_, "url": s.node.metadata.get("source")} for s in response.source_nodes] - # }) - return StreamingResponse(response.response_gen, media_type='text/event-stream') + response = index.as_query_engine().query(prompt.prompt) + return JSONResponse(content={ + #"sources": [{"id": s.node.id_, "url": s.node.metadata.get("source"), "index_id": s.node.metadata.get("index_id")} for s in response.sources] + "response": response.response + }) + #return StreamingResponse(response.response_gen, media_type='text/event-stream') @app.post("/index/{index_id}/chat_stream") -async def chat_stream(index_id, chat_history: ChatHistory): +def chat_stream(index_id, chat_history: ChatHistory): index = get_index(index_id=index_id) @@ -148,15 +148,14 @@ async def chat(index_id, chat_history: ChatHistory): last_message = messages[-1] response = chat_engine.chat(message=last_message.content, chat_history=messages) - return JSONResponse(content={ - "response": response.response, - "sources": [{"id": s.node.id_, "url": s.node.metadata.get("source"), "index_id": s.node.metadata.get("index_id")} for s in response.source_nodes] + #"sources": [{"id": s.node.id_, "url": s.node.metadata.get("source"), "index_id": s.node.metadata.get("index_id")} for s in response.sources] + "response": response.response }) @app.post("/compose") -async def compose(c: Composition): +def compose(c: Composition): id_resp = redisClient.hkeys("user_indexes:by_did:" + c.did.lower()) @@ -176,8 +175,8 @@ async def compose(c: Composition): query_engine = graph.as_query_engine() response = query_engine.query(c.prompt) return JSONResponse(content={ - "response": response.response, - "sources": [{"id": s.node.id_, "url": s.node.metadata.get("source"), "index_id": s.node.metadata.get("index_id")} for s in response.source_nodes] + #"sources": [{"id": s.node.id_, "url": s.node.metadata.get("source"), "index_id": s.node.metadata.get("index_id")} for s in response.source_nodes], + "response": response.response }) From 294bc872a6fafe72af97bd88ae51dd478a14155b Mon Sep 17 00:00:00 2001 From: serafettin Date: Wed, 19 Jul 2023 01:38:08 +0200 Subject: [PATCH 05/37] Update main.py --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index ae66ce31..b1db05fc 100644 --- a/app/main.py +++ b/app/main.py @@ -63,7 +63,7 @@ def get_index(index_id: str): - collection = chroma_client.get_or_create_collection(name="index-" + index_id) + collection = chroma_client.get_or_create_collection(name=index_id) vector_store = ChromaVectorStore(chroma_collection=collection) index = VectorStoreIndex.from_vector_store(vector_store=vector_store, service_context=service_context) From 49c24ee821c9316dfa64c45dbcef365ef714a0dd Mon Sep 17 00:00:00 2001 From: serafettin Date: Wed, 19 Jul 2023 01:41:51 +0200 Subject: [PATCH 06/37] Update build.yaml --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 80272962..386a3b2c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -60,4 +60,4 @@ jobs: - name: Deploy run: |- - kubectl set image deployment/llm-indexer api=534970752686.dkr.ecr.us-east-1.amazonaws.com/indexas/llm-indexer:${{ steps.build-time.outputs.time }} --namespace web3-${GITHUB_REF#refs/heads/} + kubectl set image deployment/llm-indexer llm-indexer=534970752686.dkr.ecr.us-east-1.amazonaws.com/indexas/llm-indexer:${{ steps.build-time.outputs.time }} --namespace web3-${GITHUB_REF#refs/heads/} From 4439cc29204bbf1b72817041ba63920995f28a2c Mon Sep 17 00:00:00 2001 From: serafettin Date: Wed, 19 Jul 2023 08:39:59 +0200 Subject: [PATCH 07/37] Update main.py --- app/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index b1db05fc..f71a16bd 100644 --- a/app/main.py +++ b/app/main.py @@ -159,7 +159,8 @@ def compose(c: Composition): id_resp = redisClient.hkeys("user_indexes:by_did:" + c.did.lower()) - index_ids = [item.rstrip(":my_indexes") for item in id_resp] + + index_ids = [item.decode("utf-8").rstrip(":my_indexes") for item in id_resp] indexes = list(map(lambda index_id: get_index(index_id=index_id), index_ids)) indexes = [get_index(index_id=index_id) for index_id in index_ids if get_index(index_id=index_id)] From 7388951b518188cbdf44552f0e0e458464560474 Mon Sep 17 00:00:00 2001 From: serafettin Date: Wed, 19 Jul 2023 09:23:23 +0200 Subject: [PATCH 08/37] Update main.py --- app/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index f71a16bd..791ee053 100644 --- a/app/main.py +++ b/app/main.py @@ -160,11 +160,12 @@ def compose(c: Composition): id_resp = redisClient.hkeys("user_indexes:by_did:" + c.did.lower()) - index_ids = [item.decode("utf-8").rstrip(":my_indexes") for item in id_resp] + index_ids = [item.decode('utf-8').split(':')[0] for item in id_resp] indexes = list(map(lambda index_id: get_index(index_id=index_id), index_ids)) indexes = [get_index(index_id=index_id) for index_id in index_ids if get_index(index_id=index_id)] - + + summaries = redisClient.hmget("summaries", index_ids) graph = ComposableGraph.from_indices( From 40b88c0af47994188ec9fb4d308685db662b7689 Mon Sep 17 00:00:00 2001 From: serafettin Date: Sat, 5 Aug 2023 10:38:23 +0300 Subject: [PATCH 09/37] iterate --- Dockerfile | 1 + app/main.py | 90 +++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 68 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9e48e409..c861f8f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ FROM python:3.11.3 WORKDIR /code RUN pip3 install numpy==1.24.2 fastapi==0.85.1 pydantic==1.10.11 llama_index==0.7.10 chromadb==0.3.29 openai redis --no-cache-dir + COPY ./app /code/app CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"] diff --git a/app/main.py b/app/main.py index 791ee053..72568ae6 100644 --- a/app/main.py +++ b/app/main.py @@ -10,7 +10,6 @@ from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel - from llama_index import download_loader from llama_index.llms import ChatMessage from llama_index.vector_stores import ChromaVectorStore @@ -18,8 +17,13 @@ from llama_index import VectorStoreIndex, ListIndex, LLMPredictor, ServiceContext from llama_index.chat_engine.types import ChatMode from llama_index.indices.composability import ComposableGraph +from llama_index.indices.query.query_transform import HyDEQueryTransform +from llama_index.query_engine.transform_query_engine import TransformQueryEngine + from langchain.chat_models import ChatOpenAI +from llama_index.vector_stores.types import ExactMatchFilter, MetadataFilters + import chromadb from chromadb.config import Settings @@ -57,19 +61,37 @@ persist_directory=persist_directory )) -llm = ChatOpenAI(temperature=0, model_name="gpt-3.5-turbo", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) -llm_predictor = LLMPredictor(llm=llm) -service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor) +print(os.environ["OPENAI_API_KEY"]) +embed_model = OpenAIEmbedding(model="text-embedding-ada-002", openai_api_key=os.environ["OPENAI_API_KEY"],) +llm = ChatOpenAI(temperature=0, model_name="gpt-4", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) +service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model) +hyde = HyDEQueryTransform(include_original=True) -def get_index(index_id: str): - collection = chroma_client.get_or_create_collection(name=index_id) +def get_collection(): + collection = chroma_client.get_or_create_collection(name="indexes") vector_store = ChromaVectorStore(chroma_collection=collection) index = VectorStoreIndex.from_vector_store(vector_store=vector_store, service_context=service_context) return index +def get_query_engine(indexes): + collection = get_collection() + + if len(indexes) > 1: + index_filters = {"$or": [{"index_id": {"$eq": i}} for i in indexes]} + else: + index_filters = {"index_id": {"$eq": indexes[0]}} + + return collection.as_query_engine(streaming=True, verbose=True, vector_store_kwargs={"where": index_filters}) + +def get_index_chat_engine(indexes): + collection = get_collection() + index_filters = [{"index_id": {"$eq": i}} for i in indexes] + return collection.as_chat_engine(chat_mode="condense_question", streaming=True, verbose=True, vector_store_kwargs={"where": {"$or": index_filters}}) + class ChatHistory(BaseModel): + id: str messages: list[ChatMessage] class Prompt(BaseModel): @@ -82,21 +104,32 @@ class Composition(BaseModel): did: str prompt: str +def response_generator(response): + yield json.dumps({ + "sources": [{"id": s.node.id_, "url": s.node.metadata.get("source")} for s in response.source_nodes] + }) + yield "\n ###endjson### \n\n" + for text in response.response_gen: + yield text + +@app.get("/") +def home(): + return JSONResponse(content="ia") + @app.post("/index/{index_id}/links") def add(index_id, link: Link): - index = get_index(index_id=index_id) + collection = get_collection() UnstructuredURLLoader = download_loader("UnstructuredURLLoader") loader = UnstructuredURLLoader(urls=[link.url]) kb_data = loader.load() - - index.insert(kb_data[0]) + kb_data[0].metadata["index_id"] = index_id + collection.insert(kb_data[0],) chroma_client.persist() - summary = index.as_query_engine().query("summarize").response - x = redisClient.hset("summaries", index_id, summary) - print(x) + #summary = collection.as_query_engine().query("summarize").response #WRONG ONLY SUMMARIZE THIS DOC + #redisClient.hset("summaries", index_id, summary) return JSONResponse(content={'message': 'Document added successfully'}) @@ -105,28 +138,39 @@ def remove(index_id: str, link: Link): return JSONResponse(content={"message": "Documents deleted successfully"}) -@app.post("/index/{index_id}/prompt") -def query(index_id, prompt: Prompt): - index = get_index(index_id=index_id) - response = index.as_query_engine().query(prompt.prompt) +@app.post("/compose_new") +def query(prompt: Prompt): + + collection = get_collection() + + + hyde_query_engine = TransformQueryEngine( collection.as_query_engine(), hyde) + + response = hyde_query_engine.query(prompt.prompt) + + print(response.get_formatted_sources()) + return JSONResponse(content={ - #"sources": [{"id": s.node.id_, "url": s.node.metadata.get("source"), "index_id": s.node.metadata.get("index_id")} for s in response.sources] + "sources": [{"id": s.node.id_, "url": s.node.metadata.get("source"), "index_id": s.node.metadata.get("index_id")} for s in response.source_nodes], "response": response.response }) - #return StreamingResponse(response.response_gen, media_type='text/event-stream') -@app.post("/index/{index_id}/chat_stream") -def chat_stream(index_id, chat_history: ChatHistory): +@app.post("/index/{index_id}/prompt") +async def query(index_id, prompt: Prompt): + response = get_query_engine([index_id]).query(prompt.prompt) + return StreamingResponse(response_generator(response), media_type='text/event-stream') - index = get_index(index_id=index_id) - chat_engine = index.as_chat_engine(streaming=True, verbose=True) +@app.post("/index/{index_id}/chat_stream") +def chat_stream(index_id, chat_history: ChatHistory): + print(index_id) + index = get_index_chat_engine([index_id, index_id]) messages = chat_history.messages last_message = messages[-1] print(last_message.content, messages) # TODO Pop last "user" message from chat history. - streaming_response = chat_engine.stream_chat(message=last_message.content, chat_history=messages) + streaming_response = index.stream_chat(message=last_message.content, chat_history=messages) def response_generator(): yield json.dumps({ From cd4b8898a619aafa9804fc727572d16317f61dd1 Mon Sep 17 00:00:00 2001 From: serafettin Date: Sat, 5 Aug 2023 13:08:14 +0300 Subject: [PATCH 10/37] Update main.py --- app/main.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/app/main.py b/app/main.py index 72568ae6..c6d9eaf5 100644 --- a/app/main.py +++ b/app/main.py @@ -19,11 +19,10 @@ from llama_index.indices.composability import ComposableGraph from llama_index.indices.query.query_transform import HyDEQueryTransform from llama_index.query_engine.transform_query_engine import TransformQueryEngine +from llama_index.vector_stores.types import ExactMatchFilter, MetadataFilters from langchain.chat_models import ChatOpenAI -from llama_index.vector_stores.types import ExactMatchFilter, MetadataFilters - import chromadb from chromadb.config import Settings @@ -173,12 +172,15 @@ def chat_stream(index_id, chat_history: ChatHistory): streaming_response = index.stream_chat(message=last_message.content, chat_history=messages) def response_generator(): - yield json.dumps({ - "sources": [{"id": s.node.id_, "url": s.node.metadata.get("source")} for s in streaming_response.source_nodes] - }) - yield "\n ###endjson### \n\n" for text in streaming_response.response_gen: yield text + + yield '\n\nSources:' + for s in streaming_response.source_nodes: + yield '\n\n' + yield s.node.metadata.get("source") + + return StreamingResponse(response_generator(), media_type='text/event-stream') From 432e25ce178827b4eafeff57910fd4c3c42df561 Mon Sep 17 00:00:00 2001 From: serafettin Date: Tue, 8 Aug 2023 20:52:57 +0300 Subject: [PATCH 11/37] Update main.py --- app/main.py | 66 ++++++++++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/app/main.py b/app/main.py index c6d9eaf5..f9ed460b 100644 --- a/app/main.py +++ b/app/main.py @@ -9,6 +9,8 @@ from fastapi.responses import StreamingResponse from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel +from typing import Optional, List + from llama_index import download_loader from llama_index.llms import ChatMessage @@ -60,8 +62,6 @@ persist_directory=persist_directory )) -print(os.environ["OPENAI_API_KEY"]) - embed_model = OpenAIEmbedding(model="text-embedding-ada-002", openai_api_key=os.environ["OPENAI_API_KEY"],) llm = ChatOpenAI(temperature=0, model_name="gpt-4", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model) @@ -86,12 +86,19 @@ def get_query_engine(indexes): def get_index_chat_engine(indexes): collection = get_collection() - index_filters = [{"index_id": {"$eq": i}} for i in indexes] - return collection.as_chat_engine(chat_mode="condense_question", streaming=True, verbose=True, vector_store_kwargs={"where": {"$or": index_filters}}) + + if len(indexes) > 1: + index_filters = {"$or": [{"index_id": {"$eq": i}} for i in indexes]} + else: + index_filters = {"index_id": {"$eq": indexes[0]}} + + return collection.as_chat_engine(chat_mode="context", streaming=True, verbose=True, vector_store_kwargs={"where": index_filters}) -class ChatHistory(BaseModel): +class ChatStream(BaseModel): id: str - messages: list[ChatMessage] + did: Optional[str] + indexes: Optional[List[str]] + messages: List[ChatMessage] class Prompt(BaseModel): prompt: str @@ -160,45 +167,38 @@ async def query(index_id, prompt: Prompt): return StreamingResponse(response_generator(response), media_type='text/event-stream') -@app.post("/index/{index_id}/chat_stream") -def chat_stream(index_id, chat_history: ChatHistory): - print(index_id) - index = get_index_chat_engine([index_id, index_id]) - - messages = chat_history.messages +@app.post("/chat_stream") +def chat_stream(params: ChatStream): + + if params.did: + id_resp = redisClient.hkeys("user_indexes:by_did:" + params.did.lower()) + if not id_resp: + return "You have no indexes" + indexes = [item.decode('utf-8').split(':')[0] for item in id_resp] + elif params.indexes: + indexes = params.indexes + + print(indexes) + index = get_index_chat_engine(indexes) + + messages = params.messages last_message = messages[-1] - print(last_message.content, messages) - # TODO Pop last "user" message from chat history. + streaming_response = index.stream_chat(message=last_message.content, chat_history=messages) def response_generator(): for text in streaming_response.response_gen: yield text - yield '\n\nSources:' - for s in streaming_response.source_nodes: - yield '\n\n' - yield s.node.metadata.get("source") +# yield '\n\nSources:' +# for s in streaming_response.source_nodes: +# yield '\n\n' +# yield s.node.metadata.get("source") return StreamingResponse(response_generator(), media_type='text/event-stream') -@app.post("/index/{index_id}/chat") -async def chat(index_id, chat_history: ChatHistory): - - index = get_index(index_id=index_id) - chat_engine = index.as_chat_engine() - - messages = chat_history.messages - last_message = messages[-1] - - response = chat_engine.chat(message=last_message.content, chat_history=messages) - return JSONResponse(content={ - #"sources": [{"id": s.node.id_, "url": s.node.metadata.get("source"), "index_id": s.node.metadata.get("index_id")} for s in response.sources] - "response": response.response - }) - @app.post("/compose") def compose(c: Composition): From b64d20b4728dcb9516e946a91c95ae93cf20e53d Mon Sep 17 00:00:00 2001 From: serafettin Date: Tue, 8 Aug 2023 21:17:13 +0300 Subject: [PATCH 12/37] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c861f8f5..9e49280d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.11.3 WORKDIR /code -RUN pip3 install numpy==1.24.2 fastapi==0.85.1 pydantic==1.10.11 llama_index==0.7.10 chromadb==0.3.29 openai redis --no-cache-dir +RUN pip3 install numpy==1.24.2 fastapi==0.85.1 pydantic==1.10.11 llama_index==0.7.14 chromadb==0.3.29 openai redis --no-cache-dir COPY ./app /code/app CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"] From 5e0a9dbb4a8fc37bdc87fed2a60f6be16f5fb4ab Mon Sep 17 00:00:00 2001 From: serafettin Date: Wed, 9 Aug 2023 11:50:11 +0300 Subject: [PATCH 13/37] Update main.py --- app/main.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/app/main.py b/app/main.py index f9ed460b..5f103fcf 100644 --- a/app/main.py +++ b/app/main.py @@ -67,6 +67,8 @@ service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model) hyde = HyDEQueryTransform(include_original=True) +UnstructuredURLLoader = download_loader("UnstructuredURLLoader") + def get_collection(): collection = chroma_client.get_or_create_collection(name="indexes") vector_store = ChromaVectorStore(chroma_collection=collection) @@ -127,16 +129,24 @@ def add(index_id, link: Link): collection = get_collection() - UnstructuredURLLoader = download_loader("UnstructuredURLLoader") loader = UnstructuredURLLoader(urls=[link.url]) + + try: + kb_data = loader.load() + if not kb_data: + return JSONResponse(content={'message': 'No data loaded from the provided link'}, status_code=400) + except Exception as e: + return JSONResponse(content={'message': 'Url load error'}, status_code=400) + kb_data = loader.load() + + if not kb_data: + return JSONResponse(content={'message': 'No data loaded from the provided link'}, status_code=400) + kb_data[0].metadata["index_id"] = index_id - collection.insert(kb_data[0],) + collection.insert(kb_data[0]) chroma_client.persist() - #summary = collection.as_query_engine().query("summarize").response #WRONG ONLY SUMMARIZE THIS DOC - #redisClient.hset("summaries", index_id, summary) - return JSONResponse(content={'message': 'Document added successfully'}) @app.delete("/index/{index_id}/links") From 955ca99019cf165b690e88c275b3629eb25e4f7d Mon Sep 17 00:00:00 2001 From: serafettin Date: Sat, 19 Aug 2023 10:22:07 +0300 Subject: [PATCH 14/37] Update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7478b2ca..c0f99fa1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ .chroma/* chroma-indexes/* .idea/* -__pycache__/* +app/__pycache__/* .DS_Store .env \ No newline at end of file From fd64db1ba84910db125a3d3d741984ac57cbef28 Mon Sep 17 00:00:00 2001 From: serafettin Date: Sat, 19 Aug 2023 10:25:03 +0300 Subject: [PATCH 15/37] Update main.py --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 5f103fcf..dd2d28a3 100644 --- a/app/main.py +++ b/app/main.py @@ -94,7 +94,7 @@ def get_index_chat_engine(indexes): else: index_filters = {"index_id": {"$eq": indexes[0]}} - return collection.as_chat_engine(chat_mode="context", streaming=True, verbose=True, vector_store_kwargs={"where": index_filters}) + return collection.as_chat_engine(chat_mode="context", similarity_top_k=5, streaming=True, verbose=True, vector_store_kwargs={"where": index_filters}) class ChatStream(BaseModel): id: str From 9aae373069c1b0a963e57ba0af3b568a59bda53a Mon Sep 17 00:00:00 2001 From: serafettin Date: Sat, 19 Aug 2023 10:31:39 +0300 Subject: [PATCH 16/37] Update build.yaml --- .github/workflows/build.yaml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 386a3b2c..919df841 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -2,8 +2,8 @@ name: deploy on: push: branches: - - dev - testnet + - dev jobs: main: runs-on: ubuntu-latest @@ -13,7 +13,7 @@ jobs: uses: azure/setup-kubectl@v2.0 with: version: 'v1.23.6' # default is latest stable - id: install + id: install - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -26,8 +26,15 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v1 - - run: |- - aws eks --region us-east-1 update-kubeconfig --name indexas + - name: Install kubectl + uses: azure/setup-kubectl@v2.0 + + - name: Set kubectl context + uses: azure/k8s-set-context@v3 + with: + method: kubeconfig + kubeconfig: ${{ secrets.KUBECONFIG }} + context: microk8s - name: Check k8s connection run: kubectl get pods @@ -60,4 +67,4 @@ jobs: - name: Deploy run: |- - kubectl set image deployment/llm-indexer llm-indexer=534970752686.dkr.ecr.us-east-1.amazonaws.com/indexas/llm-indexer:${{ steps.build-time.outputs.time }} --namespace web3-${GITHUB_REF#refs/heads/} + kubectl set image deployment/llm-indexer llm-indexer=534970752686.dkr.ecr.us-east-1.amazonaws.com/indexas/llm-indexer:${{ steps.build-time.outputs.time }} --namespace ${GITHUB_REF#refs/heads/} From 287231ab9680461f32088e74cfad95f5dea61cd7 Mon Sep 17 00:00:00 2001 From: serafettin Date: Sat, 19 Aug 2023 15:18:06 +0300 Subject: [PATCH 17/37] Update main.py --- app/main.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app/main.py b/app/main.py index dd2d28a3..4f5d730a 100644 --- a/app/main.py +++ b/app/main.py @@ -62,17 +62,19 @@ persist_directory=persist_directory )) -embed_model = OpenAIEmbedding(model="text-embedding-ada-002", openai_api_key=os.environ["OPENAI_API_KEY"],) -llm = ChatOpenAI(temperature=0, model_name="gpt-4", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) -service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model) +UnstructuredURLLoader = download_loader("UnstructuredURLLoader") hyde = HyDEQueryTransform(include_original=True) -UnstructuredURLLoader = download_loader("UnstructuredURLLoader") +def get_service_context(): + embed_model = OpenAIEmbedding(model="text-embedding-ada-002", openai_api_key=os.environ["OPENAI_API_KEY"]) + llm = ChatOpenAI(temperature=0, model_name="gpt-4", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) + service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model) + return service_context def get_collection(): collection = chroma_client.get_or_create_collection(name="indexes") vector_store = ChromaVectorStore(chroma_collection=collection) - index = VectorStoreIndex.from_vector_store(vector_store=vector_store, service_context=service_context) + index = VectorStoreIndex.from_vector_store(vector_store=vector_store, service_context=get_service_context()) return index From c6424f6ce58c8736aa710fb88256271c5d5119c6 Mon Sep 17 00:00:00 2001 From: serafettin Date: Sun, 17 Sep 2023 18:11:07 +0300 Subject: [PATCH 18/37] Index new filetypes csv, epub, xlsx, xls, md, org, odt, pdf, txt, ppt, pptx, rst, rtf, tsv, doc, docx, xml, json, html --- .gitignore | 3 +- app/document_parser.py | 124 +++++++++++++++++++++++++++++++++++++++++ app/main.py | 71 +++++++++++------------ requirements.txt | 39 +++++++++++++ 4 files changed, 197 insertions(+), 40 deletions(-) create mode 100644 app/document_parser.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index c0f99fa1..27bf8b6b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ chroma-indexes/* .idea/* app/__pycache__/* .DS_Store -.env \ No newline at end of file +.env +files \ No newline at end of file diff --git a/app/document_parser.py b/app/document_parser.py new file mode 100644 index 00000000..43a6a2fa --- /dev/null +++ b/app/document_parser.py @@ -0,0 +1,124 @@ +import mimetypes +import os +import requests +import json +import uuid +import jq +from unstructured.partition.auto import partition +from langchain.document_loaders import JSONLoader, UnstructuredFileLoader + + +from fastapi.responses import JSONResponse + + +class Transformers: + @staticmethod + def unstructured(url, file_type): + fileName = save_file(url, file_type) + print("Processing with unstructured...", url, file_type, fileName) + loader = UnstructuredFileLoader(file_path=fileName, mode="paged") + return loader.load() + + @staticmethod + def apify(url): + print("Processing with Apify...") + # Actual logic for JSON Transformer goes here + return url + + @staticmethod + def langchainJSON(url, file_type): + print("Processing with Langchain JSON Loader...") + fileName = save_file(url, file_type) + loader = JSONLoader(file_path=fileName, jq_schema='.', text_content=False) + return loader.load() + +fileTypes = { + 'csv': Transformers.unstructured, + 'epub': Transformers.unstructured, + 'xlsx': Transformers.unstructured, + 'xls': Transformers.unstructured, + 'md': Transformers.unstructured, + 'org': Transformers.unstructured, + 'odt': Transformers.unstructured, + 'pdf': Transformers.unstructured, + 'txt': Transformers.unstructured, + 'ppt': Transformers.unstructured, + 'pptx': Transformers.unstructured, + 'rst': Transformers.unstructured, + 'rtf': Transformers.unstructured, + 'tsv': Transformers.unstructured, + 'doc': Transformers.unstructured, + 'docx': Transformers.unstructured, + 'xml': Transformers.unstructured, + 'json': Transformers.langchainJSON, + 'html': Transformers.unstructured, # ? +} + +def save_file(url, file_type): + # Fetch the content from the URL + response = requests.get(url) + response.raise_for_status() # Raise exception if the request was unsuccessful + + # Generate a unique filename based on UUID + unique_filename = f"{uuid.uuid4()}.{file_type}" + + output_path = os.environ.get('OUTPUT_PATH', '.') + full_path = os.path.join(output_path, unique_filename) + + # Save the content to the specified path with the generated filename + with open(full_path, 'wb') as f: + f.write(response.content) + + return full_path + +def guess_file_type(url_or_file_path): + mime_type, encoding = mimetypes.guess_type(url_or_file_path) + extension = url_or_file_path.split('.')[-1] if '.' in url_or_file_path else None + return extension, mime_type + +def get_mime_from_head_request(url): + try: + response = requests.head(url, allow_redirects=True) + return response.headers.get('Content-Type', '').split(';')[0] # Exclude charset if present + except requests.RequestException: + return None + +def map_mime_to_filetype(mime_type): + mime_to_filetype_map = { + 'text/csv': 'csv', + 'application/epub+zip': 'epub', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'xlsx', + 'application/vnd.ms-excel': 'xls', + 'text/html': 'html', + 'text/markdown': 'md', + 'text/org': 'org', + 'application/vnd.oasis.opendocument.text': 'odt', + 'application/pdf': 'pdf', + 'text/plain': 'txt', + 'application/vnd.ms-powerpoint': 'ppt', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'pptx', + 'text/rtf': 'rtf', + 'text/tab-separated-values': 'tsv', + 'application/msword': 'doc', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx', + 'application/xml': 'xml', + 'application/json': 'json' + } + return mime_to_filetype_map.get(mime_type) + + +def resolve_file(url): + file_type, _ = guess_file_type(url) + transformer = fileTypes.get(file_type) + if transformer: + return transformer, file_type + + mime_type = get_mime_from_head_request(url) + if mime_type: + file_type = map_mime_to_filetype(mime_type) + transformer = fileTypes.get(file_type) + return transformer, file_type + +def get_document(url): + transformer, file_type = resolve_file(url) + return transformer(url, file_type) diff --git a/app/main.py b/app/main.py index 4f5d730a..fc02ac2d 100644 --- a/app/main.py +++ b/app/main.py @@ -24,11 +24,18 @@ from llama_index.vector_stores.types import ExactMatchFilter, MetadataFilters from langchain.chat_models import ChatOpenAI +from llama_index import Document + +from unstructured.partition.auto import partition + +from app.document_parser import get_document import chromadb from chromadb.config import Settings -import openai # +import openai # + + import redis @@ -53,18 +60,16 @@ ) openai.api_key = os.environ["OPENAI_API_KEY"] +openai.log = "error" local_directory = "chroma-indexes" persist_directory = os.path.join(os.getcwd(), local_directory) -chroma_client = chromadb.Client(Settings( - chroma_db_impl="duckdb+parquet", - persist_directory=persist_directory -)) +chroma_client = chromadb.PersistentClient(path=persist_directory) -UnstructuredURLLoader = download_loader("UnstructuredURLLoader") hyde = HyDEQueryTransform(include_original=True) + def get_service_context(): embed_model = OpenAIEmbedding(model="text-embedding-ada-002", openai_api_key=os.environ["OPENAI_API_KEY"]) llm = ChatOpenAI(temperature=0, model_name="gpt-4", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) @@ -75,12 +80,12 @@ def get_collection(): collection = chroma_client.get_or_create_collection(name="indexes") vector_store = ChromaVectorStore(chroma_collection=collection) index = VectorStoreIndex.from_vector_store(vector_store=vector_store, service_context=get_service_context()) - + return index def get_query_engine(indexes): collection = get_collection() - + if len(indexes) > 1: index_filters = {"$or": [{"index_id": {"$eq": i}} for i in indexes]} else: @@ -90,7 +95,7 @@ def get_query_engine(indexes): def get_index_chat_engine(indexes): collection = get_collection() - + if len(indexes) > 1: index_filters = {"$or": [{"index_id": {"$eq": i}} for i in indexes]} else: @@ -128,26 +133,14 @@ def home(): @app.post("/index/{index_id}/links") def add(index_id, link: Link): - + collection = get_collection() - loader = UnstructuredURLLoader(urls=[link.url]) - - try: - kb_data = loader.load() - if not kb_data: - return JSONResponse(content={'message': 'No data loaded from the provided link'}, status_code=400) - except Exception as e: - return JSONResponse(content={'message': 'Url load error'}, status_code=400) - - kb_data = loader.load() + documents = get_document(link.url) - if not kb_data: - return JSONResponse(content={'message': 'No data loaded from the provided link'}, status_code=400) - - kb_data[0].metadata["index_id"] = index_id - collection.insert(kb_data[0]) - chroma_client.persist() + for node in documents: + node.metadata = {"source": link.url,"index_id": index_id} + collection.insert(Document.from_langchain_format(node)) return JSONResponse(content={'message': 'Document added successfully'}) @@ -161,7 +154,7 @@ def query(prompt: Prompt): collection = get_collection() - + hyde_query_engine = TransformQueryEngine( collection.as_query_engine(), hyde) response = hyde_query_engine.query(prompt.prompt) @@ -175,13 +168,13 @@ def query(prompt: Prompt): @app.post("/index/{index_id}/prompt") async def query(index_id, prompt: Prompt): - response = get_query_engine([index_id]).query(prompt.prompt) + response = get_query_engine([index_id]).query(prompt.prompt) return StreamingResponse(response_generator(response), media_type='text/event-stream') @app.post("/chat_stream") def chat_stream(params: ChatStream): - + if params.did: id_resp = redisClient.hkeys("user_indexes:by_did:" + params.did.lower()) if not id_resp: @@ -206,8 +199,8 @@ def response_generator(): # for s in streaming_response.source_nodes: # yield '\n\n' # yield s.node.metadata.get("source") - - + + return StreamingResponse(response_generator(), media_type='text/event-stream') @@ -217,28 +210,28 @@ def compose(c: Composition): id_resp = redisClient.hkeys("user_indexes:by_did:" + c.did.lower()) - + index_ids = [item.decode('utf-8').split(':')[0] for item in id_resp] - + indexes = list(map(lambda index_id: get_index(index_id=index_id), index_ids)) indexes = [get_index(index_id=index_id) for index_id in index_ids if get_index(index_id=index_id)] - - + + summaries = redisClient.hmget("summaries", index_ids) - + graph = ComposableGraph.from_indices( ListIndex, indexes, index_summaries=summaries, - max_keywords_per_chunk=2000, + max_keywords_per_chunk=2000, ) query_engine = graph.as_query_engine() response = query_engine.query(c.prompt) return JSONResponse(content={ - #"sources": [{"id": s.node.id_, "url": s.node.metadata.get("source"), "index_id": s.node.metadata.get("index_id")} for s in response.source_nodes], + #"sources": [{"id": s.node.id_, "url": s.node.metadata.get("source"), "index_id": s.node.metadata.get("index_id")} for s in response.source_nodes], "response": response.response }) - + if __name__ == "__main__": import uvicorn diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..6c075cb5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,39 @@ +chroma-hnswlib==0.7.1 +chromadb==0.3.29 +ebooklib==0.18 +effdet==0.4.1 +fqdn==1.5.1 +httptools==0.5.0 +importlib-resources==6.0.0 +isoduration==20.11.0 +jq==1.6.0 +jsonify==0.5 +jsonpointer==2.4 +langchainplus-sdk==0.0.17 +llama-index==0.7.14 +make-response==1 +markdown==3.4.3 +msg-parser==1.2.0 +nbdoc==0.0.82 +openapi-schema-pydantic==1.2.4 +openpyxl==3.1.2 +pinecone-client==2.2.2 +pip-chill==1.0.3 +pypandoc==1.11 +pypika==0.48.9 +pytesseract==0.3.10 +python-docx==0.8.11 +python-dotenv==1.0.0 +python-pptx==0.6.21 +pywatchman==1.4.1 +redis==4.6.0 +tokenize-rt==5.1.0 +typing==3.7.4.3 +unstructured==0.10.14 +unstructured-inference==0.5.27 +uri-template==1.3.0 +uvloop==0.17.0 +watchfiles==0.19.0 +webcolors==1.13 +websockets==11.0.3 +xlrd==2.0.1 From 3011e13561d98b58f379f822b49cf7b381d8ff8d Mon Sep 17 00:00:00 2001 From: serafettin Date: Sun, 17 Sep 2023 18:17:13 +0300 Subject: [PATCH 19/37] iterate --- Dockerfile | 2 +- app/document_parser.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9e49280d..1599a0f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.11.3 WORKDIR /code -RUN pip3 install numpy==1.24.2 fastapi==0.85.1 pydantic==1.10.11 llama_index==0.7.14 chromadb==0.3.29 openai redis --no-cache-dir +RUN pip3 install numpy==1.24.2 fastapi==0.85.1 pydantic==1.10.11 llama_index==0.7.14 chromadb==0.4.10 "unstructured[all-docs]" openai redis --no-cache-dir COPY ./app /code/app CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"] diff --git a/app/document_parser.py b/app/document_parser.py index 43a6a2fa..d038d1ae 100644 --- a/app/document_parser.py +++ b/app/document_parser.py @@ -1,9 +1,7 @@ import mimetypes import os import requests -import json import uuid -import jq from unstructured.partition.auto import partition from langchain.document_loaders import JSONLoader, UnstructuredFileLoader From 47c3698a478561036d756c31ed7cb45e0e6821f8 Mon Sep 17 00:00:00 2001 From: serafettin Date: Sun, 17 Sep 2023 18:31:41 +0300 Subject: [PATCH 20/37] Update Dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1599a0f6..d863cae7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM python:3.11.3 WORKDIR /code -RUN pip3 install numpy==1.24.2 fastapi==0.85.1 pydantic==1.10.11 llama_index==0.7.14 chromadb==0.4.10 "unstructured[all-docs]" openai redis --no-cache-dir +RUN pip3 install numpy==1.24.2 fastapi==0.85.1 pydantic==1.10.11 llama_index==0.7.14 chromadb==0.4.10 uuid openai redis --no-cache-dir +RUN pip3 install "unstructured[all-docs]" COPY ./app /code/app CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"] From 7ff113477db13b2ac0cc6459186dcf3ab0f49f3a Mon Sep 17 00:00:00 2001 From: serafettin Date: Sun, 17 Sep 2023 18:36:04 +0300 Subject: [PATCH 21/37] iterate --- Dockerfile | 2 +- requirements.txt | 39 --------------------------------------- 2 files changed, 1 insertion(+), 40 deletions(-) delete mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index d863cae7..790a99fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.11.3 WORKDIR /code -RUN pip3 install numpy==1.24.2 fastapi==0.85.1 pydantic==1.10.11 llama_index==0.7.14 chromadb==0.4.10 uuid openai redis --no-cache-dir +RUN pip3 install numpy==1.24.2 fastapi==0.99.1 pydantic==1.10.11 llama_index==0.7.14 chromadb==0.4.10 uuid openai redis --no-cache-dir RUN pip3 install "unstructured[all-docs]" COPY ./app /code/app diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 6c075cb5..00000000 --- a/requirements.txt +++ /dev/null @@ -1,39 +0,0 @@ -chroma-hnswlib==0.7.1 -chromadb==0.3.29 -ebooklib==0.18 -effdet==0.4.1 -fqdn==1.5.1 -httptools==0.5.0 -importlib-resources==6.0.0 -isoduration==20.11.0 -jq==1.6.0 -jsonify==0.5 -jsonpointer==2.4 -langchainplus-sdk==0.0.17 -llama-index==0.7.14 -make-response==1 -markdown==3.4.3 -msg-parser==1.2.0 -nbdoc==0.0.82 -openapi-schema-pydantic==1.2.4 -openpyxl==3.1.2 -pinecone-client==2.2.2 -pip-chill==1.0.3 -pypandoc==1.11 -pypika==0.48.9 -pytesseract==0.3.10 -python-docx==0.8.11 -python-dotenv==1.0.0 -python-pptx==0.6.21 -pywatchman==1.4.1 -redis==4.6.0 -tokenize-rt==5.1.0 -typing==3.7.4.3 -unstructured==0.10.14 -unstructured-inference==0.5.27 -uri-template==1.3.0 -uvloop==0.17.0 -watchfiles==0.19.0 -webcolors==1.13 -websockets==11.0.3 -xlrd==2.0.1 From 4f521ecf82ef6a6e05501dd29bc5f0d31fc891f6 Mon Sep 17 00:00:00 2001 From: serafettin Date: Sun, 17 Sep 2023 18:37:45 +0300 Subject: [PATCH 22/37] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 790a99fc..b20c17a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.11.3 WORKDIR /code -RUN pip3 install numpy==1.24.2 fastapi==0.99.1 pydantic==1.10.11 llama_index==0.7.14 chromadb==0.4.10 uuid openai redis --no-cache-dir +RUN pip3 install numpy==1.24.2 fastapi==0.99.1 pydantic==1.10.11 llama_index==0.8.28 chromadb==0.4.10 uuid openai redis --no-cache-dir RUN pip3 install "unstructured[all-docs]" COPY ./app /code/app From cd99edd164fd224bf4e59241e8e16acefa36046d Mon Sep 17 00:00:00 2001 From: serafettin Date: Sun, 17 Sep 2023 19:18:29 +0300 Subject: [PATCH 23/37] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b20c17a5..5c8d4229 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.3 +FROM python:3.11.5 WORKDIR /code RUN pip3 install numpy==1.24.2 fastapi==0.99.1 pydantic==1.10.11 llama_index==0.8.28 chromadb==0.4.10 uuid openai redis --no-cache-dir RUN pip3 install "unstructured[all-docs]" From 096c913d4acd4d2ae3e45b90143eca11757ce488 Mon Sep 17 00:00:00 2001 From: serafettin Date: Sun, 17 Sep 2023 19:42:41 +0300 Subject: [PATCH 24/37] Update document_parser.py --- app/document_parser.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/app/document_parser.py b/app/document_parser.py index d038d1ae..d1010b36 100644 --- a/app/document_parser.py +++ b/app/document_parser.py @@ -11,9 +11,7 @@ class Transformers: @staticmethod - def unstructured(url, file_type): - fileName = save_file(url, file_type) - print("Processing with unstructured...", url, file_type, fileName) + def unstructured(fileName, file_type): loader = UnstructuredFileLoader(file_path=fileName, mode="paged") return loader.load() @@ -24,9 +22,8 @@ def apify(url): return url @staticmethod - def langchainJSON(url, file_type): + def langchainJSON(fileName, file_type): print("Processing with Langchain JSON Loader...") - fileName = save_file(url, file_type) loader = JSONLoader(file_path=fileName, jq_schema='.', text_content=False) return loader.load() @@ -69,6 +66,15 @@ def save_file(url, file_type): return full_path + +def delete_file(file_path): + try: + os.remove(file_path) + except FileNotFoundError: + print(f"File {file_path} not found!") + except OSError as e: + print(f"Error deleting file {file_path}. Reason: {e}") + def guess_file_type(url_or_file_path): mime_type, encoding = mimetypes.guess_type(url_or_file_path) extension = url_or_file_path.split('.')[-1] if '.' in url_or_file_path else None @@ -119,4 +125,8 @@ def resolve_file(url): def get_document(url): transformer, file_type = resolve_file(url) - return transformer(url, file_type) + fileName = save_file(url, file_type) + print("Processing", url, file_type, transformer, fileName) + nodes = transformer(fileName, file_type) + delete_file(fileName) + return nodes From 877d42b9bd2f432fcd77486c24ea9ee99d2d0bf5 Mon Sep 17 00:00:00 2001 From: serafettin Date: Thu, 2 Nov 2023 18:28:37 +0300 Subject: [PATCH 25/37] Update main.py --- app/main.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index fc02ac2d..63182291 100644 --- a/app/main.py +++ b/app/main.py @@ -176,10 +176,21 @@ async def query(index_id, prompt: Prompt): def chat_stream(params: ChatStream): if params.did: - id_resp = redisClient.hkeys("user_indexes:by_did:" + params.did.lower()) + id_resp = redisClient.hgetall("user_indexes:by_did:" + params.did.lower()) if not id_resp: return "You have no indexes" - indexes = [item.decode('utf-8').split(':')[0] for item in id_resp] + + # Decode the keys and values from bytes to UTF-8 strings + decoded_id_resp = { + key.decode('utf-8').split(':')[0]: value.decode('utf-8') + for key, value in id_resp.items() + } + + indexes = [ + key + for key, value in decoded_id_resp.items() + if 'deletedAt' not in value or 'null' in value or 'None' in value + ] elif params.indexes: indexes = params.indexes From f3f4c7c448b998cbaa4616b380f61bdb836c21e3 Mon Sep 17 00:00:00 2001 From: serafettin Date: Thu, 2 Nov 2023 19:11:38 +0300 Subject: [PATCH 26/37] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5c8d4229..22df5bfe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.11.5 WORKDIR /code -RUN pip3 install numpy==1.24.2 fastapi==0.99.1 pydantic==1.10.11 llama_index==0.8.28 chromadb==0.4.10 uuid openai redis --no-cache-dir +RUN pip3 install numpy==1.24.2 fastapi==0.99.1 pydantic==1.10.11 llama_index==0.8.58 chromadb==0.4.10 uuid openai redis --no-cache-dir RUN pip3 install "unstructured[all-docs]" COPY ./app /code/app From 5ab65e6ab4a34e71ba3408e4da0e51cc64f37d80 Mon Sep 17 00:00:00 2001 From: serafettin Date: Thu, 2 Nov 2023 19:13:18 +0300 Subject: [PATCH 27/37] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 22df5bfe..c38beb8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.11.5 WORKDIR /code RUN pip3 install numpy==1.24.2 fastapi==0.99.1 pydantic==1.10.11 llama_index==0.8.58 chromadb==0.4.10 uuid openai redis --no-cache-dir -RUN pip3 install "unstructured[all-docs]" +RUN pip3 install "unstructured[all-docs]==0.10.28" COPY ./app /code/app CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"] From c97977037311051d464c49465c6e9063ddb962b0 Mon Sep 17 00:00:00 2001 From: serafettin Date: Thu, 2 Nov 2023 19:30:53 +0300 Subject: [PATCH 28/37] Update Dockerfile --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c38beb8f..4b3bfca7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ FROM python:3.11.5 WORKDIR /code -RUN pip3 install numpy==1.24.2 fastapi==0.99.1 pydantic==1.10.11 llama_index==0.8.58 chromadb==0.4.10 uuid openai redis --no-cache-dir -RUN pip3 install "unstructured[all-docs]==0.10.28" +RUN pip3 install numpy==1.24.2 fastapi==0.99.1 pydantic==1.10.11 llama_index==0.8.28 chromadb==0.4.10 uuid openai redis --no-cache-dir +RUN pip3 install "unstructured[all-docs]" +RUN pip3 install opencv-python-headless COPY ./app /code/app CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"] From 892c75c4b9a3629a7468db682bee3b45d33a0724 Mon Sep 17 00:00:00 2001 From: serafettin Date: Thu, 2 Nov 2023 19:56:07 +0300 Subject: [PATCH 29/37] Update main.py --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 63182291..715454d2 100644 --- a/app/main.py +++ b/app/main.py @@ -72,7 +72,7 @@ def get_service_context(): embed_model = OpenAIEmbedding(model="text-embedding-ada-002", openai_api_key=os.environ["OPENAI_API_KEY"]) - llm = ChatOpenAI(temperature=0, model_name="gpt-4", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) + llm = ChatOpenAI(temperature=0, model_name="gpt-3.5-turbo", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model) return service_context From 6f43d6a9bab512a31692e940cb82a464b5b0dc91 Mon Sep 17 00:00:00 2001 From: serafettin Date: Thu, 2 Nov 2023 20:14:48 +0300 Subject: [PATCH 30/37] Update main.py --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 715454d2..63182291 100644 --- a/app/main.py +++ b/app/main.py @@ -72,7 +72,7 @@ def get_service_context(): embed_model = OpenAIEmbedding(model="text-embedding-ada-002", openai_api_key=os.environ["OPENAI_API_KEY"]) - llm = ChatOpenAI(temperature=0, model_name="gpt-3.5-turbo", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) + llm = ChatOpenAI(temperature=0, model_name="gpt-4", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model) return service_context From c7fafb940c88ffa24139ed9791304c3bb69b4b92 Mon Sep 17 00:00:00 2001 From: serafettin Date: Fri, 3 Nov 2023 18:54:44 +0300 Subject: [PATCH 31/37] Update main.py --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 63182291..39d3cf2b 100644 --- a/app/main.py +++ b/app/main.py @@ -72,7 +72,7 @@ def get_service_context(): embed_model = OpenAIEmbedding(model="text-embedding-ada-002", openai_api_key=os.environ["OPENAI_API_KEY"]) - llm = ChatOpenAI(temperature=0, model_name="gpt-4", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) + llm = ChatOpenAI(temperature=0, model_name="gpt-3.5-turbo-16k", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model) return service_context From f608885cfdd94732bc521a0a65e5edea3ecc20cc Mon Sep 17 00:00:00 2001 From: serafettin Date: Tue, 7 Nov 2023 10:55:22 +0300 Subject: [PATCH 32/37] Update main.py --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 39d3cf2b..fcae3df0 100644 --- a/app/main.py +++ b/app/main.py @@ -72,7 +72,7 @@ def get_service_context(): embed_model = OpenAIEmbedding(model="text-embedding-ada-002", openai_api_key=os.environ["OPENAI_API_KEY"]) - llm = ChatOpenAI(temperature=0, model_name="gpt-3.5-turbo-16k", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) + llm = ChatOpenAI(temperature=0, model_name="gpt-4-1106-preview", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model) return service_context From b32f2925d9399ba4cb219eec345460ff64457cf3 Mon Sep 17 00:00:00 2001 From: serafettin Date: Tue, 7 Nov 2023 12:13:07 +0300 Subject: [PATCH 33/37] Update main.py --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index fcae3df0..39d3cf2b 100644 --- a/app/main.py +++ b/app/main.py @@ -72,7 +72,7 @@ def get_service_context(): embed_model = OpenAIEmbedding(model="text-embedding-ada-002", openai_api_key=os.environ["OPENAI_API_KEY"]) - llm = ChatOpenAI(temperature=0, model_name="gpt-4-1106-preview", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) + llm = ChatOpenAI(temperature=0, model_name="gpt-3.5-turbo-16k", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model) return service_context From a6192ba87453047e3ad3181422785e6ac6f06d71 Mon Sep 17 00:00:00 2001 From: serafettin Date: Tue, 7 Nov 2023 17:51:53 +0300 Subject: [PATCH 34/37] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4b3bfca7..f775e415 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.11.5 WORKDIR /code -RUN pip3 install numpy==1.24.2 fastapi==0.99.1 pydantic==1.10.11 llama_index==0.8.28 chromadb==0.4.10 uuid openai redis --no-cache-dir +RUN pip3 install numpy==1.24.2 fastapi==0.99.1 pydantic==1.10.11 langchain==0.0.326 llama_index==0.8.28 chromadb==0.4.10 uuid openai==0.28.0 redis --no-cache-dir RUN pip3 install "unstructured[all-docs]" RUN pip3 install opencv-python-headless From a4258f4dcba5456150d2cbaef7bc3a78ac0e2c28 Mon Sep 17 00:00:00 2001 From: serafettin Date: Tue, 7 Nov 2023 18:19:47 +0300 Subject: [PATCH 35/37] Update main.py --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 39d3cf2b..fcae3df0 100644 --- a/app/main.py +++ b/app/main.py @@ -72,7 +72,7 @@ def get_service_context(): embed_model = OpenAIEmbedding(model="text-embedding-ada-002", openai_api_key=os.environ["OPENAI_API_KEY"]) - llm = ChatOpenAI(temperature=0, model_name="gpt-3.5-turbo-16k", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) + llm = ChatOpenAI(temperature=0, model_name="gpt-4-1106-preview", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model) return service_context From dafd7d2b5352abb5916e3f0c8f9d611354b643bf Mon Sep 17 00:00:00 2001 From: serafettin Date: Tue, 7 Nov 2023 18:49:23 +0300 Subject: [PATCH 36/37] Update main.py --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index fcae3df0..39d3cf2b 100644 --- a/app/main.py +++ b/app/main.py @@ -72,7 +72,7 @@ def get_service_context(): embed_model = OpenAIEmbedding(model="text-embedding-ada-002", openai_api_key=os.environ["OPENAI_API_KEY"]) - llm = ChatOpenAI(temperature=0, model_name="gpt-4-1106-preview", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) + llm = ChatOpenAI(temperature=0, model_name="gpt-3.5-turbo-16k", openai_api_key=os.environ["OPENAI_API_KEY"], streaming=True) service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model) return service_context From e505fca522a6689073f533c762617384ffd26f2c Mon Sep 17 00:00:00 2001 From: seref Date: Wed, 10 Jan 2024 22:12:43 +0300 Subject: [PATCH 37/37] Update build.yaml --- .github/workflows/build.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 919df841..6b7961ab 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -56,15 +56,15 @@ jobs: - name: Build, tag, and push image to Amazon ECR env: - DOCKER_TAG: indexas/llm-indexer:${{ steps.build-time.outputs.time }} - DOCKER_REGISTRY: 534970752686.dkr.ecr.us-east-1.amazonaws.com + DOCKER_TAG: indexnetwork/llm-indexer:${{ steps.build-time.outputs.time }} + DOCKER_REGISTRY: 236785930124.dkr.ecr.us-east-1.amazonaws.com run: | docker build -t $DOCKER_TAG . docker tag $DOCKER_TAG $DOCKER_REGISTRY/$DOCKER_TAG docker push $DOCKER_REGISTRY/$DOCKER_TAG - docker tag $DOCKER_TAG $DOCKER_REGISTRY/indexas/llm-indexer:latest-${GITHUB_REF#refs/heads/} - docker push $DOCKER_REGISTRY/indexas/llm-indexer:latest-${GITHUB_REF#refs/heads/} + docker tag $DOCKER_TAG $DOCKER_REGISTRY/indexnetwork/llm-indexer:latest-${GITHUB_REF#refs/heads/} + docker push $DOCKER_REGISTRY/indexnetwork/llm-indexer:latest-${GITHUB_REF#refs/heads/} - name: Deploy run: |- - kubectl set image deployment/llm-indexer llm-indexer=534970752686.dkr.ecr.us-east-1.amazonaws.com/indexas/llm-indexer:${{ steps.build-time.outputs.time }} --namespace ${GITHUB_REF#refs/heads/} + kubectl set image deployment/llm-indexer llm-indexer=236785930124.dkr.ecr.us-east-1.amazonaws.com/indexnetwork/llm-indexer:${{ steps.build-time.outputs.time }} --namespace env-${GITHUB_REF#refs/heads/}