Add Retry Logic to http requests #151
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,6 +12,7 @@ __pycache__/ | |
| *.pyc | ||
| *.pyo | ||
| .python-version | ||
| api/api.egg-info/* | ||
|
|
||
| # Visual Studio Code | ||
| .vscode/ | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,82 +1,119 @@ | ||
| from fastapi import APIRouter, HTTPException, BackgroundTasks | ||
| from fastapi.responses import StreamingResponse, JSONResponse | ||
|
|
||
| from api import schemas | ||
| from api.dependencies import app, honcho | ||
|
|
||
| from agent.chain import ThinkCall, RespondCall | ||
|
|
||
| import logging | ||
|
|
||
| router = APIRouter(prefix="/api", tags=["chat"]) | ||
|
|
||
|
|
||
| @router.post("/stream") | ||
| async def stream(inp: schemas.ConversationInput, background_tasks: BackgroundTasks): | ||
| """Stream the response to the user, currently only used by the Web UI and has integration to be able to use Honcho if not anonymous""" | ||
| try: | ||
| user = honcho.apps.users.get_or_create(app_id=app.id, name=inp.user_id) | ||
|
|
||
| def convo_turn(): | ||
| thought = "" | ||
| response = "" | ||
| try: | ||
| thought_stream = ThinkCall( | ||
| user_input=inp.message, | ||
| app_id=app.id, | ||
| user_id=user.id, | ||
| session_id=str(inp.conversation_id), | ||
| honcho=honcho, | ||
| ).stream() | ||
| for chunk in thought_stream: | ||
| thought += chunk | ||
| yield chunk | ||
|
|
||
| yield "❀" | ||
| response_stream = RespondCall( | ||
| user_input=inp.message, | ||
| thought=thought, | ||
| app_id=app.id, | ||
| user_id=user.id, | ||
| session_id=str(inp.conversation_id), | ||
| honcho=honcho, | ||
| ).stream() | ||
| for chunk in response_stream: | ||
| response += chunk | ||
| yield chunk | ||
| yield "❀" | ||
| except Exception as e: | ||
| logging.error(f"Error during streaming: {str(e)}") | ||
| yield f"Error: {str(e)}" | ||
| return | ||
|
|
||
| background_tasks.add_task( | ||
| create_messages_and_metamessages, | ||
| app.id, user.id, inp.conversation_id, inp.message, thought, response | ||
| ) | ||
|
There was a problem hiding this comment. See comment about concerns with this approach There was a problem hiding this comment. Ok, something like this perhaps? Handling it async in the try/except should add logging for us and properly utilize the retry logic from the client with the http errors. |
||
|
|
||
| return StreamingResponse(convo_turn()) | ||
| except Exception as e: | ||
| logging.error(f"An error occurred: {str(e)}") | ||
|
There was a problem hiding this comment. Might just be I don't know how these work, but is there a change that one of these errors could occur after the generator is finished and the messages and metamessages are created? Unclear on the relationship between the try catch block and the Streaming Response method . If there is an error in the middle of the stream what happens? There was a problem hiding this comment. Very good questions! My latest commit employs a background process for the honcho calls to separately handle/log any potential errors while (ideally) not interfering with the There was a problem hiding this comment. My worry with this approach is that if there is an error while saving a message to honcho then that is not propagated to the front-end / user. Meaning it will look like their message sent and the conversation is fine, but if they reload the messages will be gone without any indication of an error occurring. It might make sense to make separate try catch blocks or separate Exceptions for the different types of errors that the LLM vs the honcho calls are making and report them to the user differently, without using the background tasks |
||
| if "rate limit" in str(e).lower(): | ||
| return JSONResponse( | ||
| status_code=429, | ||
| content={"error": "rate_limit_exceeded", "message": "Rate limit exceeded. Please try again later."} | ||
| ) | ||
| else: | ||
| return JSONResponse( | ||
| status_code=500, | ||
| content={"error": "internal_server_error", "message": "An internal server error has occurred."} | ||
| ) | ||
|
|
||
|
|
||
| @router.get("/thought/{message_id}") | ||
| async def get_thought(conversation_id: str, message_id: str, user_id: str): | ||
| try: | ||
| user = honcho.apps.users.get_or_create(app_id=app.id, name=user_id) | ||
| thought = honcho.apps.users.sessions.metamessages.list( | ||
| session_id=conversation_id, | ||
| app_id=app.id, | ||
| user_id=user.id, | ||
| message_id=message_id, | ||
| metamessage_type="thought", | ||
| ) | ||
| # In practice, there should only be one thought per message | ||
| return {"thought": thought.items[0].content if thought.items else None} | ||
| except Exception as e: | ||
| logging.error(f"An error occurred: {str(e)}") | ||
| return JSONResponse( | ||
| status_code=500, | ||
| content={"error": "internal_server_error", "message": "An internal server error has occurred."} | ||
| ) | ||
|
|
||
|
|
||
| def create_messages_and_metamessages(app_id, user_id, conversation_id, user_message, thought, ai_response): | ||
| try: | ||
| honcho.apps.users.sessions.messages.create( | ||
| is_user=True, | ||
| session_id=str(conversation_id), | ||
| app_id=app_id, | ||
| user_id=user_id, | ||
| content=user_message, | ||
| ) | ||
| new_ai_message = honcho.apps.users.sessions.messages.create( | ||
| is_user=False, | ||
| session_id=str(conversation_id), | ||
| app_id=app_id, | ||
| user_id=user_id, | ||
| content=ai_response, | ||
| ) | ||
| honcho.apps.users.sessions.metamessages.create( | ||
| app_id=app_id, | ||
| session_id=str(conversation_id), | ||
| user_id=user_id, | ||
| message_id=new_ai_message.id, | ||
| metamessage_type="thought", | ||
| content=thought, | ||
| ) | ||
| except Exception as e: | ||
| logging.error(f"Error in background task: {str(e)}") | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Only nitpick is you can probably get rid of the background tasks import here, but looks good now