From 101c43e66ca1897287f41537cbdbf79a451399d9 Mon Sep 17 00:00:00 2001 From: Harley Trung Date: Thu, 18 May 2023 15:09:35 +0700 Subject: [PATCH 1/3] can stream response --- content/overview.md | 9 +- render_body.py | 5 +- render_chat_form.py | 82 ++++++++- sample-stream.json | 420 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 508 insertions(+), 8 deletions(-) create mode 100644 sample-stream.json diff --git a/content/overview.md b/content/overview.md index 9c1067c..ae23d9a 100644 --- a/content/overview.md +++ b/content/overview.md @@ -4,12 +4,11 @@ Thanks for trying this out. It's super alpha, but each person can log in via Goo Limitations: -- you have to log in again if you reload the page (need to work around streamlit later) -- openai api gives rate limit errors sometimes, causing crashes -- gpt4 feels slower than ai.com naturally -- can't delete personal conversations yet. don't put anything sensitive there. +- **you have to log in again if you reload** the page (streamlit's limitation; work around later) +- openai api response can be slow, with rate limit errors sometimes (chat.openai.com has advantage) +- can only delete all chat history, not individual conversation yet (contribution welcome!) How to help: - report bugs to Hien or myself. -- run your code locally https://github.com/CoderPush/chatlit by getting your own keys at .env \ No newline at end of file +- run your code locally https://github.com/CoderPush/chatlit by asking me for .env file \ No newline at end of file diff --git a/render_body.py b/render_body.py index 8c50bf9..2746699 100644 --- a/render_body.py +++ b/render_body.py @@ -1,5 +1,5 @@ from render_conversation import render_conversation -from render_chat_form import render_chat_form +from render_chat_form import render_chat_form, render_chat_stream from firestore_utils import get_firestore_db from utils import get_cid_from_params from firestore_utils import clear_user_history @@ -52,7 +52,8 @@ def render_body(st): render_conversation(st, conversation) if st.session_state.get("user_info"): - render_chat_form(st) + # render_chat_form(st) + render_chat_stream(st) else: # load homepage.md into a string with open("content/overview.md", "r") as f: diff --git a/render_chat_form.py b/render_chat_form.py index 3707bbc..4297066 100644 --- a/render_chat_form.py +++ b/render_chat_form.py @@ -43,7 +43,7 @@ def generate_response(st, prompt): return messages, usage -def save_to_firestore(st, messages, usage): +def save_to_firestore(st, messages, usage=None): model = st.session_state["model"] if len(messages) > 0: conversation = st.session_state.get("conversation", {}) @@ -85,3 +85,83 @@ def render_chat_form(st): if new_conversation is not None: st.experimental_set_query_params(cid=new_conversation.id) st.experimental_rerun() + +# see sample-stream.json to know how to parse it +def generate_stream(st, holder, user_input): + model = st.session_state["model"] + messages = load_messages(st) + messages.append({"role": "user", "content": user_input}) + + print("openai.ChatCompletion.create with", model, messages) + completion = openai.ChatCompletion.create( + model=model, + messages=messages, + stream=True, + ) + + # first chunk should be + # { + # "choices": [ + # { + # "delta": { + # "role": "assistant" + # }, + # "finish_reason": null, + # "index": 0 + # } + # ], + # "created": 1684389483, + # "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + # "model": "gpt-3.5-turbo-0301", + # "object": "chat.completion.chunk" + # } + + # middle chunks are content: + content = "" + for chunk in completion: + delta = chunk["choices"][0]["delta"] + if "content" in delta: + content += delta["content"] + holder.markdown(content) + + # last chunk should be + # { + # "choices": [ + # { + # "delta": {}, + # "finish_reason": "stop", + # "index": 0 + # } + # ], + # "created": 1684389483, + # "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + # "model": "gpt-3.5-turbo-0301", + # "object": "chat.completion.chunk" + # } + + messages.append({"role": "assistant", "content": content}) + + # No usage info in stream mode yet + # https://community.openai.com/t/usage-info-in-api-responses/18862 + + return messages + + +def render_chat_stream(st): + name = st.session_state.get("user_info", {}).get("name", "You") + + with st.form(key="chat_prompt", clear_on_submit=True): + holder = st.empty() + user_input = st.text_area( + f"{name}:", key="text_area_stream", label_visibility="collapsed" + ) + submit_button = st.form_submit_button(label="Send") + + if submit_button and user_input: + messages = generate_stream(st, holder, user_input) + # print("messages: ", messages) + new_conversation = save_to_firestore(st, messages) + if new_conversation is not None: + st.experimental_set_query_params(cid=new_conversation.id) + st.experimental_rerun() + diff --git a/sample-stream.json b/sample-stream.json new file mode 100644 index 0000000..ca88c5f --- /dev/null +++ b/sample-stream.json @@ -0,0 +1,420 @@ +[ + { + "choices": [ + { + "delta": { + "role": "assistant" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": "Sure" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": "," + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " here" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": "'s" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " a" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " joke" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " for" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " you" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": ":" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " Why" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " did" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " the" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " scare" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": "crow" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " win" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " an" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " award" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": "?" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " Because" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " he" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " was" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " outstanding" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " in" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " his" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": " field" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": { + "content": "!" + }, + "finish_reason": null, + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + }, + { + "choices": [ + { + "delta": {}, + "finish_reason": "stop", + "index": 0 + } + ], + "created": 1684389483, + "id": "chatcmpl-7HQwF5QPvTrDtYPOvBZbzFfDb9tcI", + "model": "gpt-3.5-turbo-0301", + "object": "chat.completion.chunk" + } +] From e6546a1262dbeb093c7e2097a3984d8ae3c587ad Mon Sep 17 00:00:00 2001 From: Harley Trung Date: Thu, 18 May 2023 15:20:50 +0700 Subject: [PATCH 2/3] support streaming in chat --- render_conversation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/render_conversation.py b/render_conversation.py index afe4267..f5f9b2b 100644 --- a/render_conversation.py +++ b/render_conversation.py @@ -26,6 +26,7 @@ def render_messages(st, messages): def render_conversation(st, conversation): - messages = conversation["messages"] + with st.container(): + messages = conversation["messages"] render_messages(st, messages) From 383b0886b773ac4e94cd5b823082bfc9d052538b Mon Sep 17 00:00:00 2001 From: Harley Trung Date: Thu, 18 May 2023 17:48:43 +0700 Subject: [PATCH 3/3] move profile setting to sidebar --- chat.py | 29 +++++++++++++++++++++++ render_body.py | 53 ++++++++++++------------------------------ render_chat_form.py | 45 +++++++++++++++++++++-------------- render_conversation.py | 4 +--- utils.py | 15 ++++++++++++ 5 files changed, 87 insertions(+), 59 deletions(-) diff --git a/chat.py b/chat.py index c3894a4..7392a7b 100644 --- a/chat.py +++ b/chat.py @@ -4,6 +4,7 @@ from render_body import render_body from render_my_conversations import render_my_conversations import streamlit as st +from firestore_utils import clear_user_history from dotenv import load_dotenv @@ -29,6 +30,7 @@ def load_and_store_conversation(st, cid: str): def controller(): # TODO: display useful total cost in $ st.session_state["total_cost"] = 0.0 + st.session_state["conversation_expanded"] = True # set model in session if specified in params model_from_param = get_key_from_params(st, "model") @@ -88,9 +90,36 @@ def render_history_menu(sidebar): render_my_conversations(st, sidebar) +def render_profile(sidebar): + user_info = st.session_state.get("user_info") + if not user_info: + return + + status = f"Signed in as {user_info.get('email')}" + with sidebar.expander(status): + st.image(user_info.get("picture"), width=50) + signout = st.button("Sign out", key="button_signout", type="primary") + if signout: + st.session_state.clear() + st.experimental_rerun() + st.write( + "While it's useful to resume past conversations, sometimes you may want to clear your chat history." + ) + placeholder = st.empty() + with placeholder: + clear_history = st.button( + "Clear History", key="button_clear_history", type="primary" + ) + if clear_history: + clear_user_history(user_info["id"]) + placeholder.info("Chat history cleared", icon="✅") + st.snow() + + def render_sidebar(sidebar): render_new_chat(sidebar) render_auth(st) + render_profile(sidebar) sidebar.divider() render_history_menu(sidebar) diff --git a/render_body.py b/render_body.py index 2746699..e6d70df 100644 --- a/render_body.py +++ b/render_body.py @@ -1,11 +1,10 @@ from render_conversation import render_conversation from render_chat_form import render_chat_form, render_chat_stream from firestore_utils import get_firestore_db -from utils import get_cid_from_params -from firestore_utils import clear_user_history +from utils import get_cid_from_params, get_expander_text -def load_conversation(st): +def load_conversation_from_db(st): cid = get_cid_from_params(st) if cid: db = get_firestore_db() @@ -14,46 +13,24 @@ def load_conversation(st): return {} -def get_expander_text(st): - user = st.session_state.get("user_info", {}).get("name", None) - model = st.session_state.get("model") - if user: - text = f"### {model} with {user}" - else: - text = f"### {model}" - return text +def load_conversation_from_session_state(st): + return st.session_state.get("conversation", {}) def render_body(st): - with st.expander(get_expander_text(st)): - user_info = st.session_state.get("user_info") - if user_info: - st.write(f"Signed in as {user_info.get('email')}") - st.image(user_info.get("picture"), width=50) - signout = st.button("Sign out", key="button_signout", type="primary") - if signout: - st.session_state.clear() - st.experimental_rerun() - st.write( - "While it's useful to resume past conversations, sometimes you may want to clear your chat history." - ) - placeholder = st.empty() - with placeholder: - clear_history = st.button( - "Clear History", key="button_clear_history", type="primary" - ) - if clear_history: - clear_user_history(user_info["id"]) - placeholder.info("Chat history cleared", icon="✅") - st.snow() - - conversation = load_conversation(st) - if conversation: - render_conversation(st, conversation) + messages_holder = st.expander( + get_expander_text(st), expanded=st.session_state["conversation_expanded"] + ) + with messages_holder: + # load_conversation from session_state + conversation = load_conversation_from_session_state(st) + if conversation: + render_conversation(st, conversation) if st.session_state.get("user_info"): - # render_chat_form(st) - render_chat_stream(st) + with st.container(): + # render_chat_form(st) + render_chat_stream(st) else: # load homepage.md into a string with open("content/overview.md", "r") as f: diff --git a/render_chat_form.py b/render_chat_form.py index 4297066..f78ac16 100644 --- a/render_chat_form.py +++ b/render_chat_form.py @@ -1,13 +1,21 @@ import openai from firestore_utils import firestore_save from utils import generate_conversation_title, get_oauth_uid +import time -def load_messages(st): +def extract_messages(st): + default = [ + { + "role": "system", + "content": "You are a helpful assistant. Please use concise language to save bandwidth and token usage. Avoid 'AI language model' disclaimer whenever possible.", + } + ] + conversation = st.session_state.get("conversation", {}) - default = [{"role": "system", "content": "You are a helpful assistant."}] + messages = conversation.get("messages", default) - return conversation.get("messages", default) + return messages def get_content(st, response): @@ -23,7 +31,7 @@ def get_content(st, response): def generate_response(st, prompt): model = st.session_state["model"] - messages = load_messages(st) + messages = extract_messages(st) messages.append({"role": "user", "content": prompt}) print("openai.ChatCompletion.create with") @@ -69,6 +77,7 @@ def save_to_firestore(st, messages, usage=None): return new_conversation +# non-streaming version, with usage def render_chat_form(st): name = st.session_state.get("user_info", {}).get("name", "You") model = st.session_state["model"] @@ -86,10 +95,11 @@ def render_chat_form(st): st.experimental_set_query_params(cid=new_conversation.id) st.experimental_rerun() + # see sample-stream.json to know how to parse it def generate_stream(st, holder, user_input): model = st.session_state["model"] - messages = load_messages(st) + messages = extract_messages(st) messages.append({"role": "user", "content": user_input}) print("openai.ChatCompletion.create with", model, messages) @@ -117,12 +127,13 @@ def generate_stream(st, holder, user_input): # } # middle chunks are content: - content = "" - for chunk in completion: - delta = chunk["choices"][0]["delta"] - if "content" in delta: - content += delta["content"] - holder.markdown(content) + with holder.container(): + content = "" + for chunk in completion: + delta = chunk["choices"][0]["delta"] + if "content" in delta: + content += delta["content"] + holder.markdown(content) # last chunk should be # { @@ -148,20 +159,18 @@ def generate_stream(st, holder, user_input): def render_chat_stream(st): - name = st.session_state.get("user_info", {}).get("name", "You") - with st.form(key="chat_prompt", clear_on_submit=True): - holder = st.empty() + stream_holder = st.empty() user_input = st.text_area( - f"{name}:", key="text_area_stream", label_visibility="collapsed" + f"You:", key="text_area_stream", label_visibility="collapsed" ) submit_button = st.form_submit_button(label="Send") if submit_button and user_input: - messages = generate_stream(st, holder, user_input) + st.session_state["conversation_expanded"] = False + messages = generate_stream(st, stream_holder, user_input) # print("messages: ", messages) new_conversation = save_to_firestore(st, messages) if new_conversation is not None: st.experimental_set_query_params(cid=new_conversation.id) - st.experimental_rerun() - + st.experimental_rerun() diff --git a/render_conversation.py b/render_conversation.py index f5f9b2b..c1059dc 100644 --- a/render_conversation.py +++ b/render_conversation.py @@ -26,7 +26,5 @@ def render_messages(st, messages): def render_conversation(st, conversation): - with st.container(): - messages = conversation["messages"] - + messages = conversation["messages"] render_messages(st, messages) diff --git a/utils.py b/utils.py index 515f82a..5e6c1dd 100644 --- a/utils.py +++ b/utils.py @@ -84,3 +84,18 @@ def get_cid_from_params(st): def get_oauth_uid(st): user_info = st.session_state.get("user_info", {}) return user_info.get("id", None) + + +def get_expander_text(st): + user = st.session_state.get("user_info", {}).get("name", None) + model = st.session_state.get("model") + messages = st.session_state.get("conversation", {}).get("messages", []) + user_messages = [m for m in messages if m.get("role") == "user"] + if user: + text = f"### {model} with {user}" + else: + text = f"### {model}" + + if len(messages) > 0: + text += f" ({len(user_messages)} messages)" + return text