Merge pull request #79 from trubrics/st-feedback-streaming

St feedback streaming

README.md

@@ -108,7 +108,6 @@ if st.session_state.logged_prompt:
         model=st.session_state.logged_prompt.config_model.model,
         prompt_id=st.session_state.logged_prompt.id,
         align="flex-start",
-        single_submit=False
     )
 ```
 

docs/integrations/streamlit.md

@@ -90,7 +90,6 @@ if st.session_state.logged_prompt:
         model=st.session_state.logged_prompt.config_model.model,
         prompt_id=st.session_state.logged_prompt.id,
         align="flex-start",
-        single_submit=False
     )
 ```
 

examples/streamlit/basic_app.py

@@ -23,5 +23,4 @@
         model=st.session_state.logged_prompt.config_model.model,
         prompt_id=st.session_state.logged_prompt.id,
         align="flex-start",
-        single_submit=False,
     )

examples/streamlit/llm_app.py

@@ -6,6 +6,8 @@
 
 if "response" not in st.session_state:
     st.session_state.response = ""
+if "feedback_key" not in st.session_state:
+    st.session_state.feedback_key = 0
 if "logged_prompt" not in st.session_state:
     st.session_state.logged_prompt = ""
 
@@ -15,13 +17,17 @@
     email, password = trubrics_config()
 
 if email and password:
-    collector = FeedbackCollector(
-        project="default",
-        email=email,
-        password=password,
-    )
+    try:
+        collector = FeedbackCollector(email=email, password=password, project="default")
+    except Exception:
+        st.error(f"Error authenticating '{email}' with [Trubrics](https://trubrics.streamlit.app/). Please try again.")
+        st.stop()
 else:
-    st.warning("To save some feedback to Trubrics, add your account details in the sidebar.")
+    st.info(
+        "To ask a question to an LLM and save your feedback to Trubrics, add your email and password in the sidebar."
+        " Don't have an account yet? Create one for free [here](https://trubrics.streamlit.app/)!"
+    )
+    st.stop()
 
 models = ("gpt-3.5-turbo",)
 model = st.selectbox(
@@ -32,7 +38,8 @@
 
 openai.api_key = st.secrets.get("OPENAI_API_KEY")
 if openai.api_key is None:
-    raise ValueError("OpenAI key is missing. Set OPENAI_API_KEY in st.secrets")
+    st.info("Please add your OpenAI API key to continue.")
+    st.stop()
 
 prompt = st.text_area(label="Prompt", label_visibility="collapsed", placeholder="What would you like to know?")
 button = st.button(f"Ask {model}")
@@ -41,9 +48,10 @@
     response = openai.ChatCompletion.create(model=model, messages=[{"role": "user", "content": prompt}])
     response_text = response.choices[0].message["content"]
     st.session_state.logged_prompt = collector.log_prompt(
-        config_model={"model": model}, prompt=prompt, generation=response_text, tags=["llm_app.py"]
+        config_model={"model": model}, prompt=prompt, generation=response_text, tags=["llm_app.py"], user_id=email
     )
     st.session_state.response = response_text
+    st.session_state.feedback_key += 1
 
 if st.session_state.response:
     st.markdown(f"#### :violet[{st.session_state.response}]")
@@ -55,8 +63,9 @@
         prompt_id=st.session_state.logged_prompt.id,
         model=model,
         align="flex-start",
-        single_submit=False,
         tags=["llm_app.py"],
+        key=f"feedback_{st.session_state.feedback_key}",  # overwrite with new key
+        user_id=email,
     )
 
     if feedback:

examples/streamlit/llm_chatbot.py

@@ -6,20 +6,31 @@
 
 from trubrics.integrations.streamlit import FeedbackCollector
 
+st.title("💬 Trubrics - Chat with user feedback")
+
 with st.sidebar:
     email, password = trubrics_config()
 
+if not email or not password:
+    st.info(
+        "To chat with an LLM and save your feedback to Trubrics, add your email and password in the sidebar."
+        " Don't have an account yet? Create one for free [here](https://trubrics.streamlit.app/)!"
+    )
+    st.stop()
+
 
 @st.cache_data
 def init_trubrics(email, password):
-    collector = FeedbackCollector(email=email, password=password, project="default")
-    return collector
+    try:
+        collector = FeedbackCollector(email=email, password=password, project="default")
+        return collector
+    except Exception:
+        st.error(f"Error authenticating '{email}' with [Trubrics](https://trubrics.streamlit.app/). Please try again.")
+        st.stop()
 
 
 collector = init_trubrics(email, password)
 
-
-st.title("💬 Trubrics - Chat with user feedback")
 if "messages" not in st.session_state:
     st.session_state["messages"] = [{"role": "assistant", "content": "How can I help you?"}]
 if "prompt_ids" not in st.session_state:
@@ -30,46 +41,66 @@ def init_trubrics(email, password):
 model = st.secrets.get("OPENAI_API_MODEL") or "gpt-3.5-turbo"
 
 openai_api_key = st.secrets.get("OPENAI_API_KEY")
+
+messages = st.session_state.messages
+feedback_kwargs = {
+    "component": "default",
+    "feedback_type": "thumbs",
+    "open_feedback_label": "[Optional] Provide additional feedback",
+    "model": model,
+    "tags": ["llm_chatbot.py"],
+}
+
+for n, msg in enumerate(messages):
+    st.chat_message(msg["role"]).write(msg["content"])
+
+    if msg["role"] == "assistant" and n > 1:
+        feedback_key = f"feedback_{int(n/2)}"
+
+        if feedback_key not in st.session_state:
+            st.session_state[feedback_key] = None
+
+        disable_with_score = st.session_state[feedback_key].get("score") if st.session_state[feedback_key] else None
+        feedback = collector.st_feedback(
+            **feedback_kwargs,
+            key=feedback_key,
+            disable_with_score=disable_with_score,
+            prompt_id=st.session_state.prompt_ids[int(n / 2) - 1],
+            user_id=email,
+        )
+        if feedback:
+            trubrics_successful_feedback(feedback)
+
+
 if prompt := st.chat_input():
+    messages.append({"role": "user", "content": prompt})
+    st.chat_message("user").write(prompt)
     if not openai_api_key:
         st.info("Please add your OpenAI API key to continue.")
         st.stop()
     else:
         openai.api_key = openai_api_key
-
-    st.session_state.messages.append({"role": "user", "content": prompt})
-    response = openai.ChatCompletion.create(model=model, messages=st.session_state.messages)
-    msg = response.choices[0].message
-    logged_prompt = collector.log_prompt(
-        config_model={"model": model},
-        prompt=prompt,
-        generation=msg["content"],
-        session_id=st.session_state.session_id,
-        tags=["llm_chatbot.py"],
+    response = openai.ChatCompletion.create(model=model, messages=messages)
+    generation = response.choices[0].message.content
+
+    with st.chat_message("assistant"):
+        logged_prompt = collector.log_prompt(
+            config_model={"model": model},
+            prompt=prompt,
+            generation=generation,
+            session_id=st.session_state.session_id,
+            tags=["llm_chatbot.py"],
+            user_id=email,
+        )
+        st.session_state.prompt_ids.append(logged_prompt.id)
+        messages.append({"role": "assistant", "content": generation})
+        st.write(generation)
+
+    feedback = collector.st_feedback(
+        **feedback_kwargs,
+        key=f"feedback_{int(len(messages)/2)}",
+        prompt_id=st.session_state.prompt_ids[int(len(messages) / 2) - 1],
+        user_id=email,
     )
-    st.session_state.prompt_ids.append(logged_prompt.id)
-    st.session_state.messages.append(msg)
-
-feedback = None
-for n, msg in enumerate(st.session_state.messages):
-    st.chat_message(msg["role"]).write(msg["content"])
-
-    if msg["role"] == "assistant" and msg["content"] != "How can I help you?":
-        if email and password:
-            feedback = collector.st_feedback(
-                component="default",
-                feedback_type="thumbs",
-                model=model,
-                prompt_id=st.session_state.prompt_ids[int(n / 2) - 1],
-                open_feedback_label="[Optional] Provide additional feedback",
-                align="flex-end",
-                single_submit=True,
-                key=f"feedback_{int(n/2)}",
-                tags=["llm_chatbot.py"],
-            )
-
-        else:
-            st.warning("To save some feedback to Trubrics, add your account details in the sidebar.")
-
-if feedback:
-    trubrics_successful_feedback(feedback)
+    if feedback:
+        trubrics_successful_feedback(feedback)

examples/streamlit/llm_chatbot_stream.py

@@ -0,0 +1,112 @@
+import uuid
+
+import openai
+import streamlit as st
+from trubrics_utils import trubrics_config, trubrics_successful_feedback
+
+from trubrics.integrations.streamlit import FeedbackCollector
+
+st.title("💬 Trubrics - Streaming chat with user feedback")
+
+with st.sidebar:
+    email, password = trubrics_config()
+
+if not email or not password:
+    st.info(
+        "To chat with an LLM and save your feedback to Trubrics, add your email and password in the sidebar."
+        " Don't have an account yet? Create one for free [here](https://trubrics.streamlit.app/)!"
+    )
+    st.stop()
+
+
+@st.cache_data
+def init_trubrics(email, password):
+    try:
+        collector = FeedbackCollector(email=email, password=password, project="default")
+        return collector
+    except Exception:
+        st.error(f"Error authenticating '{email}' with [Trubrics](https://trubrics.streamlit.app/). Please try again.")
+        st.stop()
+
+
+collector = init_trubrics(email, password)
+
+if "messages" not in st.session_state:
+    st.session_state["messages"] = [{"role": "assistant", "content": "How can I help you?"}]
+if "prompt_ids" not in st.session_state:
+    st.session_state["prompt_ids"] = []
+if "session_id" not in st.session_state:
+    st.session_state["session_id"] = str(uuid.uuid4())
+
+model = st.secrets.get("OPENAI_API_MODEL") or "gpt-3.5-turbo"
+
+openai_api_key = st.secrets.get("OPENAI_API_KEY")
+
+messages = st.session_state.messages
+feedback_kwargs = {
+    "component": "default",
+    "feedback_type": "thumbs",
+    "open_feedback_label": "[Optional] Provide additional feedback",
+    "model": model,
+    "tags": ["llm_chatbot.py"],
+}
+
+for n, msg in enumerate(st.session_state.messages):
+    st.chat_message(msg["role"]).write(msg["content"])
+
+    if msg["role"] == "assistant" and n > 1:
+        feedback_key = f"feedback_{int(n/2)}"
+
+        if feedback_key not in st.session_state:
+            st.session_state[feedback_key] = None
+
+        disable_with_score = st.session_state[feedback_key].get("score") if st.session_state[feedback_key] else None
+        feedback = collector.st_feedback(
+            **feedback_kwargs,
+            key=feedback_key,
+            disable_with_score=disable_with_score,
+            prompt_id=st.session_state.prompt_ids[int(n / 2) - 1],
+            user_id=email,
+        )
+        if feedback:
+            trubrics_successful_feedback(feedback)
+
+if prompt := st.chat_input("What is up?"):
+    st.session_state.messages.append({"role": "user", "content": prompt})
+    with st.chat_message("user"):
+        st.markdown(prompt)
+    with st.chat_message("assistant"):
+        message_placeholder = st.empty()
+        full_response = ""
+        if not openai_api_key:
+            st.info("Please add your OpenAI API key to continue.")
+            st.stop()
+        else:
+            openai.api_key = openai_api_key
+
+        for response in openai.ChatCompletion.create(
+            model=model,
+            messages=[{"role": m["role"], "content": m["content"]} for m in st.session_state.messages],
+            stream=True,
+        ):
+            full_response += response.choices[0].delta.get("content", "")
+            message_placeholder.markdown(full_response + "▌")
+        message_placeholder.markdown(full_response)
+    st.session_state.messages.append({"role": "assistant", "content": full_response})
+    logged_prompt = collector.log_prompt(
+        config_model={"model": model},
+        prompt=prompt,
+        generation=full_response,
+        session_id=st.session_state.session_id,
+        tags=["llm_chatbot.py"],
+        user_id=email,
+    )
+    st.session_state.prompt_ids.append(logged_prompt.id)
+    feedback = collector.st_feedback(
+        **feedback_kwargs,
+        key=f"feedback_{int(len(messages)/2)}",
+        prompt_id=logged_prompt.id,
+        user_id=email,
+    )
+    if feedback:
+        trubrics_successful_feedback(feedback)

examples/streamlit/titanic_app.py

@@ -62,7 +62,6 @@ def init_trubrics():
             open_feedback_label="[Optional] Provide additional feedback",
             metadata=metadata,
             align="flex-start",
-            single_submit=False,
         )
         if feedback:
             trubrics_successful_feedback(feedback)

examples/streamlit/trubrics_utils.py

@@ -14,7 +14,6 @@ def trubrics_config(default_component: bool = True):
         type="password",
         value=st.secrets.get("TRUBRICS_PASSWORD", ""),
     )
-    st.write("Don't have an account yet? Create one [here](https://trubrics.streamlit.app/)!")
 
     if default_component:
         return email, password

mkdocs.yml

@@ -1,5 +1,5 @@
 site_name: trubrics-sdk
-site_description: Combine data science knowledge with business user feedback to validate machine learning
+site_description: The first user analytics platform for AI.
 repo_url: https://github.com/trubrics/trubrics-sdk.git
 repo_name: trubrics/trubrics-sdk
 plugins:

requirements-dev.txt

@@ -24,7 +24,7 @@ pytest-cov>=3.0.0
 
 #extras (Note: in setup.cfg)
 streamlit>=1.21.0
-streamlit-feedback>=0.0.9
+streamlit-feedback==0.1.2
 numpy>=1.21.6
 pandas>=1.3.5
 scikit-learn>=1.0.0,<1.1.0

setup.cfg

@@ -30,4 +30,4 @@ profile = black
 
 # integrations
 [options.extras_require]
-streamlit = streamlit>=1.20.0; streamlit-feedback>=0.0.9
+streamlit = streamlit>=1.20.0; streamlit-feedback==0.1.2

tests/test_feedback/test_feedback_streamlit.py

@@ -7,10 +7,7 @@
     "kwargs",
     [
         ({"feedback_type": "random"}),
-        ({"feedback_type": "issue", "user_response": {"test": "test"}}),
         ({"feedback_type": "issue", "open_feedback_label": "test"}),
-        ({"feedback_type": "faces", "user_response": "desc"}),
-        ({"feedback_type": "thumbs", "user_response": "desc"}),
     ],
 )
 def test_st_feedback_raises(kwargs):