Support OpenAI Assistants API

livekit-agents/livekit/agents/llm/chat_context.py

@@ -41,6 +41,7 @@ class ChatMessage:
     content: str | list[str | ChatImage] | None = None
     tool_calls: list[function_context.FunctionCallInfo] | None = None
     tool_call_id: str | None = None
+    _metadata: dict[str, Any] = field(default_factory=dict, repr=False, init=False)
 
     @staticmethod
     def create_tool_from_called_function(
@@ -104,6 +105,7 @@ def copy(self):
 @dataclass
 class ChatContext:
     messages: list[ChatMessage] = field(default_factory=list)
+    _metadata: dict[str, Any] = field(default_factory=dict, repr=False, init=False)
 
     def append(
         self, *, text: str = "", images: list[ChatImage] = [], role: ChatRole = "system"

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/__init__.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
+from . import beta
 from .embeddings import EmbeddingData, create_embeddings
 from .llm import LLM, LLMStream
 from .models import TTSModels, TTSVoices, WhisperModels
@@ -25,6 +27,7 @@
     "LLM",
     "LLMStream",
     "WhisperModels",
+    "beta",
     "TTSModels",
     "TTSVoices",
     "create_embeddings",

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/beta/README.md

@@ -0,0 +1,70 @@
+# OpenAI Beta Features
+
+## Assistants API
+
+Example usage:
+
+```python
+import asyncio
+
+from dotenv import load_dotenv
+from livekit import rtc
+from livekit.agents import AutoSubscribe, JobContext, WorkerOptions, cli, llm
+from livekit.agents.voice_assistant import VoiceAssistant
+from livekit.plugins import deepgram, openai, silero
+from livekit.plugins.openai.beta import (
+    AssistantCreateOptions,
+    AssistantLLM,
+    AssistantOptions,
+)
+
+load_dotenv()
+
+
+async def entrypoint(ctx: JobContext):
+    initial_ctx = llm.ChatContext()
+
+    await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)
+
+    assistant = VoiceAssistant(
+        vad=silero.VAD.load(),
+        stt=deepgram.STT(),
+        llm=AssistantLLM(
+            assistant_opts=AssistantOptions(
+                create_options=AssistantCreateOptions(
+                    model="gpt-4o",
+                    instructions="You are a voice assistant created by LiveKit. Your interface with users will be voice.",
+                    name="KITT",
+                )
+            )
+        ),
+        tts=openai.TTS(),
+        chat_ctx=initial_ctx,
+    )
+    assistant.start(ctx.room)
+
+    # listen to incoming chat messages, only required if you'd like the agent to
+    # answer incoming messages from Chat
+    chat = rtc.ChatManager(ctx.room)
+
+    async def answer_from_text(txt: str):
+        chat_ctx = assistant.chat_ctx.copy()
+        chat_ctx.append(role="user", text=txt)
+        stream = assistant.llm.chat(chat_ctx=chat_ctx)
+        await assistant.say(stream)
+
+    @chat.on("message_received")
+    def on_chat_received(msg: rtc.ChatMessage):
+        if msg.message:
+            asyncio.create_task(answer_from_text(msg.message))
+
+    await asyncio.sleep(1)
+    await assistant.say("Hey, how can I help you today?", allow_interruptions=True)
+
+
+if __name__ == "__main__":
+    cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint))
+```
+
+## TODO
+- tool calling

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/beta/__init__.py

@@ -0,0 +1,13 @@
+from .assistant_llm import (
+    AssistantCreateOptions,
+    AssistantLLM,
+    AssistantLoadOptions,
+    AssistantOptions,
+)
+
+__all__ = [
+    "AssistantLLM",
+    "AssistantOptions",
+    "AssistantCreateOptions",
+    "AssistantLoadOptions",
+]