diff --git a/README.md b/README.md index 5bff3300a50..f14fe7a1a80 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,12 @@ ## What is AutoGen -AutoGen is a framework that enables the development of LLM applications using multiple agents that can converse with each other to solve tasks. AutoGen agents are customizable, conversable, and seamlessly allow human participation. They can operate in various modes that employ combinations of LLMs, human inputs, and tools. +AutoGen is an open-source programming framework for building AI agents and facilitating cooperation among multiple agents to solve tasks. AutoGen aims to streamline the development and research of agentic AI, much like PyTorch does for Deep Learning. It offers features such as agents capable of interacting with each other, facilitates the use of various large language models (LLMs) and tool use support, autonomous and human-in-the-loop workflows, and multi-agent conversation patterns. + +**Open Source Statement**: The project welcomes contributions from developers and organizations worldwide. Our goal is to foster a collaborative and inclusive community where diverse perspectives and expertise can drive innovation and enhance the project's capabilities. Whether you are an individual contributor or represent an organization, we invite you to join us in shaping the future of this project. Together, we can build something truly remarkable. + +The project is currently maintained by a [dynamic group of volunteers](https://butternut-swordtail-8a5.notion.site/410675be605442d3ada9a42eb4dfef30?v=fa5d0a79fd3d4c0f9c112951b2831cbb&pvs=4) from several different organizations. Contact project administrators Chi Wang and Qingyun Wu via auto-gen@outlook.com if you are interested in becoming a maintainer. + ![AutoGen Overview](https://github.com/microsoft/autogen/blob/main/website/static/img/autogen_agentchat.png) diff --git a/autogen/agentchat/contrib/llamaindex_conversable_agent.py b/autogen/agentchat/contrib/llamaindex_conversable_agent.py index f7a9c3e615d..dbf6f274ae8 100644 --- a/autogen/agentchat/contrib/llamaindex_conversable_agent.py +++ b/autogen/agentchat/contrib/llamaindex_conversable_agent.py @@ -8,15 +8,14 @@ try: from llama_index.core.agent.runner.base import AgentRunner + from llama_index.core.base.llms.types import ChatMessage from llama_index.core.chat_engine.types import AgentChatResponse - from llama_index_client import ChatMessage except ImportError as e: logger.fatal("Failed to import llama-index. Try running 'pip install llama-index'") raise e class LLamaIndexConversableAgent(ConversableAgent): - def __init__( self, name: str, diff --git a/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs b/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs index 173155a96a6..bf05ee97444 100644 --- a/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs +++ b/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs @@ -47,7 +47,7 @@ public async Task GenerateReplyAsync(IEnumerable messages, G return new MessageEnvelope(response, from: this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, + public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { await foreach (var message in _anthropicClient.StreamingChatCompletionsAsync( diff --git a/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs b/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs index f78ccd19dea..af06a054784 100644 --- a/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs +++ b/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs @@ -29,7 +29,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, : response; } - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var messages = context.Messages; @@ -37,7 +37,7 @@ public async IAsyncEnumerable InvokeAsync(MiddlewareContext c await foreach (var reply in agent.GenerateStreamingReplyAsync(chatMessages, context.Options, cancellationToken)) { - if (reply is IStreamingMessage chatMessage) + if (reply is IMessage chatMessage) { var response = ProcessChatCompletionResponse(chatMessage, agent); if (response is not null) @@ -52,7 +52,7 @@ public async IAsyncEnumerable InvokeAsync(MiddlewareContext c } } - private IStreamingMessage? ProcessChatCompletionResponse(IStreamingMessage chatMessage, + private IMessage? ProcessChatCompletionResponse(IMessage chatMessage, IStreamingAgent agent) { if (chatMessage.Content.Content is { Count: 1 } && diff --git a/dotnet/src/AutoGen.Core/Agent/IStreamingAgent.cs b/dotnet/src/AutoGen.Core/Agent/IStreamingAgent.cs index 665f18bac12..6b7794c921a 100644 --- a/dotnet/src/AutoGen.Core/Agent/IStreamingAgent.cs +++ b/dotnet/src/AutoGen.Core/Agent/IStreamingAgent.cs @@ -11,7 +11,7 @@ namespace AutoGen.Core; /// public interface IStreamingAgent : IAgent { - public IAsyncEnumerable GenerateStreamingReplyAsync( + public IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default); diff --git a/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs b/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs index 52967d6ff1c..c7643b1e473 100644 --- a/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs +++ b/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs @@ -47,7 +47,7 @@ public Task GenerateReplyAsync(IEnumerable messages, Generat return _agent.GenerateReplyAsync(messages, options, cancellationToken); } - public IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) { return _agent.GenerateStreamingReplyAsync(messages, options, cancellationToken); } @@ -83,7 +83,7 @@ public Task GenerateReplyAsync(IEnumerable messages, Generat return this.streamingMiddleware.InvokeAsync(context, (IAgent)innerAgent, cancellationToken); } - public IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) { if (streamingMiddleware is null) { diff --git a/dotnet/src/AutoGen.Core/Message/IMessage.cs b/dotnet/src/AutoGen.Core/Message/IMessage.cs index ad215d510e3..9952cbf0679 100644 --- a/dotnet/src/AutoGen.Core/Message/IMessage.cs +++ b/dotnet/src/AutoGen.Core/Message/IMessage.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // IMessage.cs +using System; using System.Collections.Generic; namespace AutoGen.Core; @@ -35,19 +36,21 @@ namespace AutoGen.Core; /// /// /// -public interface IMessage : IStreamingMessage +public interface IMessage { + string? From { get; set; } } -public interface IMessage : IMessage, IStreamingMessage +public interface IMessage : IMessage { + T Content { get; } } /// /// The interface for messages that can get text content. /// This interface will be used by to get the content from the message. /// -public interface ICanGetTextContent : IMessage, IStreamingMessage +public interface ICanGetTextContent : IMessage { public string? GetContent(); } @@ -55,17 +58,18 @@ public interface ICanGetTextContent : IMessage, IStreamingMessage /// /// The interface for messages that can get a list of /// -public interface ICanGetToolCalls : IMessage, IStreamingMessage +public interface ICanGetToolCalls : IMessage { public IEnumerable GetToolCalls(); } - +[Obsolete("Use IMessage instead")] public interface IStreamingMessage { string? From { get; set; } } +[Obsolete("Use IMessage instead")] public interface IStreamingMessage : IStreamingMessage { T Content { get; } diff --git a/dotnet/src/AutoGen.Core/Message/MessageEnvelope.cs b/dotnet/src/AutoGen.Core/Message/MessageEnvelope.cs index f83bea27926..dc9709bbde5 100644 --- a/dotnet/src/AutoGen.Core/Message/MessageEnvelope.cs +++ b/dotnet/src/AutoGen.Core/Message/MessageEnvelope.cs @@ -5,7 +5,7 @@ namespace AutoGen.Core; -public abstract class MessageEnvelope : IMessage, IStreamingMessage +public abstract class MessageEnvelope : IMessage { public MessageEnvelope(string? from = null, IDictionary? metadata = null) { @@ -23,7 +23,7 @@ public static MessageEnvelope Create(TContent content, strin public IDictionary Metadata { get; set; } } -public class MessageEnvelope : MessageEnvelope, IMessage, IStreamingMessage +public class MessageEnvelope : MessageEnvelope, IMessage { public MessageEnvelope(T content, string? from = null, IDictionary? metadata = null) : base(from, metadata) diff --git a/dotnet/src/AutoGen.Core/Message/TextMessage.cs b/dotnet/src/AutoGen.Core/Message/TextMessage.cs index addd8728a92..9419c2b3ba8 100644 --- a/dotnet/src/AutoGen.Core/Message/TextMessage.cs +++ b/dotnet/src/AutoGen.Core/Message/TextMessage.cs @@ -3,7 +3,7 @@ namespace AutoGen.Core; -public class TextMessage : IMessage, IStreamingMessage, ICanGetTextContent +public class TextMessage : IMessage, ICanGetTextContent { public TextMessage(Role role, string content, string? from = null) { @@ -51,7 +51,7 @@ public override string ToString() } } -public class TextMessageUpdate : IStreamingMessage, ICanGetTextContent +public class TextMessageUpdate : IMessage, ICanGetTextContent { public TextMessageUpdate(Role role, string? content, string? from = null) { diff --git a/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs b/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs index d0f89e1ecdd..8660b323044 100644 --- a/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs +++ b/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs @@ -109,7 +109,7 @@ public IEnumerable GetToolCalls() } } -public class ToolCallMessageUpdate : IStreamingMessage +public class ToolCallMessageUpdate : IMessage { public ToolCallMessageUpdate(string functionName, string functionArgumentUpdate, string? from = null) { diff --git a/dotnet/src/AutoGen.Core/Middleware/FunctionCallMiddleware.cs b/dotnet/src/AutoGen.Core/Middleware/FunctionCallMiddleware.cs index d0788077b59..7d30f6d0928 100644 --- a/dotnet/src/AutoGen.Core/Middleware/FunctionCallMiddleware.cs +++ b/dotnet/src/AutoGen.Core/Middleware/FunctionCallMiddleware.cs @@ -70,7 +70,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, return reply; } - public async IAsyncEnumerable InvokeAsync( + public async IAsyncEnumerable InvokeAsync( MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -86,16 +86,16 @@ public async IAsyncEnumerable InvokeAsync( var combinedFunctions = this.functions?.Concat(options.Functions ?? []) ?? options.Functions; options.Functions = combinedFunctions?.ToArray(); - IStreamingMessage? initMessage = default; + IMessage? mergedFunctionCallMessage = default; await foreach (var message in agent.GenerateStreamingReplyAsync(context.Messages, options, cancellationToken)) { if (message is ToolCallMessageUpdate toolCallMessageUpdate && this.functionMap != null) { - if (initMessage is null) + if (mergedFunctionCallMessage is null) { - initMessage = new ToolCallMessage(toolCallMessageUpdate); + mergedFunctionCallMessage = new ToolCallMessage(toolCallMessageUpdate); } - else if (initMessage is ToolCallMessage toolCall) + else if (mergedFunctionCallMessage is ToolCallMessage toolCall) { toolCall.Update(toolCallMessageUpdate); } @@ -104,13 +104,17 @@ public async IAsyncEnumerable InvokeAsync( throw new InvalidOperationException("The first message is ToolCallMessage, but the update message is not ToolCallMessageUpdate"); } } + else if (message is ToolCallMessage toolCallMessage1) + { + mergedFunctionCallMessage = toolCallMessage1; + } else { yield return message; } } - if (initMessage is ToolCallMessage toolCallMsg) + if (mergedFunctionCallMessage is ToolCallMessage toolCallMsg) { yield return await this.InvokeToolCallMessagesAfterInvokingAgentAsync(toolCallMsg, agent); } diff --git a/dotnet/src/AutoGen.Core/Middleware/IStreamingMiddleware.cs b/dotnet/src/AutoGen.Core/Middleware/IStreamingMiddleware.cs index bc7aec57f52..d550bdb519c 100644 --- a/dotnet/src/AutoGen.Core/Middleware/IStreamingMiddleware.cs +++ b/dotnet/src/AutoGen.Core/Middleware/IStreamingMiddleware.cs @@ -14,7 +14,7 @@ public interface IStreamingMiddleware : IMiddleware /// /// The streaming version of . /// - public IAsyncEnumerable InvokeAsync( + public IAsyncEnumerable InvokeAsync( MiddlewareContext context, IStreamingAgent agent, CancellationToken cancellationToken = default); diff --git a/dotnet/src/AutoGen.Core/Middleware/PrintMessageMiddleware.cs b/dotnet/src/AutoGen.Core/Middleware/PrintMessageMiddleware.cs index 099f78e5f17..a4e84de85a4 100644 --- a/dotnet/src/AutoGen.Core/Middleware/PrintMessageMiddleware.cs +++ b/dotnet/src/AutoGen.Core/Middleware/PrintMessageMiddleware.cs @@ -48,7 +48,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, } } - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { IMessage? recentUpdate = null; await foreach (var message in agent.GenerateStreamingReplyAsync(context.Messages, context.Options, cancellationToken)) diff --git a/dotnet/src/AutoGen.Gemini/GeminiChatAgent.cs b/dotnet/src/AutoGen.Gemini/GeminiChatAgent.cs index b081faae832..e759ba26d1e 100644 --- a/dotnet/src/AutoGen.Gemini/GeminiChatAgent.cs +++ b/dotnet/src/AutoGen.Gemini/GeminiChatAgent.cs @@ -143,7 +143,7 @@ public async Task GenerateReplyAsync(IEnumerable messages, G return MessageEnvelope.Create(response, this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var request = BuildChatRequest(messages, options); var response = this.client.GenerateContentStreamAsync(request); diff --git a/dotnet/src/AutoGen.Gemini/Middleware/GeminiMessageConnector.cs b/dotnet/src/AutoGen.Gemini/Middleware/GeminiMessageConnector.cs index cb18ba084d7..422fb4cd345 100644 --- a/dotnet/src/AutoGen.Gemini/Middleware/GeminiMessageConnector.cs +++ b/dotnet/src/AutoGen.Gemini/Middleware/GeminiMessageConnector.cs @@ -39,7 +39,7 @@ public GeminiMessageConnector(bool strictMode = false) public string Name => nameof(GeminiMessageConnector); - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var messages = ProcessMessage(context.Messages, agent); diff --git a/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs b/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs index cc2c7414550..ac144854fac 100644 --- a/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs +++ b/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs @@ -78,7 +78,7 @@ public async Task GenerateReplyAsync( return new MessageEnvelope(response, from: this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync( + public async IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/dotnet/src/AutoGen.Mistral/Middleware/MistralChatMessageConnector.cs b/dotnet/src/AutoGen.Mistral/Middleware/MistralChatMessageConnector.cs index 95592e97fcc..78de12a5c01 100644 --- a/dotnet/src/AutoGen.Mistral/Middleware/MistralChatMessageConnector.cs +++ b/dotnet/src/AutoGen.Mistral/Middleware/MistralChatMessageConnector.cs @@ -15,14 +15,14 @@ public class MistralChatMessageConnector : IStreamingMiddleware, IMiddleware { public string? Name => nameof(MistralChatMessageConnector); - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var messages = context.Messages; var chatMessages = ProcessMessage(messages, agent); var chunks = new List(); await foreach (var reply in agent.GenerateStreamingReplyAsync(chatMessages, context.Options, cancellationToken)) { - if (reply is IStreamingMessage chatMessage) + if (reply is IMessage chatMessage) { chunks.Add(chatMessage.Content); var response = ProcessChatCompletionResponse(chatMessage, agent); @@ -167,7 +167,7 @@ private IMessage PostProcessMessage(ChatCompletionResponse response, IAgent from } } - private IStreamingMessage? ProcessChatCompletionResponse(IStreamingMessage message, IAgent agent) + private IMessage? ProcessChatCompletionResponse(IMessage message, IAgent agent) { var response = message.Content; if (response.VarObject != "chat.completion.chunk") diff --git a/dotnet/src/AutoGen.Ollama/Agent/OllamaAgent.cs b/dotnet/src/AutoGen.Ollama/Agent/OllamaAgent.cs index 9ef68388d60..87b176d8bcc 100644 --- a/dotnet/src/AutoGen.Ollama/Agent/OllamaAgent.cs +++ b/dotnet/src/AutoGen.Ollama/Agent/OllamaAgent.cs @@ -53,7 +53,7 @@ public async Task GenerateReplyAsync( } } - public async IAsyncEnumerable GenerateStreamingReplyAsync( + public async IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/dotnet/src/AutoGen.Ollama/Middlewares/OllamaMessageConnector.cs b/dotnet/src/AutoGen.Ollama/Middlewares/OllamaMessageConnector.cs index a21ec3a1c99..3919b238d65 100644 --- a/dotnet/src/AutoGen.Ollama/Middlewares/OllamaMessageConnector.cs +++ b/dotnet/src/AutoGen.Ollama/Middlewares/OllamaMessageConnector.cs @@ -30,14 +30,14 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, }; } - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var messages = ProcessMessage(context.Messages, agent); var chunks = new List(); await foreach (var update in agent.GenerateStreamingReplyAsync(messages, context.Options, cancellationToken)) { - if (update is IStreamingMessage chatResponseUpdate) + if (update is IMessage chatResponseUpdate) { var response = chatResponseUpdate.Content switch { diff --git a/dotnet/src/AutoGen.OpenAI/Agent/GPTAgent.cs b/dotnet/src/AutoGen.OpenAI/Agent/GPTAgent.cs index cdc6cc464d1..5de481245b7 100644 --- a/dotnet/src/AutoGen.OpenAI/Agent/GPTAgent.cs +++ b/dotnet/src/AutoGen.OpenAI/Agent/GPTAgent.cs @@ -104,7 +104,7 @@ public async Task GenerateReplyAsync( return await _innerAgent.GenerateReplyAsync(messages, options, cancellationToken); } - public IAsyncEnumerable GenerateStreamingReplyAsync( + public IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) diff --git a/dotnet/src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs b/dotnet/src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs index 37a4882f69e..b192cde1024 100644 --- a/dotnet/src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs +++ b/dotnet/src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs @@ -87,7 +87,7 @@ public async Task GenerateReplyAsync( return new MessageEnvelope(reply, from: this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync( + public async IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs b/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs index c1dc2caa99f..e1dd0757fcf 100644 --- a/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs +++ b/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs @@ -47,7 +47,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, return PostProcessMessage(reply); } - public async IAsyncEnumerable InvokeAsync( + public async IAsyncEnumerable InvokeAsync( MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -57,7 +57,7 @@ public async IAsyncEnumerable InvokeAsync( string? currentToolName = null; await foreach (var reply in streamingReply) { - if (reply is IStreamingMessage update) + if (reply is IMessage update) { if (update.Content.FunctionName is string functionName) { @@ -98,7 +98,7 @@ public IMessage PostProcessMessage(IMessage message) }; } - public IStreamingMessage? PostProcessStreamingMessage(IStreamingMessage update, string? currentToolName) + public IMessage? PostProcessStreamingMessage(IMessage update, string? currentToolName) { if (update.Content.ContentUpdate is string contentUpdate) { diff --git a/dotnet/src/AutoGen.SemanticKernel/Middleware/SemanticKernelChatMessageContentConnector.cs b/dotnet/src/AutoGen.SemanticKernel/Middleware/SemanticKernelChatMessageContentConnector.cs index 6ce242eb1ab..a055c0afcb6 100644 --- a/dotnet/src/AutoGen.SemanticKernel/Middleware/SemanticKernelChatMessageContentConnector.cs +++ b/dotnet/src/AutoGen.SemanticKernel/Middleware/SemanticKernelChatMessageContentConnector.cs @@ -47,7 +47,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, return PostProcessMessage(reply); } - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var chatMessageContents = ProcessMessage(context.Messages, agent) .Select(m => new MessageEnvelope(m)); @@ -67,11 +67,11 @@ private IMessage PostProcessMessage(IMessage input) }; } - private IStreamingMessage PostProcessStreamingMessage(IStreamingMessage input) + private IMessage PostProcessStreamingMessage(IMessage input) { return input switch { - IStreamingMessage streamingMessage => PostProcessMessage(streamingMessage), + IMessage streamingMessage => PostProcessMessage(streamingMessage), IMessage msg => PostProcessMessage(msg), _ => input, }; @@ -98,7 +98,7 @@ private IMessage PostProcessMessage(IMessage messageEnvelope } } - private IStreamingMessage PostProcessMessage(IStreamingMessage streamingMessage) + private IMessage PostProcessMessage(IMessage streamingMessage) { var chatMessageContent = streamingMessage.Content; if (chatMessageContent.ChoiceIndex > 0) diff --git a/dotnet/src/AutoGen.SemanticKernel/SemanticKernelAgent.cs b/dotnet/src/AutoGen.SemanticKernel/SemanticKernelAgent.cs index 21f652f56c4..d12c54c1b3b 100644 --- a/dotnet/src/AutoGen.SemanticKernel/SemanticKernelAgent.cs +++ b/dotnet/src/AutoGen.SemanticKernel/SemanticKernelAgent.cs @@ -65,7 +65,7 @@ public async Task GenerateReplyAsync(IEnumerable messages, G return new MessageEnvelope(reply.First(), from: this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync( + public async IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs b/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs index 872cce5e645..6d095845f80 100644 --- a/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs +++ b/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs @@ -86,8 +86,8 @@ public async Task VertexGeminiAgentGenerateStreamingReplyForTextContentAsync() var message = MessageEnvelope.Create(textContent, from: agent.Name); var completion = agent.GenerateStreamingReplyAsync([message]); - var chunks = new List(); - IStreamingMessage finalReply = null!; + var chunks = new List(); + IMessage finalReply = null!; await foreach (var item in completion) { @@ -212,8 +212,8 @@ public async Task VertexGeminiAgentGenerateStreamingReplyWithToolsAsync() var message = MessageEnvelope.Create(textContent, from: agent.Name); - var chunks = new List(); - IStreamingMessage finalReply = null!; + var chunks = new List(); + IMessage finalReply = null!; var completion = agent.GenerateStreamingReplyAsync([message]); diff --git a/dotnet/test/AutoGen.Gemini.Tests/GeminiMessageTests.cs b/dotnet/test/AutoGen.Gemini.Tests/GeminiMessageTests.cs index 7ffb532ea9c..12ba9473403 100644 --- a/dotnet/test/AutoGen.Gemini.Tests/GeminiMessageTests.cs +++ b/dotnet/test/AutoGen.Gemini.Tests/GeminiMessageTests.cs @@ -225,10 +225,10 @@ public async Task ItProcessStreamingTextMessageAsync() }) .Select(m => MessageEnvelope.Create(m)); - IStreamingMessage? finalReply = null; + IMessage? finalReply = null; await foreach (var reply in agent.GenerateStreamingReplyAsync(messageChunks)) { - reply.Should().BeAssignableTo(); + reply.Should().BeAssignableTo(); finalReply = reply; } diff --git a/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs b/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs index 2f06305ed59..8063b707703 100644 --- a/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs +++ b/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs @@ -53,7 +53,7 @@ public async Task ItGenerateContentWithImageAsync() var model = "gemini-1.5-flash-001"; var text = "what's in the image"; - var imagePath = Path.Combine("testData", "images", "image.png"); + var imagePath = Path.Combine("testData", "images", "square.png"); var image = File.ReadAllBytes(imagePath); var request = new GenerateContentRequest { diff --git a/dotnet/test/AutoGen.Ollama.Tests/OllamaAgentTests.cs b/dotnet/test/AutoGen.Ollama.Tests/OllamaAgentTests.cs index c1fb466f0b0..8a416116ea9 100644 --- a/dotnet/test/AutoGen.Ollama.Tests/OllamaAgentTests.cs +++ b/dotnet/test/AutoGen.Ollama.Tests/OllamaAgentTests.cs @@ -65,8 +65,8 @@ public async Task GenerateStreamingReplyAsync_ReturnsValidMessages_WhenCalled() var msg = new Message("user", "hey how are you"); var messages = new IMessage[] { MessageEnvelope.Create(msg, from: modelName) }; - IStreamingMessage? finalReply = default; - await foreach (IStreamingMessage message in ollamaAgent.GenerateStreamingReplyAsync(messages)) + IMessage? finalReply = default; + await foreach (IMessage message in ollamaAgent.GenerateStreamingReplyAsync(messages)) { message.Should().NotBeNull(); message.From.Should().Be(ollamaAgent.Name); @@ -171,8 +171,8 @@ public async Task ItReturnValidStreamingMessageUsingLLavaAsync() var messages = new IMessage[] { MessageEnvelope.Create(imageMessage, from: modelName) }; - IStreamingMessage? finalReply = default; - await foreach (IStreamingMessage message in ollamaAgent.GenerateStreamingReplyAsync(messages)) + IMessage? finalReply = default; + await foreach (IMessage message in ollamaAgent.GenerateStreamingReplyAsync(messages)) { message.Should().NotBeNull(); message.From.Should().Be(ollamaAgent.Name); diff --git a/dotnet/test/AutoGen.Ollama.Tests/OllamaMessageTests.cs b/dotnet/test/AutoGen.Ollama.Tests/OllamaMessageTests.cs index b19291e9767..82cc462061d 100644 --- a/dotnet/test/AutoGen.Ollama.Tests/OllamaMessageTests.cs +++ b/dotnet/test/AutoGen.Ollama.Tests/OllamaMessageTests.cs @@ -57,10 +57,10 @@ public async Task ItProcessStreamingTextMessageAsync() }) .Select(m => MessageEnvelope.Create(m)); - IStreamingMessage? finalReply = null; + IMessage? finalReply = null; await foreach (var reply in agent.GenerateStreamingReplyAsync(messageChunks)) { - reply.Should().BeAssignableTo(); + reply.Should().BeAssignableTo(); finalReply = reply; } diff --git a/dotnet/test/AutoGen.Tests/BasicSampleTest.cs b/dotnet/test/AutoGen.Tests/BasicSampleTest.cs index 8da87d7902b..89925b7d3b3 100644 --- a/dotnet/test/AutoGen.Tests/BasicSampleTest.cs +++ b/dotnet/test/AutoGen.Tests/BasicSampleTest.cs @@ -44,12 +44,6 @@ public async Task MistralClientAgent_TokenCount() await Example14_MistralClientAgent_TokenCount.RunAsync(); } - [ApiKeyFact("OPENAI_API_KEY")] - public async Task DynamicGroupChatGetMLNetPRTestAsync() - { - await Example04_Dynamic_GroupChat_Coding_Task.RunAsync(); - } - [ApiKeyFact("AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_DEPLOY_NAME")] public async Task DynamicGroupChatCalculateFibonacciAsync() { diff --git a/dotnet/test/AutoGen.Tests/EchoAgent.cs b/dotnet/test/AutoGen.Tests/EchoAgent.cs index 9cead5ad251..af5490218e8 100644 --- a/dotnet/test/AutoGen.Tests/EchoAgent.cs +++ b/dotnet/test/AutoGen.Tests/EchoAgent.cs @@ -29,7 +29,7 @@ public Task GenerateReplyAsync( return Task.FromResult(lastMessage); } - public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { foreach (var message in messages) { diff --git a/dotnet/test/AutoGen.Tests/SingleAgentTest.cs b/dotnet/test/AutoGen.Tests/SingleAgentTest.cs index 5a3a9734cd1..64bdc062eb7 100644 --- a/dotnet/test/AutoGen.Tests/SingleAgentTest.cs +++ b/dotnet/test/AutoGen.Tests/SingleAgentTest.cs @@ -297,7 +297,7 @@ public async Task EchoFunctionCallExecutionStreamingTestAsync(IStreamingAgent ag }; var replyStream = agent.GenerateStreamingReplyAsync(messages: new[] { helloWorld }, option); var answer = "[ECHO] Hello world"; - IStreamingMessage? finalReply = default; + IMessage? finalReply = default; await foreach (var reply in replyStream) { reply.From.Should().Be(agent.Name); diff --git a/notebook/agentchat_nested_chats_chess_altmodels.ipynb b/notebook/agentchat_nested_chats_chess_altmodels.ipynb new file mode 100644 index 00000000000..69d3edbcfb5 --- /dev/null +++ b/notebook/agentchat_nested_chats_chess_altmodels.ipynb @@ -0,0 +1,584 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Conversational Chess using non-OpenAI clients\n", + "\n", + "This notebook provides tips for using non-OpenAI models when using functions/tools.\n", + "\n", + "The code is based on [this notebook](/docs/notebooks/agentchat_nested_chats_chess),\n", + "which provides a detailed look at nested chats for tool use. Please refer to that\n", + "notebook for more on nested chats as this will be concentrated on tweaks to\n", + "improve performance with non-OpenAI models.\n", + "\n", + "The notebook represents a chess game between two players with a nested chat to\n", + "determine the available moves and select a move to make.\n", + "\n", + "This game contains a couple of functions/tools that the LLMs must use correctly by the\n", + "LLMs:\n", + "- `get_legal_moves` to get a list of current legal moves.\n", + "- `make_move` to make a move.\n", + "\n", + "Two agents will be used to represent the white and black players, each associated with\n", + "a different LLM cloud provider and model:\n", + "- Anthropic's Sonnet 3.5 will be Player_White\n", + "- Mistral's Mixtral 8x7B (using Together.AI) will be Player_Black\n", + "\n", + "As this involves function calling, we use larger, more capable, models from these providers.\n", + "\n", + "The nested chat will be supported be a board proxy agent who is set up to execute\n", + "the tools and manage the game.\n", + "\n", + "Tips to improve performance with these non-OpenAI models will be noted throughout **in bold**." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Installation\n", + "\n", + "First, you need to install the `pyautogen` and `chess` packages to use AutoGen. We'll include Anthropic and Together.AI libraries." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "! pip install -qqq pyautogen[anthropic,together] chess" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting up LLMs\n", + "\n", + "We'll use the Anthropic (`api_type` is `anthropic`) and Together.AI (`api_type` is `together`) client classes, with their respective models, which both support function calling." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import chess\n", + "import chess.svg\n", + "from IPython.display import display\n", + "from typing_extensions import Annotated\n", + "\n", + "from autogen import ConversableAgent, register_function\n", + "\n", + "# Let's set our two player configs, specifying clients and models\n", + "\n", + "# Anthropic's Sonnet for player white\n", + "player_white_config_list = [\n", + " {\n", + " \"api_type\": \"anthropic\",\n", + " \"model\": \"claude-3-5-sonnet-20240620\",\n", + " \"api_key\": os.getenv(\"ANTHROPIC_API_KEY\"),\n", + " \"cache_seed\": None,\n", + " },\n", + "]\n", + "\n", + "# Mistral's Mixtral 8x7B for player black (through Together.AI)\n", + "player_black_config_list = [\n", + " {\n", + " \"api_type\": \"together\",\n", + " \"model\": \"mistralai/Mixtral-8x7B-Instruct-v0.1\",\n", + " \"api_key\": os.environ.get(\"TOGETHER_API_KEY\"),\n", + " \"cache_seed\": None,\n", + " },\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll setup game variables and the two functions for getting the available moves and then making a move." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the board.\n", + "board = chess.Board()\n", + "\n", + "# Keep track of whether a move has been made.\n", + "made_move = False\n", + "\n", + "\n", + "def get_legal_moves() -> Annotated[\n", + " str,\n", + " \"Call this tool to list of all legal chess moves on the board, output is a list in UCI format, e.g. e2e4,e7e5,e7e8q.\",\n", + "]:\n", + " return \"Possible moves are: \" + \",\".join([str(move) for move in board.legal_moves])\n", + "\n", + "\n", + "def make_move(\n", + " move: Annotated[\n", + " str,\n", + " \"Call this tool to make a move after you have the list of legal moves and want to make a move. Takes UCI format, e.g. e2e4 or e7e5 or e7e8q.\",\n", + " ]\n", + ") -> Annotated[str, \"Result of the move.\"]:\n", + " move = chess.Move.from_uci(move)\n", + " board.push_uci(str(move))\n", + " global made_move\n", + " made_move = True\n", + " # Display the board.\n", + " display(\n", + " chess.svg.board(board, arrows=[(move.from_square, move.to_square)], fill={move.from_square: \"gray\"}, size=200)\n", + " )\n", + " # Get the piece name.\n", + " piece = board.piece_at(move.to_square)\n", + " piece_symbol = piece.unicode_symbol()\n", + " piece_name = (\n", + " chess.piece_name(piece.piece_type).capitalize()\n", + " if piece_symbol.isupper()\n", + " else chess.piece_name(piece.piece_type)\n", + " )\n", + " return f\"Moved {piece_name} ({piece_symbol}) from {chess.SQUARE_NAMES[move.from_square]} to {chess.SQUARE_NAMES[move.to_square]}.\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Creating agents\n", + "\n", + "Our main player agents are created next, with a few tweaks to help our models play:\n", + "\n", + "- Explicitly **telling agents their names** (as the name field isn't sent to the LLM).\n", + "- Providing simple instructions on the **order of functions** (not all models will need it).\n", + "- Asking the LLM to **include their name in the response** so the message content will include their names, helping the LLM understand who has made which moves.\n", + "- Ensure **no spaces are in the agent names** so that their name is distinguishable in the conversation.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "player_white = ConversableAgent(\n", + " name=\"Player_White\",\n", + " system_message=\"You are a chess player and you play as white, your name is 'Player_White'. \"\n", + " \"First call the function get_legal_moves() to get list of legal moves. \"\n", + " \"Then call the function make_move(move) to make a move. \"\n", + " \"Then tell Player_Black you have made your move and it is their turn. \"\n", + " \"Make sure you tell Player_Black you are Player_White.\",\n", + " llm_config={\"config_list\": player_white_config_list, \"cache_seed\": None},\n", + ")\n", + "\n", + "player_black = ConversableAgent(\n", + " name=\"Player_Black\",\n", + " system_message=\"You are a chess player and you play as black, your name is 'Player_Black'. \"\n", + " \"First call the function get_legal_moves() to get list of legal moves. \"\n", + " \"Then call the function make_move(move) to make a move. \"\n", + " \"Then tell Player_White you have made your move and it is their turn. \"\n", + " \"Make sure you tell Player_White you are Player_Black.\",\n", + " llm_config={\"config_list\": player_black_config_list, \"cache_seed\": None},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we create a proxy agent that will be used to move the pieces on the board." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Check if the player has made a move, and reset the flag if move is made.\n", + "def check_made_move(msg):\n", + " global made_move\n", + " if made_move:\n", + " made_move = False\n", + " return True\n", + " else:\n", + " return False\n", + "\n", + "\n", + "board_proxy = ConversableAgent(\n", + " name=\"Board_Proxy\",\n", + " llm_config=False,\n", + " # The board proxy will only terminate the conversation if the player has made a move.\n", + " is_termination_msg=check_made_move,\n", + " # The auto reply message is set to keep the player agent retrying until a move is made.\n", + " default_auto_reply=\"Please make a move.\",\n", + " human_input_mode=\"NEVER\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our functions are then assigned to the agents so they can be passed to the LLM to choose from.\n", + "\n", + "We have tweaked the descriptions to provide **more guidance on when** to use it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "register_function(\n", + " make_move,\n", + " caller=player_white,\n", + " executor=board_proxy,\n", + " name=\"make_move\",\n", + " description=\"Call this tool to make a move after you have the list of legal moves.\",\n", + ")\n", + "\n", + "register_function(\n", + " get_legal_moves,\n", + " caller=player_white,\n", + " executor=board_proxy,\n", + " name=\"get_legal_moves\",\n", + " description=\"Call this to get a legal moves before making a move.\",\n", + ")\n", + "\n", + "register_function(\n", + " make_move,\n", + " caller=player_black,\n", + " executor=board_proxy,\n", + " name=\"make_move\",\n", + " description=\"Call this tool to make a move after you have the list of legal moves.\",\n", + ")\n", + "\n", + "register_function(\n", + " get_legal_moves,\n", + " caller=player_black,\n", + " executor=board_proxy,\n", + " name=\"get_legal_moves\",\n", + " description=\"Call this to get a legal moves before making a move.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Almost there, we now create nested chats between players and the board proxy agent to work out the available moves and make the move." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "player_white.register_nested_chats(\n", + " trigger=player_black,\n", + " chat_queue=[\n", + " {\n", + " # The initial message is the one received by the player agent from\n", + " # the other player agent.\n", + " \"sender\": board_proxy,\n", + " \"recipient\": player_white,\n", + " # The final message is sent to the player agent.\n", + " \"summary_method\": \"last_msg\",\n", + " }\n", + " ],\n", + ")\n", + "\n", + "player_black.register_nested_chats(\n", + " trigger=player_white,\n", + " chat_queue=[\n", + " {\n", + " # The initial message is the one received by the player agent from\n", + " # the other player agent.\n", + " \"sender\": board_proxy,\n", + " \"recipient\": player_black,\n", + " # The final message is sent to the player agent.\n", + " \"summary_method\": \"last_msg\",\n", + " }\n", + " ],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Playing the game\n", + "\n", + "Now the game can begin!" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mPlayer_Black\u001b[0m (to Player_White):\n", + "\n", + "Let's play chess! Your move.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[34m\n", + "********************************************************************************\u001b[0m\n", + "\u001b[34mStarting a new chat....\u001b[0m\n", + "\u001b[34m\n", + "********************************************************************************\u001b[0m\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "Let's play chess! Your move.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_White\u001b[0m (to Board_Proxy):\n", + "\n", + "Certainly! I'd be happy to play chess with you. As White, I'll make the first move. Let me start by checking the legal moves available to me.\n", + "\u001b[32m***** Suggested tool call (toolu_015sLMucefMVqS5ZNyWVGjgu): get_legal_moves *****\u001b[0m\n", + "Arguments: \n", + "{}\n", + "\u001b[32m*********************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION get_legal_moves...\u001b[0m\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "\u001b[32m***** Response from calling tool (toolu_015sLMucefMVqS5ZNyWVGjgu) *****\u001b[0m\n", + "Possible moves are: g1h3,g1f3,b1c3,b1a3,h2h3,g2g3,f2f3,e2e3,d2d3,c2c3,b2b3,a2a3,h2h4,g2g4,f2f4,e2e4,d2d4,c2c4,b2b4,a2a4\n", + "\u001b[32m***********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_White\u001b[0m (to Board_Proxy):\n", + "\n", + "Thank you for initiating a game of chess! As Player_White, I'll make the first move. After analyzing the legal moves, I've decided to make a classic opening move.\n", + "\u001b[32m***** Suggested tool call (toolu_01VjmBhHcGw5RTRKYC4Y5MeV): make_move *****\u001b[0m\n", + "Arguments: \n", + "{\"move\": \"e2e4\"}\n", + "\u001b[32m***************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION make_move...\u001b[0m\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "
r n b q k b n r\n",
+       "p p p p p p p p\n",
+       ". . . . . . . .\n",
+       ". . . . . . . .\n",
+       ". . . . P . . .\n",
+       ". . . . . . . .\n",
+       "P P P P . P P P\n",
+       "R N B Q K B N R
" + ], + "text/plain": [ + "'
r n b q k b n r\\np p p p p p p p\\n. . . . . . . .\\n. . . . . . . .\\n. . . . P . . .\\n. . . . . . . .\\nP P P P . P P P\\nR N B Q K B N R
'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "\u001b[32m***** Response from calling tool (toolu_01VjmBhHcGw5RTRKYC4Y5MeV) *****\u001b[0m\n", + "Moved pawn (♙) from e2 to e4.\n", + "\u001b[32m***********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_White\u001b[0m (to Board_Proxy):\n", + "\n", + "Hello, Player_Black! I'm Player_White, and I've just made my move. I've chosen to play the classic opening move e2e4, moving my king's pawn forward two squares. This opens up lines for both my queen and king's bishop, and stakes a claim to the center of the board. It's now your turn to make a move. Good luck!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mPlayer_White\u001b[0m (to Player_Black):\n", + "\n", + "Hello, Player_Black! I'm Player_White, and I've just made my move. I've chosen to play the classic opening move e2e4, moving my king's pawn forward two squares. This opens up lines for both my queen and king's bishop, and stakes a claim to the center of the board. It's now your turn to make a move. Good luck!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[34m\n", + "********************************************************************************\u001b[0m\n", + "\u001b[34mStarting a new chat....\u001b[0m\n", + "\u001b[34m\n", + "********************************************************************************\u001b[0m\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "Hello, Player_Black! I'm Player_White, and I've just made my move. I've chosen to play the classic opening move e2e4, moving my king's pawn forward two squares. This opens up lines for both my queen and king's bishop, and stakes a claim to the center of the board. It's now your turn to make a move. Good luck!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_Black\u001b[0m (to Board_Proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_z6jagiqn59m784w1n0zhmiop): get_legal_moves *****\u001b[0m\n", + "Arguments: \n", + "{}\n", + "\u001b[32m********************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION get_legal_moves...\u001b[0m\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_z6jagiqn59m784w1n0zhmiop) *****\u001b[0m\n", + "Possible moves are: g8h6,g8f6,b8c6,b8a6,h7h6,g7g6,f7f6,e7e6,d7d6,c7c6,b7b6,a7a6,h7h5,g7g5,f7f5,e7e5,d7d5,c7c5,b7b5,a7a5\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_Black\u001b[0m (to Board_Proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_59t20pl0ab68z4xx2workgbc): make_move *****\u001b[0m\n", + "Arguments: \n", + "{\"move\":\"g8h6\"}\n", + "\u001b[32m**************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION make_move...\u001b[0m\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "
r n b q k b . r\n",
+       "p p p p p p p p\n",
+       ". . . . . . . n\n",
+       ". . . . . . . .\n",
+       ". . . . P . . .\n",
+       ". . . . . . . .\n",
+       "P P P P . P P P\n",
+       "R N B Q K B N R
" + ], + "text/plain": [ + "'
r n b q k b . r\\np p p p p p p p\\n. . . . . . . n\\n. . . . . . . .\\n. . . . P . . .\\n. . . . . . . .\\nP P P P . P P P\\nR N B Q K B N R
'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_59t20pl0ab68z4xx2workgbc) *****\u001b[0m\n", + "Moved knight (♞) from g8 to h6.\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_Black\u001b[0m (to Board_Proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_jwv1d86srs1fnvu33cky9tgv): make_move *****\u001b[0m\n", + "Arguments: \n", + "{\"move\":\"g8h6\"}\n", + "\u001b[32m**************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mPlayer_Black\u001b[0m (to Player_White):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n" + ] + } + ], + "source": [ + "# Clear the board.\n", + "board = chess.Board()\n", + "\n", + "chat_result = player_black.initiate_chat(\n", + " player_white,\n", + " message=\"Let's play chess! Your move.\",\n", + " max_turns=10,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "At this stage, it's hard to tell who's going to win, but they're playing well and using the functions correctly." + ] + } + ], + "metadata": { + "front_matter": { + "description": "LLM-backed agents playing chess with each other using nested chats.", + "tags": [ + "nested chat", + "tool use", + "orchestration" + ] + }, + "kernelspec": { + "display_name": "autogen", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/setup.py b/setup.py index 9a67c70f49d..cfaad885968 100644 --- a/setup.py +++ b/setup.py @@ -72,10 +72,7 @@ "mathchat": ["sympy", "pydantic==1.10.9", "wolframalpha"], "retrievechat": retrieve_chat, "retrievechat-pgvector": retrieve_chat_pgvector, - "retrievechat-qdrant": [ - *retrieve_chat, - "qdrant_client[fastembed]<1.9.2", - ], + "retrievechat-qdrant": [*retrieve_chat, "qdrant_client", "fastembed>=0.3.1"], "autobuild": ["chromadb", "sentence-transformers", "huggingface-hub", "pysqlite3"], "teachable": ["chromadb"], "lmm": ["replicate", "pillow"], diff --git a/website/blog/2024-06-24-AltModels-Classes/img/agentstogether.jpeg b/website/blog/2024-06-24-AltModels-Classes/img/agentstogether.jpeg new file mode 100644 index 00000000000..fd859fc6207 --- /dev/null +++ b/website/blog/2024-06-24-AltModels-Classes/img/agentstogether.jpeg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:964628601b60ddbab8940fea45014dcd841b89783bb0b7e9ac9d3690f1c41798 +size 659594 diff --git a/website/blog/2024-06-24-AltModels-Classes/index.mdx b/website/blog/2024-06-24-AltModels-Classes/index.mdx new file mode 100644 index 00000000000..9c94094e7e4 --- /dev/null +++ b/website/blog/2024-06-24-AltModels-Classes/index.mdx @@ -0,0 +1,393 @@ +--- +title: Enhanced Support for Non-OpenAI Models +authors: + - marklysze + - Hk669 +tags: [mistral ai,anthropic,together.ai,gemini] +--- + +![agents](img/agentstogether.jpeg) + +## TL;DR + +- **AutoGen has expanded integrations with a variety of cloud-based model providers beyond OpenAI.** +- **Leverage models and platforms from Gemini, Anthropic, Mistral AI, Together.AI, and Groq for your AutoGen agents.** +- **Utilise models specifically for chat, language, image, and coding.** +- **LLM provider diversification can provide cost and resilience benefits.** + +In addition to the recently released AutoGen [Google Gemini](https://ai.google.dev/) client, new client classes for [Mistral AI](https://mistral.ai/), [Anthropic](https://www.anthropic.com/), [Together.AI](https://www.together.ai/), and [Groq](https://groq.com/) enable you to utilize over 75 different large language models in your AutoGen agent workflow. + +These new client classes tailor AutoGen's underlying messages to each provider's unique requirements and remove that complexity from the developer, who can then focus on building their AutoGen workflow. + +Using them is as simple as installing the client-specific library and updating your LLM config with the relevant `api_type` and `model`. We'll demonstrate how to use them below. + +The community is continuing to enhance and build new client classes as cloud-based inference providers arrive. So, watch this space, and feel free to [discuss](https://discord.gg/pAbnFJrkgZ) or [develop](https://github.com/microsoft/autogen/pulls) another one. + +## Benefits of choice + +The need to use only the best models to overcome workflow-breaking LLM inconsistency has diminished considerably over the last 12 months. + +These new classes provide access to the very largest trillion-parameter models from OpenAI, Google, and Anthropic, continuing to provide the most consistent +and competent agent experiences. However, it's worth trying smaller models from the likes of Meta, Mistral AI, Microsoft, Qwen, and many others. Perhaps they +are capable enough for a task, or sub-task, or even better suited (such as a coding model)! + +Using smaller models will have cost benefits, but they also allow you to test models that you could run locally, allowing you to determine if you can remove cloud inference costs +altogether or even run an AutoGen workflow offline. + +On the topic of cost, these client classes also include provider-specific token cost calculations so you can monitor the cost impact of your workflows. With costs per million +tokens as low as 10 cents (and some are even free!), cost savings can be noticeable. + +## Mix and match + +How does Google's Gemini 1.5 Pro model stack up against Anthropic's Opus or Meta's Llama 3? + +Now you have the ability to quickly change your agent configs and find out. If you want to run all three in the one workflow, +AutoGen's ability to associate specific configurations to each agent means you can select the best LLM for each agent. + +## Capabilities + +The common requirements of text generation and function/tool calling are supported by these client classes. + +Multi-modal support, such as for image/audio/video, is an area of active development. The [Google Gemini](https://microsoft.github.io/autogen/docs/topics/non-openai-models/cloud-gemini) client class can be +used to create a multimodal agent. + +## Tips + +Here are some tips when working with these client classes: + +- **Most to least capable** - start with larger models and get your workflow working, then iteratively try smaller models. +- **Right model** - choose one that's suited to your task, whether it's coding, function calling, knowledge, or creative writing. +- **Agent names** - these cloud providers do not use the `name` field on a message, so be sure to use your agent's name in their `system_message` and `description` fields, as well as instructing the LLM to 'act as' them. This is particularly important for "auto" speaker selection in group chats as we need to guide the LLM to choose the next agent based on a name, so tweak `select_speaker_message_template`, `select_speaker_prompt_template`, and `select_speaker_auto_multiple_template` with more guidance. +- **Context length** - as your conversation gets longer, models need to support larger context lengths, be mindful of what the model supports and consider using [Transform Messages](https://microsoft.github.io/autogen/docs/topics/handling_long_contexts/intro_to_transform_messages) to manage context size. +- **Provider parameters** - providers have parameters you can set such as temperature, maximum tokens, top-k, top-p, and safety. See each client class in AutoGen's [API Reference](https://microsoft.github.io/autogen/docs/reference/oai/gemini) or [documentation](https://microsoft.github.io/autogen/docs/topics/non-openai-models/cloud-gemini) for details. +- **Prompts** - prompt engineering is critical in guiding smaller LLMs to do what you need. [ConversableAgent](https://microsoft.github.io/autogen/docs/reference/agentchat/conversable_agent), [GroupChat](https://microsoft.github.io/autogen/docs/reference/agentchat/groupchat), [UserProxyAgent](https://microsoft.github.io/autogen/docs/reference/agentchat/user_proxy_agent), and [AssistantAgent](https://microsoft.github.io/autogen/docs/reference/agentchat/assistant_agent) all have customizable prompt attributes that you can tailor. Here are some prompting tips from [Anthropic](https://docs.anthropic.com/en/docs/build-with-claude/prompt-engineering/overview)([+Library](https://docs.anthropic.com/en/prompt-library/library)), [Mistral AI](https://docs.mistral.ai/guides/prompting_capabilities/), [Together.AI](https://docs.together.ai/docs/examples), and [Meta](https://llama.meta.com/docs/how-to-guides/prompting/). +- **Help!** - reach out on the AutoGen [Discord](https://discord.gg/pAbnFJrkgZ) or [log an issue](https://github.com/microsoft/autogen/issues) if you need help with or can help improve these client classes. + +Now it's time to try them out. + +## Quickstart + +### Installation + +Install the appropriate client based on the model you wish to use. + +```sh +pip install pyautogen["mistral"] # for Mistral AI client +pip install pyautogen["anthropic"] # for Anthropic client +pip install pyautogen["together"] # for Together.AI client +pip install pyautogen["groq"] # for Groq client +``` + +### Configuration Setup + +Add your model configurations to the `OAI_CONFIG_LIST`. Ensure you specify the `api_type` to initialize the respective client (Anthropic, Mistral, or Together). + +```yaml +[ + { + "model": "your anthropic model name", + "api_key": "your Anthropic api_key", + "api_type": "anthropic" + }, + { + "model": "your mistral model name", + "api_key": "your Mistral AI api_key", + "api_type": "mistral" + }, + { + "model": "your together.ai model name", + "api_key": "your Together.AI api_key", + "api_type": "together" + }, + { + "model": "your groq model name", + "api_key": "your Groq api_key", + "api_type": "groq" + } +] +``` + +### Usage + +The `[config_list_from_json](https://microsoft.github.io/autogen/docs/reference/oai/openai_utils/#config_list_from_json)` function loads a list of configurations from an environment variable or a json file. + +```py +import autogen +from autogen import AssistantAgent, UserProxyAgent + +config_list = autogen.config_list_from_json( + "OAI_CONFIG_LIST" +) +``` + +### Construct Agents + +Construct a simple conversation between a User proxy and an Assistant agent + +```py +user_proxy = UserProxyAgent( + name="User_proxy", + code_execution_config={ + "last_n_messages": 2, + "work_dir": "groupchat", + "use_docker": False, # Please set use_docker = True if docker is available to run the generated code. Using docker is safer than running the generated code directly. + }, + human_input_mode="ALWAYS", + is_termination_msg=lambda msg: not msg["content"] +) + +assistant = AssistantAgent( + name="assistant", + llm_config = {"config_list": config_list} +) +``` + +### Start chat + +```py + +user_proxy.intiate_chat(assistant, message="Write python code to print Hello World!") + +``` + +**NOTE: To integrate this setup into GroupChat, follow the [tutorial](https://microsoft.github.io/autogen/docs/notebooks/agentchat_groupchat) with the same config as above.** + + +## Function Calls + +Now, let's look at how Anthropic's Sonnet 3.5 is able to suggest multiple function calls in a single response. + +This example is a simple travel agent setup with an agent for function calling and a user proxy agent for executing the functions. + +One thing you'll note here is Anthropic's models are more verbose than OpenAI's and will typically provide chain-of-thought or general verbiage when replying. Therefore we provide more explicit instructions to `functionbot` to not reply with more than necessary. Even so, it can't always help itself! + +Let's start with setting up our configuration and agents. + +```py +import os +import autogen +import json +from typing import Literal +from typing_extensions import Annotated + +# Anthropic configuration, using api_type='anthropic' +anthropic_llm_config = { + "config_list": + [ + { + "api_type": "anthropic", + "model": "claude-3-5-sonnet-20240620", + "api_key": os.getenv("ANTHROPIC_API_KEY"), + "cache_seed": None + } + ] +} + +# Our functionbot, who will be assigned two functions and +# given directions to use them. +functionbot = autogen.AssistantAgent( + name="functionbot", + system_message="For currency exchange tasks, only use " + "the functions you have been provided with. Do not " + "reply with helpful tips. Once you've recommended functions " + "reply with 'TERMINATE'.", + is_termination_msg=lambda x: x.get("content", "") and (x.get("content", "").rstrip().endswith("TERMINATE") or x.get("content", "") == ""), + llm_config=anthropic_llm_config, +) + +# Our user proxy agent, who will be used to manage the customer +# request and conversation with the functionbot, terminating +# when we have the information we need. +user_proxy = autogen.UserProxyAgent( + name="user_proxy", + system_message="You are a travel agent that provides " + "specific information to your customers. Get the " + "information you need and provide a great summary " + "so your customer can have a great trip. If you " + "have the information you need, simply reply with " + "'TERMINATE'.", + is_termination_msg=lambda x: x.get("content", "") and (x.get("content", "").rstrip().endswith("TERMINATE") or x.get("content", "") == ""), + human_input_mode="NEVER", + max_consecutive_auto_reply=10, +) +``` + +We define the two functions. +```py +CurrencySymbol = Literal["USD", "EUR"] + +def exchange_rate(base_currency: CurrencySymbol, quote_currency: CurrencySymbol) -> float: + if base_currency == quote_currency: + return 1.0 + elif base_currency == "USD" and quote_currency == "EUR": + return 1 / 1.1 + elif base_currency == "EUR" and quote_currency == "USD": + return 1.1 + else: + raise ValueError(f"Unknown currencies {base_currency}, {quote_currency}") + +def get_current_weather(location, unit="fahrenheit"): + """Get the weather for some location""" + if "chicago" in location.lower(): + return json.dumps({"location": "Chicago", "temperature": "13", "unit": unit}) + elif "san francisco" in location.lower(): + return json.dumps({"location": "San Francisco", "temperature": "55", "unit": unit}) + elif "new york" in location.lower(): + return json.dumps({"location": "New York", "temperature": "11", "unit": unit}) + else: + return json.dumps({"location": location, "temperature": "unknown"}) +``` + +And then associate them with the `user_proxy` for execution and `functionbot` for the LLM to consider using them. + +```py +@user_proxy.register_for_execution() +@functionbot.register_for_llm(description="Currency exchange calculator.") +def currency_calculator( + base_amount: Annotated[float, "Amount of currency in base_currency"], + base_currency: Annotated[CurrencySymbol, "Base currency"] = "USD", + quote_currency: Annotated[CurrencySymbol, "Quote currency"] = "EUR", +) -> str: + quote_amount = exchange_rate(base_currency, quote_currency) * base_amount + return f"{quote_amount} {quote_currency}" + +@user_proxy.register_for_execution() +@functionbot.register_for_llm(description="Weather forecast for US cities.") +def weather_forecast( + location: Annotated[str, "City name"], +) -> str: + weather_details = get_current_weather(location=location) + weather = json.loads(weather_details) + return f"{weather['location']} will be {weather['temperature']} degrees {weather['unit']}" +``` + +Finally, we start the conversation with a request for help from our customer on their upcoming trip to New York and the Euro they would like exchanged to USD. + +Importantly, we're also using Anthropic's Sonnet to provide a summary through the `summary_method`. Using `summary_prompt`, we guide Sonnet to give us an email output. + +```py +# start the conversation +res = user_proxy.initiate_chat( + functionbot, + message="My customer wants to travel to New York and " + "they need to exchange 830 EUR to USD. Can you please " + "provide them with a summary of the weather and " + "exchanged currently in USD?", + summary_method="reflection_with_llm", + summary_args={ + "summary_prompt": """Summarize the conversation by + providing an email response with the travel information + for the customer addressed as 'Dear Customer'. Do not + provide any additional conversation or apologise, + just provide the relevant information and the email.""" + }, +) +``` + +After the conversation has finished, we'll print out the summary. + +```py +print(f"Here's the LLM summary of the conversation:\n\n{res.summary['content']}") +``` + +Here's the resulting output. + +```text +user_proxy (to functionbot): + +My customer wants to travel to New York and they need to exchange 830 EUR +to USD. Can you please provide them with a summary of the weather and +exchanged currently in USD? + +-------------------------------------------------------------------------------- +functionbot (to user_proxy): + +Certainly! I'd be happy to help your customer with information about the +weather in New York and the currency exchange from EUR to USD. Let's use +the available tools to get this information. + +***** Suggested tool call (toolu_016wBUKVX2TToBaMqmiGvhav): weather_forecast ***** +Arguments: +{"location": "New York"} +********************************************************************************** +***** Suggested tool call (toolu_01Nxjeew2BspfKdZ85on3XkP): currency_calculator ***** +Arguments: +{"base_amount": 830, "base_currency": "EUR", "quote_currency": "USD"} +************************************************************************************* + +-------------------------------------------------------------------------------- + +>>>>>>>> EXECUTING FUNCTION weather_forecast... + +>>>>>>>> EXECUTING FUNCTION currency_calculator... +user_proxy (to functionbot): + +user_proxy (to functionbot): + +***** Response from calling tool (toolu_016wBUKVX2TToBaMqmiGvhav) ***** +New York will be 11 degrees fahrenheit +*********************************************************************** + +-------------------------------------------------------------------------------- +user_proxy (to functionbot): + +***** Response from calling tool (toolu_01Nxjeew2BspfKdZ85on3XkP) ***** +913.0000000000001 USD +*********************************************************************** + +-------------------------------------------------------------------------------- +functionbot (to user_proxy): + +Thank you for providing the information. I can now give your customer a +summary of the weather in New York and the currency exchange from EUR to USD. + +Weather in New York: +The current forecast for New York indicates that it will be 11 degrees +Fahrenheit. This is quite cold, so your customer should be prepared with +warm clothing for their trip. + +Currency Exchange: +For 830 EUR, your customer will receive approximately 913 USD. This is based +on the current exchange rate provided by our currency calculator. + +To summarize: +1. Weather in New York: 11°F (very cold) +2. Currency exchange: 830 EUR = 913 USD + +Your customer should pack warm clothes for the cold weather in New York and +can expect to have about 913 USD for their trip after exchanging 830 EUR. + +TERMINATE + +-------------------------------------------------------------------------------- +Here's the LLM summary of the conversation: + +Certainly. I'll provide an email response to the customer with the travel +information as requested. + +Dear Customer, + +We are pleased to provide you with the following information for your +upcoming trip to New York: + +Weather Forecast: +The current forecast for New York indicates a temperature of 11 degrees +Fahrenheit. Please be prepared for very cold weather and pack appropriate +warm clothing. + +Currency Exchange: +We have calculated the currency exchange for you. Your 830 EUR will be +equivalent to approximately 913 USD at the current exchange rate. + +We hope this information helps you prepare for your trip to New York. Have +a safe and enjoyable journey! + +Best regards, +Travel Assistance Team +``` + +So we can see how Anthropic's Sonnet is able to suggest multiple tools in a single response, with AutoGen executing them both and providing the results back to Sonnet. Sonnet then finishes with a nice email summary that can be the basis for continued real-life conversation with the customer. + +## More tips and tricks + +For an interesting chess game between Anthropic's Sonnet and Mistral's Mixtral, we've put together a sample notebook that highlights some of the tips and tricks for working with non-OpenAI LLMs. [See the notebook here](https://microsoft.github.io/autogen/docs/notebooks/agentchat_nested_chats_chess_altmodels). diff --git a/website/blog/authors.yml b/website/blog/authors.yml index 83ea099ade4..0e023514465 100644 --- a/website/blog/authors.yml +++ b/website/blog/authors.yml @@ -128,3 +128,15 @@ jluey: name: James Woffinden-Luey title: Senior Research Engineer at Microsoft Research url: https://github.com/jluey1 + +Hk669: + name: Hrushikesh Dokala + title: CS Undergraduate Based in India + url: https://github.com/Hk669 + image_url: https://github.com/Hk669.png + +marklysze: + name: Mark Sze + title: AI Freelancer + url: https://github.com/marklysze + image_url: https://github.com/marklysze.png diff --git a/website/docs/Examples.md b/website/docs/Examples.md index 2ec83d1e0f2..3b71dd682e9 100644 --- a/website/docs/Examples.md +++ b/website/docs/Examples.md @@ -80,6 +80,9 @@ Links to notebook examples: - OpenAI Assistant in a Group Chat - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_groupchat.ipynb) - GPTAssistantAgent based Multi-Agent Tool Use - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/gpt_assistant_agent_function_call.ipynb) +### Non-OpenAI Models +- Conversational Chess using non-OpenAI Models - [View Notebook](/docs/notebooks/agentchat_nested_chats_chess_altmodels) + ### Multimodal Agent - Multimodal Agent Chat with DALLE and GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_dalle_and_gpt4v.ipynb) diff --git a/website/docs/Getting-Started.mdx b/website/docs/Getting-Started.mdx index 0f8c7322411..3e162a09832 100644 --- a/website/docs/Getting-Started.mdx +++ b/website/docs/Getting-Started.mdx @@ -3,11 +3,12 @@ import TabItem from "@theme/TabItem"; # Getting Started -AutoGen is a framework that enables development of LLM applications using -multiple agents that can converse with each other to solve tasks. AutoGen agents -are customizable, conversable, and seamlessly allow human participation. They -can operate in various modes that employ combinations of LLMs, human inputs, and -tools. +AutoGen is an open-source programming framework for building AI agents and facilitating +cooperation among multiple agents to solve tasks. AutoGen aims to provide an easy-to-use +and flexible framework for accelerating development and research on agentic AI, +like PyTorch for Deep Learning. It offers features such as agents that can converse +with other agents, LLM and tool use support, autonomous and human-in-the-loop workflows, +and multi-agent conversation patterns. ![AutoGen Overview](/img/autogen_agentchat.png) diff --git a/website/docs/contributor-guide/contributing.md b/website/docs/contributor-guide/contributing.md index b90d81f227c..b1b6b848f66 100644 --- a/website/docs/contributor-guide/contributing.md +++ b/website/docs/contributor-guide/contributing.md @@ -1,6 +1,6 @@ # Contributing to AutoGen -This project welcomes and encourages all forms of contributions, including but not limited to: +The project welcomes contributions from developers and organizations worldwide. Our goal is to foster a collaborative and inclusive community where diverse perspectives and expertise can drive innovation and enhance the project's capabilities. Whether you are an individual contributor or represent an organization, we invite you to join us in shaping the future of this project. Together, we can build something truly remarkable. Possible contributions include but not limited to: - Pushing patches. - Code review of pull requests. @@ -32,3 +32,7 @@ To see what we are working on and what we plan to work on, please check our ## Becoming a Reviewer There is currently no formal reviewer solicitation process. Current reviewers identify reviewers from active contributors. If you are willing to become a reviewer, you are welcome to let us know on discord. + +## Contact Maintainers + +The project is currently maintained by a [dynamic group of volunteers](https://butternut-swordtail-8a5.notion.site/410675be605442d3ada9a42eb4dfef30?v=fa5d0a79fd3d4c0f9c112951b2831cbb&pvs=4) from several different organizations. Contact project administrators Chi Wang and Qingyun Wu via auto-gen@outlook.com if you are interested in becoming a maintainer. diff --git a/website/docs/tutorial/human-in-the-loop.ipynb b/website/docs/tutorial/human-in-the-loop.ipynb index 04fbdd038b5..afcdeeaf42b 100644 --- a/website/docs/tutorial/human-in-the-loop.ipynb +++ b/website/docs/tutorial/human-in-the-loop.ipynb @@ -10,7 +10,7 @@ "\n", "But many applications may require putting humans in-the-loop with agents. For example, to allow human feedback to steer agents in the right direction, specify goals, etc. In this chapter, we will show how AutoGen supports human intervention.\n", "\n", - "In AutoGen's `ConversableAgent`, the human-the-loop component sits in front\n", + "In AutoGen's `ConversableAgent`, the human-in-the-loop component sits in front\n", "of the auto-reply components. It can intercept the incoming messages and\n", "decide whether to pass them to the auto-reply components or to provide\n", "human feedback. The figure below illustrates the design.\n", @@ -285,9 +285,9 @@ "## Human Input Mode = `TERMINATE`\n", "\n", "In this mode, human input is only requested when a termination condition is\n", - "met. **If the human choose to intercept and reply, the counter will be reset**; if \n", - "the human choose to skip, automatic reply mechanism will be used; if the human\n", - "choose to terminate, the conversation will be terminated.\n", + "met. **If the human chooses to intercept and reply, the counter will be reset**; if \n", + "the human chooses to skip, the automatic reply mechanism will be used; if the human\n", + "chooses to terminate, the conversation will be terminated.\n", "\n", "Let us see this mode in action by playing the same game again, but this time\n", "the guessing agent will only have two chances to guess the number, and if it \n", diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index efc13096b0f..f0c0f84a394 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -14,7 +14,7 @@ customPostCssPlugin = () => { module.exports = { title: "AutoGen", - tagline: "Enable Next-Gen Large Language Model Applications", + tagline: "An Open-Source Programming Framework for Agentic AI", url: "https://microsoft.github.io", baseUrl: "/autogen/", onBrokenLinks: "throw", @@ -281,6 +281,10 @@ module.exports = { to: "/docs/notebooks/agentchat_nested_chats_chess", from: ["/docs/notebooks/agentchat_chess"], }, + { + to: "/docs/notebooks/agentchat_nested_chats_chess_altmodels", + from: ["/docs/notebooks/agentchat_chess_altmodels"], + }, { to: "/docs/contributor-guide/contributing", from: ["/docs/Contribute"],