Add ImageContent integration test (#5586)

dotnet · Oct 30, 2024 · e18a055 · e18a055
1 parent bf0e0a4
commit e18a055
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 0 deletions.
diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs
@@ -6,6 +6,7 @@
 using System.ComponentModel;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Text;
@@ -132,6 +133,27 @@ public virtual async Task CompleteStreamingAsync_UsageDataAvailable()
         Assert.Equal(usage.Details.InputTokenCount + usage.Details.OutputTokenCount, usage.Details.TotalTokenCount);
     }
 
+    protected virtual string? GetModel_MultiModal_DescribeImage() => null;
+
+    [ConditionalFact]
+    public virtual async Task MultiModal_DescribeImage()
+    {
+        SkipIfNotEnabled();
+
+        var response = await _chatClient.CompleteAsync(
+            [
+                new(ChatRole.User,
+                [
+                    new TextContent("What does this logo say?"),
+                    new ImageContent(GetImageDataUri()),
+                ])
+            ],
+            new() { ModelId = GetModel_MultiModal_DescribeImage() });
+
+        Assert.Single(response.Choices);
+        Assert.True(response.Message.Text?.IndexOf("net", StringComparison.OrdinalIgnoreCase) >= 0, response.Message.Text);
+    }
+
     [ConditionalFact]
     public virtual async Task FunctionInvocation_AutomaticallyInvokeFunction_Parameterless()
     {
@@ -714,6 +736,15 @@ private enum JobType
         Unknown,
     }
 
+    private static Uri GetImageDataUri()
+    {
+        using Stream? s = typeof(ChatClientIntegrationTests).Assembly.GetManifestResourceStream("Microsoft.Extensions.AI.dotnet.png");
+        Assert.NotNull(s);
+        MemoryStream ms = new();
+        s.CopyTo(ms);
+        return new Uri($"data:image/png;base64,{Convert.ToBase64String(ms.ToArray())}");
+    }
+
     [MemberNotNull(nameof(_chatClient))]
     protected void SkipIfNotEnabled()
     {

diff --git a/...icrosoft.Extensions.AI.Integration.Tests/Microsoft.Extensions.AI.Integration.Tests.csproj b/...icrosoft.Extensions.AI.Integration.Tests/Microsoft.Extensions.AI.Integration.Tests.csproj
@@ -15,6 +15,10 @@
     <InjectSharedThrow>true</InjectSharedThrow>
   </PropertyGroup>
 
+  <ItemGroup>
+    <EmbeddedResource Include="dotnet.png" />
+  </ItemGroup>
+
   <ItemGroup>
     <Compile Include="..\Microsoft.Extensions.AI.Abstractions.Tests\CapturingLogger.cs" />
     <Compile Include="..\Microsoft.Extensions.AI.Abstractions.Tests\TestChatClient.cs" />

diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/dotnet.png b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/dotnet.png
diff --git a/test/Libraries/Microsoft.Extensions.AI.Ollama.Tests/OllamaChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Ollama.Tests/OllamaChatClientIntegrationTests.cs
@@ -30,6 +30,8 @@ public override Task FunctionInvocation_RequireAny() =>
     public override Task FunctionInvocation_RequireSpecific() =>
         throw new SkipTestException("Ollama does not currently support requiring function invocation.");
 
+    protected override string? GetModel_MultiModal_DescribeImage() => "llava";
+
     [ConditionalFact]
     public async Task PromptBasedFunctionCalling_NoArgs()
     {