feat/image (#52)

* do not use fetch event source directly

* refactor

* fix type errors

* fix command menu

* initial image support

* render images back to the user

* add more image processing

* update image button style

* update styling

* tauri support for images

src/lib/components/ChatMessageItem.svelte

@@ -9,11 +9,9 @@
   import MarkdownHtmlBlock from "./MarkdownHtmlBlock.svelte";
   import "./markdown.css";
   import { currentlyEditingMessage, inProgressMessageId } from "$lib/stores/stores";
-  import { onMount } from "svelte";
   import ChatMessageControls from "./ChatMessageControls.svelte";
   import { autosize } from "$lib/utils";
-  import { fly, slide } from "svelte/transition";
-  import { toast } from "$lib/toast";
+  import { slide } from "svelte/transition";
   let _class: string = "";
   export { _class as class };
   export let item: ChatMessage;
@@ -101,8 +99,21 @@
         class="w-full bg-transparent outline-none resize-none mb-3"
       />
     {:else if item.role === "user"}
-      <!-- User input is not considered markdown, but whitespace should be respected -->
-      <p class="whitespace-pre-wrap">{item.content}</p>
+      {#if typeof item.content === "string" && item.content.startsWith("[{")}
+        {#each JSON.parse(item.content) as part}
+          {#if part.type === "image_url"}
+            <img
+              src={part.image_url.url}
+              alt="Attached image"
+              class="max-w-sm max-h-[200px] rounded-lg my-2 object-contain"
+            />
+          {:else if part.type === "text"}
+            <p class="whitespace-pre-wrap">{part.text}</p>
+          {/if}
+        {/each}
+      {:else}
+        <p class="whitespace-pre-wrap">{item.content}</p>
+      {/if}
     {:else}
       <SvelteMarkdown
         source={item.content || NBSP}

src/lib/components/ImageAttachment.svelte

@@ -0,0 +1,59 @@
+<script lang="ts">
+  import { XCircle, Image } from "lucide-svelte";
+  import { attachedImage } from "$lib/stores/stores";
+  import { processImageForAI } from "$lib/utils";
+  import { toast } from "$lib/toast";
+  import { getSystem } from "$lib/gui";
+
+  const sys = getSystem();
+
+  async function handleFileSelect(e: MouseEvent) {
+    e.preventDefault();
+    e.stopPropagation();
+    try {
+      const result = await sys.chooseAndOpenImageFile();
+      if (!result) return;
+
+      const file = new File([result.data], result.name, {
+        type: result.name.endsWith(".svg") ? "image/svg+xml" : "image/jpeg", // fallback type
+      });
+
+      const processed = await processImageForAI(file);
+      attachedImage.set(processed);
+    } catch (error) {
+      console.error("Error processing file:", error);
+      toast({
+        type: "error",
+        title: "Error processing image",
+        message:
+          error instanceof Error ? error.message : "Could not process the selected image file",
+      });
+    }
+  }
+</script>
+
+<div class="flex items-end pb-[2px] space-x-2 h-auto pr-[2px] border-r border-zinc-700 relative">
+  <button
+    type="button"
+    class="cursor-pointer p-2 hover:bg-zinc-700 rounded-lg !ml-[2px]"
+    on:click={handleFileSelect}
+  >
+    <Image class="w-5 h-5" />
+  </button>
+
+  {#if $attachedImage}
+    <div class="absolute left-full bottom-[calc(100%+8px)] w-20 h-20 drop-shadow-lg">
+      <img
+        src={$attachedImage.base64}
+        alt="Attached"
+        class="w-full h-full object-cover rounded-lg"
+      />
+      <button
+        class="absolute -top-2 -right-2 bg-zinc-800 rounded-full"
+        on:click={() => attachedImage.set(null)}
+      >
+        <XCircle class="w-4 h-4" />
+      </button>
+    </div>
+  {/if}
+</div>

src/lib/components/ModelPicker.svelte

@@ -4,38 +4,29 @@
   import * as Popover from "$lib/components/ui/popover";
   import { Button } from "$lib/components/ui/button";
   import { cn, groupBy } from "$lib/utils";
-  import type { ComponentType } from "svelte";
+  import type { Component as SvelteComponent } from "svelte";
   import IconBrain from "$lib/components/IconBrain.svelte";
   import { onMount } from "svelte";
   import { llmProviders, chatModels, modelPickerOpen } from "$lib/stores/stores/llmProvider";
   import IconOpenAi from "./IconOpenAI.svelte";
   import { gptProfileStore } from "$lib/stores/stores/llmProfile";
   import { showInitScreen } from "$lib/stores/stores";
-  import { writable } from "svelte/store";
   import { toast } from "$lib/toast";
+  import { commandScore } from "./ui/command/command-score";
   let _class: string = "";
   export { _class as class };
 
-  type IconSource =
-    | { char: string }
-    | { component: ComponentType; class?: string }
-    | { src: string };
+  type IconSource = { component: SvelteComponent; class?: string };
 
-  type Status = {
-    value: string;
-    label: string;
-    icon?: IconSource;
-  };
-
-  $: options = $chatModels.models
-    .map((x) => {
+  $: options = [
+    ...$chatModels.models.map((x) => {
       const provider = llmProviders.byId(x.provider.id);
       let icon: IconSource | undefined = undefined;
 
       if (provider?.id === "prompta") {
         icon = { component: IconBrain, class: "w-5 h-5 text-[#30CEC0] " };
       } else if (provider?.id === "openai") {
-        icon = { component: IconOpenAi };
+        icon = { component: IconOpenAi, class: "" };
       }
 
       return {
@@ -44,8 +35,9 @@
         icon,
         provider,
       };
-    })
-    .concat(llmProviders.getSpecialProviders());
+    }),
+    ...llmProviders.getSpecialProviders(),
+  ];
   $: optionGroups = groupBy(options, (x) => x.provider?.name ?? "Other");
 
   let value = $gptProfileStore.model || "";
@@ -83,6 +75,30 @@
   onMount(() => {
     chatModels.refresh();
   });
+
+  let searchValue = "";
+  let selectedItem = value;
+
+  $: filteredOptions = searchValue
+    ? options.filter((opt) => {
+        const score = commandScore(opt.label.toLowerCase(), searchValue.toLowerCase());
+        return score > 0;
+      })
+    : options;
+
+  $: filteredGroups = groupBy(filteredOptions, (x) => x.provider?.name ?? "Other");
+
+  function handleSearch(event: CustomEvent<string>) {
+    searchValue = event.detail;
+  }
+
+  function handleKeydown(event: CustomEvent<KeyboardEvent>) {
+    const e = event.detail;
+    if (e.key === "ArrowDown" || e.key === "ArrowUp") {
+      e.preventDefault();
+      // The CommandPrimitive will handle the actual selection
+    }
+  }
 </script>
 
 <div class={classNames("", _class)}>
@@ -98,15 +114,15 @@
         })}
       >
         <!--  NOTE: Svelte is very anoyingly wrong about these TS errors. It cannot seem to discriminate types -->
-        {#if selectedStatus?.icon?.src}
-          <img src={selectedStatus.icon.src} class="h-4 w-4 shrink-0" alt="" />
-        {:else if selectedStatus?.icon?.component}
+        {#if selectedStatus?.icon?.component}
           <svelte:component
             this={selectedStatus.icon.component}
-            class={cn("h-4 w-4 shrink-0", selectedStatus.icon.class)}
+            class={cn(
+              "h-4 w-4 shrink-0",
+              // @ts-ignore
+              selectedStatus.icon.class
+            )}
           />
-        {:else if selectedStatus?.icon?.char}
-          <code class="text-xl inline-block">{selectedStatus.icon.char}</code>
         {:else}
           <IconBrain class="w-5 h-5 !text-[#30CEC0] scale-[1.2]" />
         {/if}
@@ -121,35 +137,20 @@
       side="bottom"
       align="start"
     >
-      <Command.Root>
-        <Command.Input placeholder={loading ? "Loading..." : "Model..."} />
+      <Command.Root on:keydown={handleKeydown}>
+        <Command.Input placeholder={loading ? "Loading..." : "Model..."} on:input={handleSearch} />
         <Command.List>
           <Command.Empty>No results found.</Command.Empty>
-          {#each Object.entries(optionGroups) as [name, models]}
+          {#each Object.entries(filteredGroups) as [name, models]}
             <Command.Group heading={name}>
               {#each models as opt}
-                <Command.Item
-                  value={opt.value}
-                  onSelect={(currentValue) => {
-                    handleChange(currentValue);
-                  }}
-                >
-                  {#if opt.icon?.char}
-                    <code class="text-xl inline-block">{opt.icon.char}</code>
-                  {:else if opt.icon?.src}
-                    <img
-                      src={opt.icon.src}
-                      class={cn(
-                        "mr-2 h-4 w-4",
-                        opt.value !== selectedStatus?.value && "text-foreground/40"
-                      )}
-                      alt=""
-                    />
-                  {:else if opt.icon?.component}
+                <Command.Item value={opt.value} onSelect={handleChange}>
+                  {#if opt.icon?.component}
                     <svelte:component
                       this={opt.icon.component}
                       class={cn(
                         "mr-2 h-4 w-4",
+                        // @ts-ignore
                         opt.icon.class,
                         opt.value !== selectedStatus?.value && "text-foreground/40"
                       )}

src/lib/components/ui/command/command-input.svelte

@@ -1,23 +1,34 @@
 <script lang="ts">
-	import { Command as CommandPrimitive } from "cmdk-sv";
-	import { Search } from "lucide-svelte";
-	import { cn } from "$lib/utils";
+  import { Command as CommandPrimitive } from "cmdk-sv";
+  import { Search } from "lucide-svelte";
+  import { cn } from "$lib/utils";
+  import { createEventDispatcher } from "svelte";
 
-	type $$Props = CommandPrimitive.InputProps;
+  type $$Props = CommandPrimitive.InputProps;
 
-	let className: string | undefined | null = undefined;
-	export { className as class };
-	export let value: string = "";
+  let className: string | undefined | null = undefined;
+  export { className as class };
+  export let value: string = "";
+
+  const dispatch = createEventDispatcher();
+
+  function handleInput(event: Event) {
+    const target = event.target as HTMLInputElement;
+    value = target.value;
+    dispatch("input", value);
+  }
 </script>
 
-<div class="flex items-center border-b px-2" data-cmdk-input-wrapper="">
-	<Search class="mr-2 h-4 w-4 shrink-0 opacity-50" />
-	<CommandPrimitive.Input
-		class={cn(
-			"flex h-11 w-full rounded-md bg-transparent py-3 text-sm outline-none placeholder:text-muted-foreground disabled:cursor-not-allowed disabled:opacity-50",
-			className
-		)}
-		{...$$restProps}
-		bind:value
-	/>
+<div class="flex items-center border-b px-3" data-cmdk-input-wrapper="">
+  <Search class="mr-2 h-4 w-4 shrink-0 opacity-50" />
+  <input
+    class={cn(
+      "flex h-11 w-full rounded-md bg-transparent py-3 text-sm outline-none placeholder:text-muted-foreground disabled:cursor-not-allowed disabled:opacity-50",
+      className
+    )}
+    type="text"
+    bind:value
+    on:input={handleInput}
+    {...$$restProps}
+  />
 </div>

src/lib/components/ui/command/command-score.ts

@@ -0,0 +1,35 @@
+/**
+ * Calculate a score for how well a string matches a search query.
+ * Higher scores indicate better matches.
+ */
+export function commandScore(str: string, query: string): number {
+  if (!str || !query) return 0;
+
+  str = str.toLowerCase();
+  query = query.toLowerCase();
+
+  // Exact match gets highest score
+  if (str === query) return 1;
+
+  // Check if string starts with query
+  if (str.startsWith(query)) return 0.8;
+
+  // Check if string contains query
+  if (str.includes(query)) return 0.5;
+
+  // Check if all characters in query appear in order in str
+  let strIndex = 0;
+  let queryIndex = 0;
+
+  while (strIndex < str.length && queryIndex < query.length) {
+    if (str[strIndex] === query[queryIndex]) {
+      queryIndex++;
+    }
+    strIndex++;
+  }
+
+  // If all characters were found in order, give a lower score
+  if (queryIndex === query.length) return 0.3;
+
+  return 0;
+}

src/lib/components/ui/command/command.svelte

@@ -1,22 +1,29 @@
 <script lang="ts">
-	import { Command as CommandPrimitive } from "cmdk-sv";
-	import { cn } from "$lib/utils";
+  import { Command as CommandPrimitive } from "cmdk-sv";
+  import { cn } from "$lib/utils";
+  import { createEventDispatcher } from "svelte";
 
-	type $$Props = CommandPrimitive.CommandProps;
+  type $$Props = CommandPrimitive.CommandProps;
 
-	export let value: $$Props["value"] = undefined;
+  let className: string | undefined | null = undefined;
+  export { className as class };
+  export let value: string = "";
 
-	let className: string | undefined | null = undefined;
-	export { className as class };
+  const dispatch = createEventDispatcher();
+
+  function handleKeydown(event: CustomEvent<KeyboardEvent>) {
+    dispatch("keydown", event.detail);
+  }
 </script>
 
 <CommandPrimitive.Root
-	class={cn(
-		"flex h-full w-full flex-col overflow-hidden rounded-md bg-popover text-popover-foreground",
-		className
-	)}
-	bind:value
-	{...$$restProps}
+  class={cn(
+    "flex h-full w-full flex-col overflow-hidden rounded-md bg-popover text-popover-foreground",
+    className
+  )}
+  bind:value
+  {...$$restProps}
+  on:keydown={handleKeydown}
 >
-	<slot />
+  <slot />
 </CommandPrimitive.Root>

src/lib/db.ts

@@ -23,6 +23,7 @@ let schemaUrl = schema_0002;
 
 import { llmProviders, openAiConfig } from "./stores/stores/llmProvider";
 import { profilesStore } from "./stores/stores/llmProfile";
+import type OpenAI from "openai";
 
 const legacyDbNames = [
   "chat_db-v1",
@@ -306,6 +307,7 @@ export interface LLMProviderRow {
 export type LLMProvider = Omit<LLMProviderRow, "created_at" | "enabled"> & {
   createdAt: Date;
   enabled: boolean;
+  client?: OpenAI; // Optional client property for custom SDK instances
 };
 
 export interface VecToFragRow {