Merge pull request #180 from andrewnguonly/1.0.15

Pull request for app version `1.0.15`
andrewnguonly · May 18, 2024 · 5ef2b7c · 5ef2b7c
2 parents ee30bf5 + 58d8498
commit 5ef2b7c
Show file tree

Hide file tree

Showing 5 changed files with 93 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -113,7 +113,7 @@ If you don't have `npm` installed, you can download the pre-built extension pack
 Right-click on the extension icon and select `Options` to access the extension's [Options page](https://developer.chrome.com/docs/extensions/develop/ui/options-page).
 
 - **Ollama Model**: Select desired model (e.g. `llama2`)
-- **Ollama Embedding Model**: Select desired embedding model (e.g. `nomic-embed-text`). **Caution**: Using a different embedding model requires Ollama to swap models, which may incur undesired latency in the app. This is a known limitation in Ollama and may be improved in the future.
+- **Ollama Embedding Model**: Select desired embedding model (e.g. `nomic-embed-text`). Specify the `OLLAMA_MAX_LOADED_MODELS=2` environment variable when starting the Ollama server to allow multiple models to be loaded into memory.
 - **Ollama Host**: Select desired host (defaults to `http://localhost:11434`)
 - **Vector Store TTL (minutes)**: Number of minutes to store a URL's content in the vector store cache.
 - **Content Parser Config**: Lumos's default content parser will extract all text content between a page's `<body></body>` tag. To customize the content parser, add an entry to the configuration.
@@ -218,6 +218,7 @@ Note: Content that is highlighted will not be cached in the vector store cache.
 ## Shortcuts
 
 - `cmd + c`: Copy last message to clipboard.
+- `cmd + b`: Load clipboard text as a file attachment.
 - `cmd + j`: Toggle `Disable content parsing` checkbox.
 - `cmd + k`: Clear all messages.
 - `cmd + ;`: Open/close Chat History panel.
@@ -249,6 +250,10 @@ Supported image types:
 - `.jpeg`, `.jpg`
 - `.png`
 
+### Clipboard Content
+
+Clipboard content can be uploaded as a file attachment. Use the `cmd + b` shortcut key to load clipboard text as a file attachment.
+
 ## Tools (Experimental)
 
 Lumos invokes [Tools](https://js.langchain.com/docs/modules/agents/tools/) automatically based on the provided prompt. See documentation and examples [here](./docs/tools.md).

diff --git a/manifest.json b/manifest.json
@@ -1,5 +1,5 @@
 {
-    "version": "1.0.14",
+    "version": "1.0.15",
     "manifest_version": 3,
     "name": "Lumos",
     "description": "An LLM co-pilot for browsing the web, powered by local LLMs. Your prompts never leave the browser.",
@@ -20,7 +20,8 @@
     "permissions": [
         "activeTab",
         "scripting",
-        "storage"
+        "storage",
+        "clipboardRead"
     ],
     "options_ui": {
         "page": "js/options.html",

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "lumos",
-  "version": "1.0.14",
+  "version": "1.0.15",
   "private": true,
   "dependencies": {
     "@chatscope/chat-ui-kit-react": "^1.10.1",

diff --git a/src/components/ChatBar.tsx b/src/components/ChatBar.tsx
@@ -40,6 +40,7 @@ import {
   DEFAULT_HOST,
   apiConnected,
   getLumosOptions,
+  preloadModel,
 } from "../pages/Options";
 import { getHtmlContent } from "../scripts/content";
 
@@ -158,6 +159,31 @@ const ChatBar: React.FC = () => {
     }
   };
 
+  /**
+   * Clipboard content is processed as an attachment.
+   */
+  const loadAttachmentFromClipboard = () => {
+    // get text from clipboard
+    navigator.clipboard.readText().then((text) => {
+      if (text !== "") {
+        // remove non-Latin1 characters, btoa() only accepts Latin1 characters
+        // eslint-disable-next-line no-control-regex
+        const latin1Text = text.replace(/[^\x00-\xFF]/g, "");
+        const currentDate = Date.now();
+
+        // set text as attachment
+        const attachment: Attachment = {
+          name: `__clipboard__${currentDate}`,
+          base64: `data:text/plain;base64,${btoa(latin1Text)}`,
+          lastModified: currentDate,
+        };
+
+        setAttachment(attachment);
+        chrome.storage.session.set({ attachment: attachment });
+      }
+    });
+  };
+
   const handleAttachmentDelete = () => {
     setAttachment(null);
     chrome.storage.session.remove(["attachment"]);
@@ -418,6 +444,13 @@ const ChatBar: React.FC = () => {
           setShowSnackbar(true);
           setSnackbarMessage("Copied!");
           break;
+        case "b":
+          // load clipboard text as attachment
+          //
+          // For security reasons, a user interaction is required to access
+          // the clipboard.
+          loadAttachmentFromClipboard();
+          break;
         case ";":
           // open message history
           setOpenChatHistory(!openChatHistory);
@@ -506,7 +539,13 @@ const ChatBar: React.FC = () => {
 
   useEffect(() => {
     chrome.storage.local.get(
-      ["chatContainerHeight", "selectedModel", "selectedHost", "chatHistory"],
+      [
+        "chatContainerHeight",
+        "selectedModel",
+        "selectedEmbeddingModel",
+        "selectedHost",
+        "chatHistory",
+      ],
       async (data) => {
         if (data.chatContainerHeight) {
           setChatContainerHeight(data.chatContainerHeight);
@@ -540,6 +579,16 @@ const ChatBar: React.FC = () => {
             // persist selected model to local storage
             chrome.storage.local.set({ selectedModel: models[0] });
           }
+
+          // preload inference model
+          const inferenceModel = data.selectedModel || models[0];
+          preloadModel(selectedHost, inferenceModel);
+
+          // preload embedding model
+          const embeddingModel = data.selectedEmbeddingModel || inferenceModel;
+          if (embeddingModel !== inferenceModel) {
+            preloadModel(selectedHost, embeddingModel, true);
+          }
         } else {
           setPromptError(true);
           setPromptPlaceholderText(errMsg);
@@ -702,7 +751,7 @@ const ChatBar: React.FC = () => {
         {attachment && (
           <Tooltip
             placement="top"
-            title={`Unattach ${attachment.name} (ctrl + x)`}
+            title={`Unattach ${attachment.name.startsWith("__clipboard__") ? "clipboard content" : attachment.name} (ctrl + x)`}
           >
             <IconButton
               disabled={submitDisabled}

diff --git a/src/pages/Options.tsx b/src/pages/Options.tsx
@@ -43,11 +43,18 @@ export const DEFAULT_TOOL_CONFIG: ToolConfig = {
     prefix: "calculate:",
   },
 };
-export const MULTIMODAL_MODELS = ["llava", "bakllava"];
+export const MULTIMODAL_MODELS = [
+  "llava",
+  "bakllava",
+  "moondream",
+  "llava-llama3",
+  "llava-phi3",
+];
 export const EMBEDDING_MODELS = [
   "nomic-embed-text",
   "all-minilm",
   "mxbai-embed-large",
+  "snowflake-arctic-embed",
 ];
 export const SUPPORTED_IMG_FORMATS = ["jpeg", "jpg", "png"];
 export const CHAT_CONTAINER_HEIGHT_MIN = 200;
@@ -132,6 +139,20 @@ export const apiConnected = async (
   return [false, [], errMsg];
 };
 
+export const preloadModel = async (
+  host: string,
+  model: string,
+  isEmbedding = false,
+): Promise<void> => {
+  fetch(`${host}/api/${isEmbedding ? "embeddings" : "chat"}`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({ model: model }),
+  });
+};
+
 const Options: React.FC = () => {
   const [model, setModel] = useState("");
   const [embeddingModel, setEmbeddingModel] = useState("");
@@ -276,6 +297,16 @@ const Options: React.FC = () => {
           if (data.selectedEmbeddingModel) {
             setEmbeddingModel(data.selectedEmbeddingModel);
           }
+
+          // preload inference model
+          const inferenceModel = data.selectedModel || models[0];
+          preloadModel(selectedHost, inferenceModel);
+
+          // preload embedding model
+          const embeddingModel = data.selectedEmbeddingModel || inferenceModel;
+          if (embeddingModel !== inferenceModel) {
+            preloadModel(selectedHost, embeddingModel, true);
+          }
         } else {
           setHostError(true);
           setHostHelpText(errMsg);