New Trello document loader

docs/modules/indexes/document_loaders/examples/trello.ipynb

@@ -0,0 +1,110 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Trello\n",
+    "\n",
+    ">[Trello](https://www.atlassian.com/software/trello) is a web-based project management and collaboration tool that allows individuals and teams to organize and track their tasks and projects. It provides a visual interface known as a \"board\" where users can create lists and cards to represent their tasks and activities.\n",
+    "\n",
+    "The TrelloLoader allows you to load cards from a Trello board and is implemented on top of [py-trello](https://pypi.org/project/py-trello/)\n",
+    "\n",
+    "This currently supports `api_key/token` only.\n",
+    "\n",
+    "1. Credentials generation: https://trello.com/power-ups/admin/\n",
+    "\n",
+    "2. Click in the manual token generation link to get the token.\n",
+    "\n",
+    "This loader allows you to provide the board name to pull in the corresponding cards into Document objects.\n",
+    "\n",
+    "Notice that the board \"name\" is also called \"title\" in oficial documentation:\n",
+    "\n",
+    "https://support.atlassian.com/trello/docs/changing-a-boards-title-and-description/\n",
+    "\n",
+    "You can also specify several load parameters to include / remove different fields both from the document page_content properties and metadata.\n",
+    "\n",
+    "## Features\n",
+    "- Load cards from a Trello board.\n",
+    "- Filter cards based on their status (open or closed).\n",
+    "- Include card names, comments, and checklists in the loaded documents.\n",
+    "- Customize the additional metadata fields to include in the document.\n",
+    "\n",
+    "By default all card fields are included for the full text page_content and metadata accordinly.\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "#!pip install py-trello beautifulsoup4"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import TrelloLoader\n",
+    "\n",
+    "loader = TrelloLoader(api_key, api_token)\n",
+    "\n",
+    "# Get the open cards from \"Awesome Board\"\n",
+    "documents = loader.load(\n",
+    "    board_name=\"Awesome Board\",\n",
+    "    card_filter = \"open\"\n",
+    "    )\n",
+    "\n",
+    "# Get all the cards from \"Awesome Board\" but only include the\n",
+    "# card list(column) as extra metadata.\n",
+    "documents = loader.load(\n",
+    "    board_name=\"Another Board\",\n",
+    "    extra_metadata=[\"list\"] \n",
+    "    )\n",
+    "\n",
+    "# Get the closed cards from \"Awesome Board\" and exclude the card name,\n",
+    "# checklist and comments from the Document page_content text.\n",
+    "documents = loader.load(\n",
+    "    board_name=\"Awesome Board\",\n",
+    "    card_filter = \"closed\",\n",
+    "    include_card_name= False,\n",
+    "    include_checklist= False,\n",
+    "    include_comments= False\n",
+    "    )"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "cc99336516f23363341912c6723b01ace86f02e26b4290be1efc0677e2e2ec24"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

langchain/document_loaders/__init__.py

@@ -87,6 +87,7 @@
 )
 from langchain.document_loaders.text import TextLoader
 from langchain.document_loaders.toml import TomlLoader
+from langchain.document_loaders.trello import TrelloLoader
 from langchain.document_loaders.twitter import TwitterTweetLoader
 from langchain.document_loaders.unstructured import (
     UnstructuredAPIFileIOLoader,
@@ -192,6 +193,7 @@
     "StripeLoader",
     "TextLoader",
     "TomlLoader",
+    "TrelloLoader",
     "TwitterTweetLoader",
     "UnstructuredAPIFileIOLoader",
     "UnstructuredAPIFileLoader",

langchain/document_loaders/trello.py

@@ -0,0 +1,124 @@
+"""Loader that loads cards from Trello"""
+from typing import List, Optional
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.base import BaseLoader
+
+class TrelloLoader(BaseLoader):
+    """Trello loader. Reads all cards from a Trello board.
+
+    Args:
+        api_key (str): Trello API key.
+        api_token (str): Trello API token.
+    """
+
+    def __init__(self, api_key: str, api_token: str):
+        """Initialize Trello loader."""
+        self.api_key = api_key
+        self.api_token = api_token
+
+    def load(
+        self,
+        board_name: str,
+        card_filter: Optional[str] = "all",
+        include_card_name: Optional[bool] = True,
+        include_comments: Optional[bool] = True,
+        include_checklist: Optional[bool] = True,
+        extra_metadata: Optional[List[str]] = [
+            "due_date", "labels", "list", "is_closed"],
+    ) -> List[Document]:
+        """Loads all cards from the specified Trello board.
+        You can filter the cards, metadata and text included by using the optional parameters.
+
+        Args:
+            board_name (str): The name of the Trello board.
+            card_filter (str, optional): Use "closed" / "open". Defaults to "all".
+            include_card_name (bool, optional): Whether to include the name of the card in the document. Defaults to False.
+            include_comments (bool, optional): Whether to include the comments on the card in the document. Defaults to False.
+            include_checklist (bool, optional): Whether to include the checklist on the card in the document. Defaults to False.
+            extra_metadata (List[str], optional): List of additional metadata fields to include as document metadata. Defaults to ["due_date", "labels", "list", "is_closed"].
+        Returns:
+            A list of documents, one for each card in the board.
+        """
+
+        try:
+            from trello import TrelloClient  # type: ignore
+        except ImportError as ex:
+            raise ImportError(
+                "Could not import trello python package. "
+                "Please install it with `pip install py-trello`."
+            ) from ex
+        try:
+            from bs4 import BeautifulSoup  # type: ignore
+        except ImportError as ex:
+            raise ImportError(
+                "`beautifulsoup4` package not found, please run"
+                " `pip install beautifulsoup4`"
+            ) from ex
+
+        docs: List[Document] = []
+        client = TrelloClient(api_key=self.api_key, token=self.api_token)
+
+        # Find the board with the matching name
+        board = next((b for b in client.list_boards()
+                     if b.name == board_name), None)
+        if board:
+            board_id = board.id
+        else:
+            raise ValueError(f"Board `{board_name}` not found.")
+
+        # Get the lists in the board
+        lists = board.list_lists()
+
+        # Create a dictionary with the list IDs as keys and the list names as values
+        list_dict = {}
+        for list_item in lists:
+            list_dict[list_item.id] = list_item.name
+
+        # Get Cards on the board
+        board = client.get_board(board_id)
+        cards = board.get_cards(card_filter=card_filter)
+        for card in cards:
+            text_content = ""
+            if include_card_name:
+                text_content = card.name + "\n"
+            description = card.description.strip()
+            if description:
+                text_content += BeautifulSoup(card.description,
+                                              "lxml").get_text()
+
+            if include_checklist:
+                # Get all the checklit items on the card
+                items = []
+                for checklist in card.checklists:
+                    items.extend(
+                        [f"{item['name']}:{item['state']}" for item in checklist.items])
+                    text_content += f"\n{checklist.name}\n" + "\n".join(items)
+
+            if include_comments:
+                # Get all the comments on the card
+                comments = [BeautifulSoup(comment['data']['text'], "lxml").get_text()
+                            for comment in card.comments]
+                text_content += "Comments:" + "\n".join(comments)
+
+            # Default metadata fields
+            metadata = {
+                "title": card.name,
+                "id": card.id,
+                "url": card.url,
+            }
+
+            # Extra metadata fields. Card object is not subscriptable.
+            if "labels" in extra_metadata:
+                metadata["labels"] = [label.name for label in card.labels]
+            if "list" in extra_metadata:
+                if card.list_id in list_dict:
+                    metadata["list"] = list_dict[card.list_id]
+            if "is_close" in extra_metadata:
+                metadata["is_closed"] = card.is_closed
+            if "due_date" in extra_metadata:
+                metadata["due_date"] = card.due_date
+
+            doc = Document(page_content=text_content, metadata=metadata)
+            docs.append(doc)
+        return docs