Lint Python notebooks with ruff. (#12677)

The new ruff version fixed the blocking bugs, and I was able to fairly
easily us to a passing state: ruff fixed some issues on its own, I fixed
a handful by hand, and I added a list of narrowly-targeted exclusions
for files that are currently failing ruff rules that we probably should
look into eventually.

I went pretty lenient on the docs / cookbooks rules, allowing dead code
and such things. Perhaps in the future we may want to tighten the rules
further, but this is already a good set of checks that found real issues
and will prevent them going forward.

.github/workflows/extract_ignored_words_list.py

@@ -3,6 +3,8 @@
 pyproject_toml = toml.load("pyproject.toml")
 
 # Extract the ignore words list (adjust the key as per your TOML structure)
-ignore_words_list = pyproject_toml.get("tool", {}).get("codespell", {}).get("ignore-words-list")
+ignore_words_list = (
+    pyproject_toml.get("tool", {}).get("codespell", {}).get("ignore-words-list")
+)
 
-print(f"::set-output name=ignore_words_list::{ignore_words_list}")
+print(f"::set-output name=ignore_words_list::{ignore_words_list}")

Makefile

@@ -43,10 +43,10 @@ spell_fix:
 
 lint:
 	poetry run ruff docs templates cookbook
-	poetry run black docs templates cookbook --diff
+	poetry run ruff format docs templates cookbook --diff
 
 format format_diff:
-	poetry run black docs templates cookbook
+	poetry run ruff format docs templates cookbook
 	poetry run ruff --select I --fix docs templates cookbook
 
 ######################

cookbook/LLaMA2_sql_chat.ipynb

@@ -67,7 +67,6 @@
     "llama2_code = ChatOllama(model=\"codellama:7b-instruct\")\n",
     "\n",
     "# API\n",
-    "from getpass import getpass\n",
     "from langchain.llms import Replicate\n",
     "\n",
     "# REPLICATE_API_TOKEN = getpass()\n",

cookbook/Semi_Structured_RAG.ipynb

@@ -102,9 +102,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from lxml import html\n",
     "from pydantic import BaseModel\n",
-    "from typing import Any, Optional\n",
+    "from typing import Any\n",
     "from unstructured.partition.pdf import partition_pdf\n",
     "\n",
     "# Get elements\n",
@@ -373,7 +372,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from operator import itemgetter\n",
     "from langchain.schema.runnable import RunnablePassthrough\n",
     "\n",
     "# Prompt template\n",

cookbook/Semi_structured_and_multi_modal_RAG.ipynb

@@ -92,9 +92,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from lxml import html\n",
     "from pydantic import BaseModel\n",
-    "from typing import Any, Optional\n",
+    "from typing import Any\n",
     "from unstructured.partition.pdf import partition_pdf\n",
     "\n",
     "# Get elements\n",
@@ -224,7 +223,7 @@
    "outputs": [],
    "source": [
     "# Prompt\n",
-    "prompt_text = \"\"\"You are an assistant tasked with summarizing tables and text. \\ \n",
+    "prompt_text = \"\"\"You are an assistant tasked with summarizing tables and text. \\\n",
     "Give a concise summary of the table or text. Table or text chunk: {element} \"\"\"\n",
     "prompt = ChatPromptTemplate.from_template(prompt_text)\n",
     "\n",
@@ -313,7 +312,7 @@
     "    # Execute the command and save the output to the defined output file\n",
     "    /Users/rlm/Desktop/Code/llama.cpp/bin/llava -m ../models/llava-7b/ggml-model-q5_k.gguf --mmproj ../models/llava-7b/mmproj-model-f16.gguf --temp 0.1 -p \"Describe the image in detail. Be specific about graphs, such as bar plots.\" --image \"$img\" > \"$output_file\"\n",
     "\n",
-    "done"
+    "done\n"
    ]
   },
   {
@@ -337,7 +336,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import os, glob\n",
+    "import os\n",
+    "import glob\n",
     "\n",
     "# Get all .txt file summaries\n",
     "file_paths = glob.glob(os.path.expanduser(os.path.join(path, \"*.txt\")))\n",
@@ -644,7 +644,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from operator import itemgetter\n",
     "from langchain.schema.runnable import RunnablePassthrough\n",
     "\n",
     "# Prompt template\n",

cookbook/Semi_structured_multi_modal_RAG_LLaMA2.ipynb

@@ -82,10 +82,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import pandas as pd\n",
-    "from lxml import html\n",
     "from pydantic import BaseModel\n",
-    "from typing import Any, Optional\n",
+    "from typing import Any\n",
     "from unstructured.partition.pdf import partition_pdf\n",
     "\n",
     "# Path to save images\n",
@@ -223,7 +221,7 @@
    "outputs": [],
    "source": [
     "# Prompt\n",
-    "prompt_text = \"\"\"You are an assistant tasked with summarizing tables and text. \\ \n",
+    "prompt_text = \"\"\"You are an assistant tasked with summarizing tables and text. \\\n",
     "Give a concise summary of the table or text. Table or text chunk: {element} \"\"\"\n",
     "prompt = ChatPromptTemplate.from_template(prompt_text)\n",
     "\n",
@@ -312,7 +310,7 @@
     "    # Execute the command and save the output to the defined output file\n",
     "    /Users/rlm/Desktop/Code/llama.cpp/bin/llava -m ../models/llava-7b/ggml-model-q5_k.gguf --mmproj ../models/llava-7b/mmproj-model-f16.gguf --temp 0.1 -p \"Describe the image in detail. Be specific about graphs, such as bar plots.\" --image \"$img\" > \"$output_file\"\n",
     "\n",
-    "done"
+    "done\n"
    ]
   },
   {
@@ -322,7 +320,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import os, glob\n",
+    "import os\n",
+    "import glob\n",
     "\n",
     "# Get all .txt files in the directory\n",
     "file_paths = glob.glob(os.path.expanduser(os.path.join(path, \"*.txt\")))\n",
@@ -531,7 +530,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from operator import itemgetter\n",
     "from langchain.schema.runnable import RunnablePassthrough\n",
     "\n",
     "# Prompt template\n",