examples: added pre-commit check and format all notebooks (#1220)

* added pre-commit check for example notebooks --------- Co-authored-by: zilto <tjean@DESKTOP-V6JDCS2>
DAGWorks-Inc · Nov 7, 2024 · d6f6f90 · d6f6f90
1 parent 6367a45
commit d6f6f90
Show file tree

Hide file tree

Showing 71 changed files with 7,385 additions and 3,964 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -5,27 +5,37 @@
 #   $ pre-commit install
 exclude: '^docs/code-comparisons/'  # skip the code comparisons directory
 repos:
-- repo: https://github.com/astral-sh/ruff-pre-commit
-  # Ruff version.
-  rev: v0.5.7
-  hooks:
-    # Run the linter.
-    - id: ruff
-      args: [ --fix ]
-    # Run the formatter.
-    - id: ruff-format
-      # args: [ --diff ]  # Use for previewing changes
--   repo: https://github.com/pre-commit/pre-commit-hooks
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.5.7
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format
+        # args: [ --diff ]  # Use for previewing changes
+  - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0
     hooks:
-    -   id: trailing-whitespace
-    # ensures files are either empty or end with a blank line
-    -   id: end-of-file-fixer
-    # sorts requirements
-    -   id: requirements-txt-fixer
-    # valid python file
-    -   id: check-ast
--   repo: https://github.com/pycqa/flake8
+      - id: trailing-whitespace
+      # ensures files are either empty or end with a blank line
+      - id: end-of-file-fixer
+      # sorts requirements
+      - id: requirements-txt-fixer
+      # valid python file
+      - id: check-ast
+  - repo: https://github.com/pycqa/flake8
     rev: 7.1.1
     hooks:
-    - id: flake8
+      - id: flake8
+  - repo: local
+    hooks:
+      - id: validate-example-notebooks
+        name: validate example notebooks
+        entry: python ./examples/validate_examples.py  # script is called from root directory
+        language: python  # script requires the Python interpreter
+        require_serial: true  # don't run in parallel because one call of the script parses all files
+        additional_dependencies: [nbformat]  # Python dependency for the script
+        files: ^examples/.*\.ipynb$  # this catches all .ipynb under examples/
+        types: [jupyter]  # jupyter is alias for .ipynb files
diff --git a/examples/LLM_Workflows/GraphRAG/ingest_notebook.ipynb b/examples/LLM_Workflows/GraphRAG/ingest_notebook.ipynb
@@ -3,43 +3,44 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "1ad0bb7d-58db-44c5-aa17-7db4f9af9f59",
+   "id": "cb3cf1fc",
    "metadata": {},
+   "outputs": [],
    "source": [
-    "# install requirements\n",
-    "!pip install falkordb openai sf-hamilton[sdk,visualization] "
-   ],
-   "outputs": []
+    "# Execute this cell to install dependencies\n",
+    "%pip install sf-hamilton[sdk,visualization] falkordb openai"
+   ]
   },
   {
    "cell_type": "markdown",
-   "id": "dbda8a07-0167-4cdf-bd1f-be233d73f53e",
+   "id": "4682d46e",
    "metadata": {},
    "source": [
-    "# Ingestion Notebook\n",
+    "# Ingestion Notebook [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dagworks-inc/hamilton/blob/main/examples/LLM_Workflows/GraphRAG/ingest_notebook.ipynb) [![GitHub badge](https://img.shields.io/badge/github-view_source-2b3137?logo=github)](https://github.com/dagworks-inc/hamilton/blob/main/examples/LLM_Workflows/GraphRAG/ingest_notebook.ipynb)\n",
+    "\n",
     "\n",
     "In this notebook we see how to load data into FalkorDB using [Hamilton](https://github.com/dagWorks-Inc/hamilton)."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 4,
-   "id": "c988cb764ad158b8",
+   "id": "9c81ff55",
    "metadata": {
     "ExecuteTime": {
      "end_time": "2024-05-26T05:05:26.275402Z",
      "start_time": "2024-05-26T05:05:25.579866Z"
     }
    },
+   "outputs": [],
    "source": [
     "# load jupyter magic to help display Hamilton in a notebook\n",
     "%load_ext hamilton.plugins.jupyter_magic"
-   ],
-   "outputs": []
+   ]
   },
   {
    "cell_type": "markdown",
-   "id": "b26d3413-42ea-4f92-b951-9a2450f0ddb4",
+   "id": "ee990a80",
    "metadata": {},
    "source": [
     "## Fighter ingestion pipeline\n",
@@ -52,13 +53,14 @@
   {
    "cell_type": "code",
    "execution_count": 5,
-   "id": "41492c8cd1ad9b26",
+   "id": "cd97754d",
    "metadata": {
     "ExecuteTime": {
      "end_time": "2024-05-26T05:12:04.660817Z",
      "start_time": "2024-05-26T05:12:04.295564Z"
     }
    },
+   "outputs": [],
    "source": [
     "%%cell_to_module -m load_fighters --display\n",
     "\n",
@@ -130,12 +132,11 @@
     "    q = \"UNWIND $fighters as fighter CREATE (f:Fighter) SET f = fighter\"\n",
     "    graph.query(q, {'fighters': records})\n",
     "    return len(records)\n"
-   ],
-   "outputs": []
+   ]
   },
   {
    "cell_type": "markdown",
-   "id": "2913b8eb-8c2f-4886-a319-da8a5563991d",
+   "id": "f56e3fa6",
    "metadata": {},
    "source": [
     "# Fights ingestion pipeline\n",
@@ -150,13 +151,14 @@
   {
    "cell_type": "code",
    "execution_count": 6,
-   "id": "b97f10cd9937d5dd",
+   "id": "c1ae988c",
    "metadata": {
     "ExecuteTime": {
      "end_time": "2024-05-26T05:44:01.899490Z",
      "start_time": "2024-05-26T05:44:01.430713Z"
     }
    },
+   "outputs": [],
    "source": [
     "%%cell_to_module -m load_fights --display\n",
     "\n",
@@ -244,12 +246,11 @@
     "\n",
     "def collect_writes(write_to_graph: Collect[str]) -> int:\n",
     "    return len(list(write_to_graph))"
-   ],
-   "outputs": []
+   ]
   },
   {
    "cell_type": "markdown",
-   "id": "235966ca-f05d-441e-aa99-27c1e1f6802a",
+   "id": "8b1a2966",
    "metadata": {},
    "source": [
     "# Exercise the pipelines\n",
@@ -260,13 +261,14 @@
   {
    "cell_type": "code",
    "execution_count": 7,
-   "id": "357252c5857a3f7b",
+   "id": "9c7fdd81",
    "metadata": {
     "ExecuteTime": {
      "end_time": "2024-05-26T05:44:05.459158Z",
      "start_time": "2024-05-26T05:44:05.448476Z"
     }
    },
+   "outputs": [],
    "source": [
     "import falkordb\n",
     "from hamilton import driver\n",
@@ -278,12 +280,11 @@
     "# Clear previous graph\n",
     "if \"UFC\" in db.list_graphs():\n",
     "    g.delete()"
-   ],
-   "outputs": []
+   ]
   },
   {
    "cell_type": "markdown",
-   "id": "f200ae21-1ab3-4bb8-ada1-876ddbd1fb5f",
+   "id": "0836b087",
    "metadata": {},
    "source": [
     "### Note about the Hamilton UI\n",
@@ -295,13 +296,14 @@
   {
    "cell_type": "code",
    "execution_count": 8,
-   "id": "ec7e41c83b69311c",
+   "id": "f5443a7f",
    "metadata": {
     "ExecuteTime": {
      "end_time": "2024-05-26T05:44:33.832688Z",
      "start_time": "2024-05-26T05:44:06.038046Z"
     }
    },
+   "outputs": [],
    "source": [
     "# if you have the Hamilton UI you can see progress:\n",
     "# from hamilton_sdk import adapters\n",
@@ -324,19 +326,19 @@
     ")\n",
     "\n",
     "fighter_loader.execute([\"write_to_graph\"], inputs={\"graph\": g})"
-   ],
-   "outputs": []
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": 9,
-   "id": "3b703eeb5a360",
+   "id": "3107f89b",
    "metadata": {
     "ExecuteTime": {
      "end_time": "2024-05-26T05:49:42.579219Z",
      "start_time": "2024-05-26T05:44:33.835479Z"
     }
    },
+   "outputs": [],
    "source": [
     "# if you have the Hamilton UI you can see progress:\n",
     "# from hamilton_sdk import adapters\n",
@@ -356,29 +358,28 @@
     "    .build()\n",
     ")\n",
     "fights_loader.execute([\"collect_writes\"], inputs={\"graph\": g})"
-   ],
-   "outputs": []
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "59e6ba771b3563f2",
+   "id": "b0a0d9fe",
    "metadata": {
     "ExecuteTime": {
      "end_time": "2024-05-26T05:49:42.591409Z",
      "start_time": "2024-05-26T05:49:42.580724Z"
     }
    },
-   "source": [],
-   "outputs": []
+   "outputs": [],
+   "source": []
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "7981e2c8294873e5",
+   "id": "4c83c11d",
    "metadata": {},
-   "source": [],
-   "outputs": []
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {