harvard-edge · profvjreddi · Dec 5, 2024 · Dec 2, 2024 · Dec 2, 2024 · Dec 2, 2024
diff --git a/_quarto.yml b/_quarto.yml
@@ -159,6 +159,7 @@ book:
       - contents/labs/raspi/image_classification/image_classification.qmd
       - contents/labs/raspi/object_detection/object_detection.qmd
       - contents/labs/raspi/llm/llm.qmd
+      - contents/labs/raspi/vlm/vlm.qmd
     - part: contents/labs/shared/shared.qmd
       chapters:
       - contents/labs/shared/kws_feature_eng/kws_feature_eng.qmd

diff --git a/contents/labs/overview.qmd b/contents/labs/overview.qmd
@@ -40,21 +40,23 @@ We have included laboratory materials for three key devices that represent diffe
 * XIAO ESP32S3: A versatile, compact board suitable for keyword spotting and motion detection tasks.
 * Raspberry Pi: A flexible platform for more computationally intensive tasks, including small language models and various classification and detection applications.
 
-+----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
-| Exercise                   | [Nicla Vision](https://store.arduino.cc/products/nicla-vision) | [XIAO ESP32S3](https://wiki.seeedstudio.com/xiao_esp32s3_getting_started/) | [Raspberry Pi](https://www.raspberrypi.com/) |
-+:===========================+:===============================================================+:===========================================================================+:=============================================+
-| Installation & Setup       | &check;                                                        | &check;                                                                    | &check;                                      |
-+----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
-| Keyword Spotting (KWS)     | &check;                                                        | &check;                                                                    |                                              |
-+----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
-| Image Classification       | &check;                                                        | &check;                                                                    | &check;                                      |
-+----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
-| Object Detection           | &check;                                                        | &check;                                                                    | &check;                                      |
-+----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
-| Motion Detection           | &check;                                                        | &check;                                                                    |                                              |
-+----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
-| Small Language Models (SLM)|                                                                |                                                                            | &check;                                      |
-+----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
++-----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
+| Exercise                    | [Nicla Vision](https://store.arduino.cc/products/nicla-vision) | [XIAO ESP32S3](https://wiki.seeedstudio.com/xiao_esp32s3_getting_started/) | [Raspberry Pi](https://www.raspberrypi.com/) |
++:============================+:==============================================================:+:==========================================================================:+:============================================:+
+| Installation & Setup        | &check;                                                        | &check;                                                                    | &check;                                      |
++-----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
+| Keyword Spotting (KWS)      | &check;                                                        | &check;                                                                    |                                              |
++-----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
+| Image Classification        | &check;                                                        | &check;                                                                    | &check;                                      |
++-----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
+| Object Detection            | &check;                                                        | &check;                                                                    | &check;                                      |
++-----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
+| Motion Detection            | &check;                                                        | &check;                                                                    |                                              |
++-----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
+| Small Language Models (SLM) |                                                                |                                                                            | &check;                                      |
++-----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
+| Vision Language Models (VLM)|                                                                |                                                                            | &check;                                      |
++-----------------------------+----------------------------------------------------------------+----------------------------------------------------------------------------+----------------------------------------------+
 
 ## Lab Structure
 

diff --git a/contents/labs/raspi/raspi.qmd b/contents/labs/raspi/raspi.qmd
@@ -6,7 +6,7 @@ These labs offer invaluable hands-on experience with machine learning systems, l
 
 ## Pre-requisites
 
-- **Raspberry Pi**: Ensure you have at least one of the boards: the Raspberry Pi Zero 2W, Raspberry Pi 4 or 5 for the Vision Labs, and the Raspberry 5 for the GenAi lab.
+- **Raspberry Pi**: Ensure you have at least one of the boards: the Raspberry Pi Zero 2W, Raspberry Pi 4 or 5 for the Vision Labs, and the Raspberry 5 for the GenAi labs.
 - **Power Adapter**: To Power on the boards.
     - Raspberry Pi Zero 2-W: 2.5W with a Micro-USB adapter
     - Raspberry Pi 4 or 5: 3.5W with a USB-C adapter
@@ -20,10 +20,14 @@ These labs offer invaluable hands-on experience with machine learning systems, l
 
 ## Exercises
 
-| **Modality** | **Task**              | **Description**            | **Link**                                                |
-| ------------ | --------------------- | -------------------------- | ------------------------------------------------------- |
-| Vision       | Image Classification  | Learn to classify images   | [Link](./image_classification/image_classification.qmd) |
-| Vision       | Object Detection      | Implement object detection | [Link](./object_detection/object_detection.qmd)         |
-| GenAI        | Small Language Models | Deploy SLMs at the Edge    | [Link](./llm/llm.qmd)                                   |
-|              |                       |                            |                                                         |
-
++--------------+------------------------+----------------------------+---------------------------------------------------------+
+| **Modality** | **Task**               | **Description**            | **Link**                                                |
++:=============+:=======================+:===========================+:========================================================+
+| Vision       | Image Classification   | Learn to classify images   | [Link](./image_classification/image_classification.qmd) |
++--------------+------------------------+----------------------------+---------------------------------------------------------+
+| Vision       | Object Detection       | Implement object detection | [Link](./object_detection/object_detection.qmd)         |
++--------------+------------------------+----------------------------+---------------------------------------------------------+
+| GenAI        | Small Language Models  | Deploy SLMs at the Edge    | [Link](./llm/llm.qmd)                                   |
++--------------+------------------------+----------------------------+---------------------------------------------------------+
+| GenAI        | Visual-Language Models | Deploy VLMs at the Edge    | [Link](./vlm/vlm.qmd)                                   |
++--------------+------------------------+----------------------------+---------------------------------------------------------+
diff --git a/contents/labs/raspi/vlm/images/jpeg/cover.jpg b/contents/labs/raspi/vlm/images/jpeg/cover.jpg
diff --git a/contents/labs/raspi/vlm/images/jpeg/florence-diagr.jpg b/contents/labs/raspi/vlm/images/jpeg/florence-diagr.jpg
diff --git a/contents/labs/raspi/vlm/images/jpeg/htop.jpg b/contents/labs/raspi/vlm/images/jpeg/htop.jpg
diff --git a/contents/labs/raspi/vlm/images/jpeg/raspi5-active-cooler.jpg b/contents/labs/raspi/vlm/images/jpeg/raspi5-active-cooler.jpg
diff --git a/contents/labs/raspi/vlm/images/png/Dataset_FLD-5B.png b/contents/labs/raspi/vlm/images/png/Dataset_FLD-5B.png
diff --git a/contents/labs/raspi/vlm/images/png/arch.png b/contents/labs/raspi/vlm/images/png/arch.png
diff --git a/contents/labs/raspi/vlm/images/png/caption-boxes.png b/contents/labs/raspi/vlm/images/png/caption-boxes.png
diff --git a/contents/labs/raspi/vlm/images/png/caption_ground.png b/contents/labs/raspi/vlm/images/png/caption_ground.png
diff --git a/contents/labs/raspi/vlm/images/png/cascade.png b/contents/labs/raspi/vlm/images/png/cascade.png
diff --git a/contents/labs/raspi/vlm/images/png/dog_bottle_seg.png b/contents/labs/raspi/vlm/images/png/dog_bottle_seg.png
diff --git a/contents/labs/raspi/vlm/images/png/fine-tuning.png b/contents/labs/raspi/vlm/images/png/fine-tuning.png
diff --git a/contents/labs/raspi/vlm/images/png/flyer.png b/contents/labs/raspi/vlm/images/png/flyer.png
diff --git a/contents/labs/raspi/vlm/images/png/fusca.png b/contents/labs/raspi/vlm/images/png/fusca.png
diff --git a/contents/labs/raspi/vlm/images/png/htop.png b/contents/labs/raspi/vlm/images/png/htop.png
diff --git a/contents/labs/raspi/vlm/images/png/img_test.png b/contents/labs/raspi/vlm/images/png/img_test.png
diff --git a/contents/labs/raspi/vlm/images/png/jupyter.png b/contents/labs/raspi/vlm/images/png/jupyter.png
diff --git a/contents/labs/raspi/vlm/images/png/od_caption_cat_dog_table.png b/contents/labs/raspi/vlm/images/png/od_caption_cat_dog_table.png
diff --git a/contents/labs/raspi/vlm/images/png/od_cat_dogs.png b/contents/labs/raspi/vlm/images/png/od_cat_dogs.png
diff --git a/contents/labs/raspi/vlm/images/png/od_table.png b/contents/labs/raspi/vlm/images/png/od_table.png
diff --git a/contents/labs/raspi/vlm/images/png/open_vacab_exemples.png b/contents/labs/raspi/vlm/images/png/open_vacab_exemples.png
diff --git a/contents/labs/raspi/vlm/images/png/orange_seg.png b/contents/labs/raspi/vlm/images/png/orange_seg.png
diff --git a/contents/labs/raspi/vlm/images/png/output_ocr.png b/contents/labs/raspi/vlm/images/png/output_ocr.png
diff --git a/contents/labs/raspi/vlm/images/png/result_test.png b/contents/labs/raspi/vlm/images/png/result_test.png
diff --git a/contents/labs/raspi/vlm/images/png/ssh.png b/contents/labs/raspi/vlm/images/png/ssh.png
diff --git a/contents/labs/raspi/vlm/images/png/test-pytorch.png b/contents/labs/raspi/vlm/images/png/test-pytorch.png
diff --git a/contents/labs/raspi/vlm/images/png/wine_glass.png b/contents/labs/raspi/vlm/images/png/wine_glass.png