From 85cb81fec21582dfff963345095e5fb6b3393398 Mon Sep 17 00:00:00 2001
From: irexyc <irexyc@gmail.com>
Date: Thu, 25 Jul 2024 17:00:58 +0800
Subject: [PATCH 1/3] adapt new code

---
 lmdeploy/vl/model/minicpmv.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/lmdeploy/vl/model/minicpmv.py b/lmdeploy/vl/model/minicpmv.py
index 284a943356..0f04b54ddb 100644
--- a/lmdeploy/vl/model/minicpmv.py
+++ b/lmdeploy/vl/model/minicpmv.py
@@ -54,6 +54,20 @@ def build_model(self):
         else:
             self._forward_func = self._forward_v2
 
+        # adapt new code commit 287e3f85
+        if not hasattr(model, 'slice_image'):
+            from transformers import AutoProcessor
+            processor = AutoProcessor.from_pretrained(self.model_path,
+                                                      trust_remote_code=True)
+            model.slice_image = processor.image_processor.slice_image
+
+            def _reshape_by_patch(x):
+                x = x.numpy()
+                x = processor.image_processor.reshape_by_patch(x)
+                return torch.from_numpy(x)
+
+            model.reshape_by_patch = _reshape_by_patch
+
     def _get_slice_image(self, image: Image):
         slice_images = []
         source_image, patches, best_grid = self.model.slice_image(image)

From 42a8aef55968069053db94a27c2a81467a742e8e Mon Sep 17 00:00:00 2001
From: irexyc <irexyc@gmail.com>
Date: Fri, 26 Jul 2024 11:11:53 +0000
Subject: [PATCH 2/3] add assert

---
 lmdeploy/vl/templates.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lmdeploy/vl/templates.py b/lmdeploy/vl/templates.py
index a75938376e..8c0bed8ac5 100644
--- a/lmdeploy/vl/templates.py
+++ b/lmdeploy/vl/templates.py
@@ -305,6 +305,8 @@ class GLM4VChatTemplateWrapper(VLChatTemplateWrapper):
 def get_vl_prompt_template(model_path: str, chat_template: BaseModel,
                            model_name: str) -> VLChatTemplateWrapper:
     """get vision language prompt template."""
+    assert type(chat_template) != type(BaseModel()), 'failed to match ' \
+        'chat template, please explicit set chat_template_config' # noqa E721
     if model_name == 'yi-vl':
         return YiVLChatTemplateWrapper(chat_template)
 

From 92216185173c79a681188533dcd1ba1342de3085 Mon Sep 17 00:00:00 2001
From: irexyc <irexyc@gmail.com>
Date: Thu, 1 Aug 2024 08:03:25 +0000
Subject: [PATCH 3/3] fix comments

---
 lmdeploy/vl/model/minicpmv.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lmdeploy/vl/model/minicpmv.py b/lmdeploy/vl/model/minicpmv.py
index 0f04b54ddb..3728a11544 100644
--- a/lmdeploy/vl/model/minicpmv.py
+++ b/lmdeploy/vl/model/minicpmv.py
@@ -62,9 +62,9 @@ def build_model(self):
             model.slice_image = processor.image_processor.slice_image
 
             def _reshape_by_patch(x):
-                x = x.numpy()
-                x = processor.image_processor.reshape_by_patch(x)
-                return torch.from_numpy(x)
+                out = x.cpu().numpy()
+                out = processor.image_processor.reshape_by_patch(out)
+                return torch.from_numpy(out).to(device=x.device)
 
             model.reshape_by_patch = _reshape_by_patch