Fix batch infer for gemma3vl (#3592)

* fix gemma3vl

* upgrade to torch2.6

Author: RunningLeon

.github/workflows/pr_ete_test.yml

@@ -58,9 +58,9 @@ jobs:
         run: |
           python3 -m pip cache dir
           python3 -m pip install --upgrade pip setuptools==69.5.1
-          python3 -m pip install torch==2.5.1 torchvision==0.20.1
+          python3 -m pip install torch==2.6.0 torchvision==0.21.0
           # the install packeage from. https://github.com/Dao-AILab/flash-attention/releases
-          python3 -m pip install /root/packages/flash_attn-2.6.3+cu123torch2.3cxx11abiFALSE-cp310-cp310-linux_x86_64.whl
+          python3 -m pip install /root/packages/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl
       - name: Build lmdeploy
         run: |
           cp /nvme/qa_test_models/offline_pkg/openmpi-4.1.5.tar.gz .

lmdeploy/vl/model/gemma3_vl.py

@@ -74,11 +74,20 @@ def preprocess(self, messages: List[Dict]) -> List[Dict]:
         )
         images = self.collect_images(messages)
         images = [image.convert('RGB') for image, _ in images]
+        num_image = len(images)
         images = make_nested_list_of_images(images)
         image_inputs = self.processor.image_processor(images, **output_kwargs['images_kwargs'])
-        image_inputs['image_tokens'] = self.image_tokens
-        image_inputs['image_token_id'] = self.image_token_id
-        messages.append(dict(role='preprocess', content=[image_inputs]))
+        outputs = []
+        for idx in range(num_image):
+            pixel_values = image_inputs['pixel_values'][idx:idx + 1, ...]
+            num_crops = image_inputs['num_crops'][:idx:idx + 1]
+            data = dict(pixel_values=pixel_values,
+                        num_crops=num_crops,
+                        image_tokens=self.image_tokens,
+                        image_token_id=self.image_token_id)
+            outputs.append(data)
+
+        messages.append(dict(role='preprocess', content=outputs))
         return messages
 
     @torch.no_grad()