Upload folder using huggingface_hub

preprocessor_config.json

@@ -1,11 +1,20 @@
 {
+  "auto_map": {
+    "AutoImageProcessor": "processing_fastvlm.FastVLMImageProcessor",
+    "AutoProcessor": "processing_fastvlm.FastVLMProcessor"
+  },
   "crop_size": {
     "height": 1024,
     "width": 1024
   },
+  "data_format": "channels_first",
+  "default_to_square": false,
+  "device": null,
+  "disable_grouping": null,
   "do_center_crop": true,
   "do_convert_rgb": true,
   "do_normalize": true,
+  "do_pad": null,
   "do_rescale": true,
   "do_resize": true,
   "image_mean": [
@@ -13,15 +22,18 @@
     0.0,
     0.0
   ],
-  "image_processor_type": "CLIPImageProcessor",
+  "image_processor_type": "FastVLMImageProcessor",
   "image_std": [
     1.0,
     1.0,
     1.0
   ],
-  "processor_class": "LlavaProcessor",
+  "input_data_format": null,
+  "pad_size": null,
+  "processor_class": "FastVLMProcessor",
   "resample": 3,
   "rescale_factor": 0.00392156862745098,
+  "return_tensors": null,
   "size": {
     "shortest_edge": 1024
   }

processing_fastvlm.py

@@ -0,0 +1,88 @@
+import re
+import torch
+from transformers import ProcessorMixin, BatchFeature, CLIPImageProcessorFast
+from transformers.image_processing_utils import BaseImageProcessor
+from transformers.image_utils import ImageInput
+from typing import Any, Dict, List, Optional, Union
+from PIL import Image
+
+from .llava_qwen import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN
+
+# Adapted from transformers.models.llava_next.image_processing_llava_next.expand_to_square
+def expand_to_square(image: torch.Tensor, background_color=0) -> torch.Tensor:
+    """
+    Expands an image to a square by adding a background color.
+    """
+    c, height, width = image.shape
+    if width == height:
+        return image
+    elif width > height:
+        result = torch.ones((c, width, width), dtype=image.dtype) * background_color
+        result[:, (width - height) // 2 : (width - height) // 2 + height, :] = image
+        return result
+    else:
+        result = torch.ones((c, height, height), dtype=image.dtype) * background_color
+        result[:, :, (height - width) // 2 : (height - width) // 2 + width] = image
+        return result
+
+
+class FastVLMImageProcessor(CLIPImageProcessorFast):
+    def _preprocess(self, images, **kwargs):
+        image_sizes = [image.shape[-2:][::-1] for image in images]
+        images = [expand_to_square(image) for image in images]
+        images = super()._preprocess(images, **kwargs)
+        pixel_values = torch.stack(images.pixel_values, dim=0)
+        return BatchFeature(data={"pixel_values": pixel_values, "image_sizes": image_sizes})
+
+class FastVLMProcessor(ProcessorMixin):
+    attributes = ["tokenizer", "image_processor"]
+    image_processor_class = "AutoImageProcessor"
+    tokenizer_class = "AutoTokenizer"
+
+    def __init__(
+        self,
+        tokenizer,
+        image_processor,
+        chat_template=None,
+        **kwargs
+    ):
+        super().__init__(tokenizer, image_processor, chat_template=chat_template, **kwargs)
+
+    def __call__(
+        self,
+        images: ImageInput = None,
+        text: Optional[Union[str, List[str]]] = None,
+        return_tensors: Optional[str] = "pt",
+        **kwargs,
+    ) -> BatchFeature:
+        if isinstance(text, str):
+            text = [text]
+        elif not isinstance(text, list) and not isinstance(text[0], str):
+            raise TypeError("Invalid input text. Please provide a string, or a list of strings")
+
+        image_inputs = {}
+        if images is not None:
+            image_inputs = self.image_processor(images=images)
+            
+            image_token = torch.tensor([[IMAGE_TOKEN_INDEX]], dtype=torch.int64)
+            input_ids = torch.tensor([], dtype=torch.int64)
+            attention_mask = torch.tensor([], dtype=torch.int64)
+            for prompt in text:
+                image_indexes = [m.start() for m in re.finditer(DEFAULT_IMAGE_TOKEN, prompt)]
+                if len(image_indexes) > 1:
+                    raise ValueError(
+                        f"Expected up to 1 image tokens per prompt, got {len(image_indexes)} instead."
+                    )
+
+                # DEFAULT_IMAGE_TOKEN is -200, not in the vocab (so we can't tokenize the full string)
+                pre, _, post = prompt.partition(DEFAULT_IMAGE_TOKEN)
+                pre_ids  = self.tokenizer(pre, return_tensors="pt", add_special_tokens=False).input_ids
+                post_ids = self.tokenizer(post, return_tensors="pt", add_special_tokens=False).input_ids
+
+                sample_ids = torch.cat([pre_ids, image_token, post_ids], dim=1).to(dtype=torch.int64)
+                sample_mask = torch.ones_like(sample_ids)
+
+                input_ids = torch.cat([input_ids, sample_ids], dim=0)
+                attention_mask = torch.cat([attention_mask, sample_mask], dim=0)
+
+        return BatchFeature(data={"input_ids": input_ids, "attention_mask": attention_mask, **image_inputs}, tensor_type=return_tensors)

processor_config.json

@@ -1,7 +1,6 @@
 {
-  "image_token": "<image>",
-  "num_additional_image_tokens": 0,
-  "patch_size": null,
-  "processor_class": "LlavaProcessor",
-  "vision_feature_select_strategy": null
+  "auto_map": {
+    "AutoProcessor": "processing_fastvlm.FastVLMProcessor"
+  },
+  "processor_class": "FastVLMProcessor"
 }

tokenizer_config.json

@@ -30,6 +30,9 @@
     "<|im_start|>",
     "<|im_end|>"
   ],
+  "auto_map": {
+    "AutoProcessor": "processing_fastvlm.FastVLMProcessor"
+  },
   "bos_token": null,
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
@@ -38,7 +41,7 @@
   "model_max_length": 8192,
   "pad_token": "<|endoftext|>",
   "padding_side": "right",
-  "processor_class": "LlavaProcessor",
+  "processor_class": "FastVLMProcessor",
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",
   "unk_token": null