Support passing raw multimodal data to model

Author: christian-pinto

vllm/model_executor/models/interfaces.py

@@ -120,6 +120,42 @@ def supports_multimodal(
 
     return isinstance(model, SupportsMultiModal)
 
+@runtime_checkable
+class SupportsMultiModalWithRawInput(SupportsMultiModal, Protocol):
+    """The interface required for all multi-modal models."""
+
+    supports_multimodal_raw_input: ClassVar[Literal[True]] = True
+    """
+    A flag that indicates this model supports multi-modal inputs and processes
+    them in their raw form and not embeddings.
+
+    Note:
+        There is no need to redefine this flag if this class is in the
+        MRO of your model class.
+    """
+
+@runtime_checkable
+class _SupportsMultiModalWithRawInput(Protocol):
+    supports_multimodal_raw_input: ClassVar[Literal[True]]
+
+
+@overload
+def supports_multimodal_raw_input(model: object) -> TypeIs[SupportsMultiModalWithRawInput]:
+    ...
+
+
+@overload
+def supports_multimodal_raw_input(model: type[object]) -> TypeIs[type[SupportsMultiModalWithRawInput]]:
+    ...
+
+
+def supports_multimodal_raw_input(
+    model: Union[type[object], object]
+) -> Union[TypeIs[type[SupportsMultiModalWithRawInput]], TypeIs[SupportsMultiModalWithRawInput]]:
+    if isinstance(model, type):
+        return isinstance(model, _SupportsMultiModalWithRawInput)
+
+    return isinstance(model, SupportsMultiModalWithRawInput)
 
 @runtime_checkable
 class SupportsLoRA(Protocol):

vllm/model_executor/models/registry.py

@@ -23,8 +23,9 @@
 
 from .interfaces import (has_inner_state, has_noops, is_attention_free,
                          is_hybrid, supports_cross_encoding,
-                         supports_multimodal, supports_pp,
-                         supports_transcription, supports_v0_only)
+                         supports_multimodal, supports_multimodal_raw_input,
+                         supports_pp, supports_transcription,
+                         supports_v0_only)
 from .interfaces_base import is_text_generation_model
 
 logger = init_logger(__name__)
@@ -267,6 +268,7 @@ class _ModelInfo:
     is_pooling_model: bool
     supports_cross_encoding: bool
     supports_multimodal: bool
+    supports_multimodal_raw_input: bool
     supports_pp: bool
     has_inner_state: bool
     is_attention_free: bool
@@ -283,6 +285,7 @@ def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
             is_pooling_model=True,  # Can convert any model into a pooling model
             supports_cross_encoding=supports_cross_encoding(model),
             supports_multimodal=supports_multimodal(model),
+            supports_multimodal_raw_input=supports_multimodal_raw_input(model),
             supports_pp=supports_pp(model),
             has_inner_state=has_inner_state(model),
             is_attention_free=is_attention_free(model),
@@ -519,6 +522,13 @@ def is_multimodal_model(
     ) -> bool:
         model_cls, _ = self.inspect_model_cls(architectures)
         return model_cls.supports_multimodal
+    
+    def supports_multimodal_raw_input(
+        self,
+        architectures: Union[str, list[str]],
+    ) -> bool:
+        model_cls, _ = self.inspect_model_cls(architectures)
+        return model_cls.supports_multimodal_raw_input
 
     def is_pp_supported_model(
         self,

vllm/v1/worker/gpu_model_runner.py

@@ -552,10 +552,34 @@ def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
         if batch_changed or batch_reordered:
             self.input_batch.refresh()
 
-    def _maybe_add_model_args(self, num_tokens: int, 
-                              model_kwargs: dict[str, Any],
+    def _add_multimodal_inputs_to_model_args(self, model_kwargs: dict[str, Any],
+                                             scheduler_output: "SchedulerOutput"):
+        # Multi-modal data.
+        if scheduler_output:
+            multi_modal_kwargs_list = []
+            for req in scheduler_output.scheduled_new_reqs:
+                req_mm_inputs = req.mm_inputs
+                if not isinstance(req_mm_inputs, list):
+                    req_mm_inputs = list(req_mm_inputs)
+                multi_modal_kwargs_list.extend(req_mm_inputs)
+            multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
+        else:
+            # The only case where SchedulerOtput is None is for a dummy run, let's get some dummy data.
+            dummy_data = self.mm_registry.get_decoder_dummy_data(model_config=self.model_config, seq_len =1)
+            multi_modal_kwargs = MultiModalKwargs.batch([dummy_data.multi_modal_data])
+            
+        model_kwargs.update(multi_modal_kwargs)
+
+    def _maybe_add_model_args(self, num_tokens: int,
+                              model_kwargs: dict[str,Any], 
                               scheduler_output: "SchedulerOutput"=None):
-        pass
+        
+        if self.supports_token_type_ids:
+            model_kwargs["token_type_ids"] =\
+                  self.get_token_type_ids()[:num_tokens]
+
+        if self.model_supports_multimodal_raw_input:
+            self._add_multimodal_inputs_to_model_args(model_kwargs, scheduler_output)
 
     def _maybe_compute_attn_prefix(
         self,