+ converting2seq_cls_models.py

Signed-off-by: wang.yuqi <noooop@126.com>

Author: noooop

examples/offline_inference/converting2seq_cls_models.py

@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# refer to https://huggingface.co/Qwen/Qwen3-Reranker-0.6B/discussions/3
+
+import argparse
+import json
+
+import torch
+import transformers
+
+
+def from_2_way_softmax(causal_lm, seq_cls_model, tokenizer,
+                       classifier_from_tokens, device):
+    # for Qwen3-Reranker
+    assert len(classifier_from_tokens) == 2
+
+    lm_head_weights = causal_lm.lm_head.weight
+
+    a = tokenizer.convert_tokens_to_ids(classifier_from_tokens[0])
+    b = tokenizer.convert_tokens_to_ids(classifier_from_tokens[1])
+
+    score_weight = lm_head_weights[b].to(torch.float32).to(device).to(
+        torch.float32) - lm_head_weights[a].to(device)
+
+    with torch.no_grad():
+        seq_cls_model.score.weight.copy_(score_weight.unsqueeze(0))
+        if seq_cls_model.score.bias is not None:
+            seq_cls_model.score.bias.zero_()
+
+
+method_map = {function.__name__: function for function in [from_2_way_softmax]}
+
+
+def converting(model_name, classifier_from_tokens, path, method, device="cpu"):
+    assert method in method_map
+
+    tokenizer = transformers.AutoTokenizer.from_pretrained(model_name)
+    causal_lm = transformers.AutoModelForCausalLM.from_pretrained(
+        model_name, device_map=device)
+
+    seq_cls_model = transformers.AutoModelForSequenceClassification.from_pretrained(
+        model_name,
+        num_labels=1,
+        ignore_mismatched_sizes=True,
+        device_map=device)
+
+    method_map[method](causal_lm, seq_cls_model, tokenizer,
+                       classifier_from_tokens, device)
+
+    seq_cls_model.config.pad_token_id = tokenizer.pad_token_id
+
+    seq_cls_model.save_pretrained(path)
+    tokenizer.save_pretrained(path)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=
+        "Converting *ForCausalLM models to *ForSequenceClassification models.")
+    parser.add_argument("--model_name",
+                        type=str,
+                        default="Qwen/Qwen3-Reranker-0.6B",
+                        help="Model name")
+    parser.add_argument("--classifier_from_tokens",
+                        type=str,
+                        default='["no", "yes"]',
+                        help="classifier from tokens")
+    parser.add_argument("--method",
+                        type=str,
+                        default='from_2_way_softmax',
+                        help="Converting converting")
+    parser.add_argument("--path",
+                        type=str,
+                        default="./converted_model",
+                        help="Path to save converted model")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    converting(model_name=args.model_name,
+               classifier_from_tokens=json.loads(args.classifier_from_tokens),
+               method=args.method,
+               path=args.path)

examples/offline_inference/qwen3_reranker.py

@@ -19,6 +19,11 @@
 # concise, for example.
 # model = LLM(model="tomaarsen/Qwen3-Reranker-0.6B-seq-cls", task="score")
 
+# Offline conversion from official original version to sequence classification
+# model code please refer to: converting2seq_cls_models.py
+# The init parameters are as follows.
+# model = LLM(model="path_to/converted_model", task="score")
+
 # If you want to load the official original version, the init parameters are
 # as follows.
 

tests/models/language/pooling/test_gte.py

@@ -4,8 +4,9 @@
 
 import pytest
 
+from ...utils import RerankModelInfo
 from .embed_utils import EmbedModelInfo, correctness_test_embed_models
-from .mteb_utils import mteb_test_embed_models
+from .mteb_utils import mteb_test_embed_models, mteb_test_rerank_models
 
 MODELS = [
     ########## BertModel
@@ -56,6 +57,12 @@
                    enable_test=False),
 ]
 
+RERANK_MODELS = [
+    RerankModelInfo("Alibaba-NLP/gte-reranker-modernbert-base",
+                    architecture="ModernBertForSequenceClassification",
+                    enable_test=False),
+]
+
 
 @pytest.mark.parametrize("model_info", MODELS)
 def test_embed_models_mteb(hf_runner, vllm_runner,
@@ -80,3 +87,9 @@ def test_embed_models_correctness(hf_runner, vllm_runner,
 
     correctness_test_embed_models(hf_runner, vllm_runner, model_info,
                                   example_prompts, vllm_extra_kwargs)
+
+
+@pytest.mark.parametrize("model_info", RERANK_MODELS)
+def test_rerank_models_mteb(hf_runner, vllm_runner,
+                            model_info: RerankModelInfo) -> None:
+    mteb_test_rerank_models(hf_runner, vllm_runner, model_info)

vllm/model_executor/models/qwen3.py

@@ -41,9 +41,8 @@
 from vllm.model_executor.layers.quantization import QuantizationConfig
 from vllm.model_executor.layers.rotary_embedding import get_rope
 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
-from vllm.model_executor.pooling_metadata import PoolingMetadata
 from vllm.model_executor.sampling_metadata import SamplingMetadata
-from vllm.sequence import IntermediateTensors, PoolerOutput
+from vllm.sequence import IntermediateTensors
 
 from .adapters import as_seq_cls_model
 from .interfaces import SupportsLoRA, SupportsPP
@@ -349,32 +348,6 @@ def config_verify(self, vllm_config: "VllmConfig"):
         config.num_labels = 1
         self.vllm_config = vllm_config
 
-    def forward(
-        self,
-        input_ids: torch.Tensor,
-        positions: torch.Tensor,
-        intermediate_tensors: Optional[IntermediateTensors] = None,
-        inputs_embeds: Optional[torch.Tensor] = None,
-    ) -> torch.Tensor:
-        return self.model(input_ids=input_ids,
-                          positions=positions,
-                          inputs_embeds=inputs_embeds,
-                          intermediate_tensors=intermediate_tensors)
-
-    def pooler(
-        self,
-        hidden_states: torch.Tensor,
-        pooling_metadata: PoolingMetadata,
-    ) -> Optional[PoolerOutput]:
-        hidden_states = self._pooler.extract_states(hidden_states,
-                                                    pooling_metadata)
-        logits, _ = self.score(hidden_states)
-        pooled_data = self._pooler.head(logits, pooling_metadata)
-        pooled_outputs = [
-            self._pooler.build_output(data.squeeze(-1)) for data in pooled_data
-        ]
-        return PoolerOutput(outputs=pooled_outputs)
-
     def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
         is_original_qwen3_reranker = getattr(self.config,
                                              "is_original_qwen3_reranker",
@@ -419,5 +392,6 @@ def load_weights_from_original_qwen3_reranker(
         self.score.weight.data.copy_(weight)
 
         del self.lm_head
-        loaded_weights.add("classifier.weight")
+        loaded_weights.add("score.weight")
         loaded_weights.discard("lm_head.weight")
+        return loaded_weights

vllm/model_executor/models/registry.py

@@ -157,8 +157,6 @@
     "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"),  # noqa: E501
     "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
     "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),  # noqa: E501
-    # [Auto-converted (see adapters.py)]
-    "Qwen2ForSequenceClassification": ("qwen2", "Qwen2ForCausalLM"),
     # Technically PrithviGeoSpatialMAE is a model that works on images, both in
     # input and output. I am adding it here because it piggy-backs on embedding
     # models for the time being.
@@ -173,7 +171,10 @@
                                             "RobertaForSequenceClassification"),
     "ModernBertForSequenceClassification": ("modernbert",
                                             "ModernBertForSequenceClassification"),
+    # [Auto-converted (see adapters.py)]
+    "Qwen2ForSequenceClassification": ("qwen2", "Qwen2ForCausalLM"), # noqa: E501
     "Qwen3ForSequenceClassification": ("qwen3", "Qwen3ForSequenceClassification"), # noqa: E501
+    "GemmaForSequenceClassification": ("gemma", "GemmaForCausalLM"),
 }
 
 _MULTIMODAL_MODELS = {