[Bugfix]: Fix messy code when using logprobs (vllm-project#20910)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>

Author: chaunceyjiang

vllm/transformers_utils/detokenizer_utils.py

@@ -78,7 +78,6 @@ def convert_prompt_ids_to_tokens(
 def convert_ids_list_to_tokens(
     tokenizer: AnyTokenizer,
     token_ids: list[int],
-    skip_special_tokens: bool = False,
 ) -> list[str]:
     """Detokenize the input ids individually.
 
@@ -92,10 +91,8 @@ def convert_ids_list_to_tokens(
     """
     token_str_lst = []
     for token_id in token_ids:
-        token_str = tokenizer.decode(
-            [token_id],
-            skip_special_tokens=skip_special_tokens,
-        )
+        # use default skip_special_tokens.
+        token_str = tokenizer.decode([token_id])
         if token_str is None:
             token_str = ""
         token_str_lst.append(token_str)