fix

Author: Your Name

vllm/beam/filtering.py

@@ -37,8 +37,7 @@ def __init__(self, classi_idx, classifier_names):
     async def get_n_valid_beams(self, create_completion: Callable,
                                 request: CompletionRequest,
                                 chunk_num: int,
-                                raw_request: Optional[Request] = None) -> list[
-        Union[AsyncGenerator[str, None], CompletionResponseChoice, ErrorResponse]]:
+                                raw_request: Optional[Request] = None) -> CompletionResponse | ErrorResponse:
         request.stream = False
         original_n = request.n
         request.n = request.n if request.n > 1 else _DEFAULT_BEAM_SIZE
@@ -70,10 +69,12 @@ async def get_n_valid_beams(self, create_completion: Callable,
 
         filtered_res = [r for r, valid in zip(res, beam_validator_res) if valid]
         logger.debug("Filtered count: %d", len(filtered_res))
+
+        raw_res.choices = filtered_res
         if len(filtered_res) == 0:
-            return res
+            return raw_res
 
-        return filtered_res
+        return raw_res
 
     def validate(self, responses: list[CompletionResponseChoice | ErrorResponse],
                  debug_infos_G: list[BeamDebugInfo] = None):

vllm/entrypoints/openai/serving_completion.py

@@ -123,14 +123,15 @@ async def _chunk_generator():
                     yield f"data: {beams.model_dump_json()}\n\n"
                     break
 
-                final = await self.beam_scorer.pick_best_beam(beams)
+                final = await self.beam_scorer.pick_best_beam(beams.choices)
                 request.prompt = final.text
                 should_stop = await _should_stop(final)
                 final.text = final.text[input_str_len:]
                 output = final.text
+                beams.choices = [final]
                 if self.request_logger:
                     logger.info(f"yielding chunk {num_chunks} text: {final.text}")
-                yield f"data: {final.model_dump_json()}\n\n"
+                yield f"data: {beams.model_dump_json()}\n\n"
 
                 if should_stop:
                     break