filtering

Author: Your Name

vllm/beam/filtering.py

@@ -47,6 +47,7 @@ async def get_n_valid_beams(self, create_completion: Callable, request: Completi
         request.n = n
         beam_validator_res = self.validate(res)
         filtered_res = [r for r, valid in zip(res, beam_validator_res) if valid]
+        print('everything is filtered', len(filtered_res) == 0)
         if len(filtered_res) == 0:
             return res
 

vllm/entrypoints/openai/serving_completion.py

@@ -114,6 +114,7 @@ async def _chunk_generator():
                 final = await self.beam_scorer.collapse_beams(beams, num_chunks)
                 request.prompt = final.choices[0].text
                 eom = final.choices[0].finish_reason == "stop"
+                final.choices[0].text = final.choices[0].text[input_str_len:]
                 yield f"data: {final.model_dump_json()}\n\n"
 
                 if eom:
@@ -122,7 +123,7 @@ async def _chunk_generator():
             # Final chunk with trimmed text
             if final:
                 final.choices[0].text = final.choices[0].text[input_str_len:]
-                yield f"data: {final.choices[0].text}\n\n"
+                yield f"data: {final.model_dump_json()}\n\n"
 
             yield "data: [DONE]\n\n"