Skip to content

Commit 9c13f73

Browse files
author
Your Name
committed
filtering
1 parent 40abaa7 commit 9c13f73

File tree

2 files changed

+3
-1
lines changed

2 files changed

+3
-1
lines changed

vllm/beam/filtering.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ async def get_n_valid_beams(self, create_completion: Callable, request: Completi
4747
request.n = n
4848
beam_validator_res = self.validate(res)
4949
filtered_res = [r for r, valid in zip(res, beam_validator_res) if valid]
50+
print('everything is filtered', len(filtered_res) == 0)
5051
if len(filtered_res) == 0:
5152
return res
5253

vllm/entrypoints/openai/serving_completion.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ async def _chunk_generator():
114114
final = await self.beam_scorer.collapse_beams(beams, num_chunks)
115115
request.prompt = final.choices[0].text
116116
eom = final.choices[0].finish_reason == "stop"
117+
final.choices[0].text = final.choices[0].text[input_str_len:]
117118
yield f"data: {final.model_dump_json()}\n\n"
118119

119120
if eom:
@@ -122,7 +123,7 @@ async def _chunk_generator():
122123
# Final chunk with trimmed text
123124
if final:
124125
final.choices[0].text = final.choices[0].text[input_str_len:]
125-
yield f"data: {final.choices[0].text}\n\n"
126+
yield f"data: {final.model_dump_json()}\n\n"
126127

127128
yield "data: [DONE]\n\n"
128129

0 commit comments

Comments
 (0)