File tree Expand file tree Collapse file tree 1 file changed +13
-0
lines changed Expand file tree Collapse file tree 1 file changed +13
-0
lines changed Original file line number Diff line number Diff line change @@ -315,6 +315,9 @@ async def completion_stream_generator(
315
315
previous_num_tokens = [0 ] * num_choices * num_prompts
316
316
has_echoed = [False ] * num_choices * num_prompts
317
317
num_prompt_tokens = [0 ] * num_prompts
318
+ accumulated_text = ["" ] * num_choices * num_prompts
319
+ accumulated_tokens = [[] * num_choices * num_prompts ]
320
+ accumulated_logprobs = [[] * num_choices * num_prompts ]
318
321
319
322
stream_options = request .stream_options
320
323
if stream_options :
@@ -370,6 +373,16 @@ async def completion_stream_generator(
370
373
* (output .logprobs or []),
371
374
]
372
375
has_echoed [i ] = True
376
+ elif request .accumulate :
377
+ i = output .index + prompt_idx * num_choices
378
+ # return the accumulated response
379
+ accumulated_text [i ] += output .text
380
+ accumulated_tokens [i ].extend (output .token_ids )
381
+ accumulated_logprobs [i ].extend (output .logprobs or [])
382
+
383
+ delta_text = accumulated_text [i ]
384
+ delta_token_ids = accumulated_tokens [i ]
385
+ out_logprobs = accumulated_logprobs [i ]
373
386
else :
374
387
# return just the delta
375
388
delta_text = output .text
You can’t perform that action at this time.
0 commit comments