Ignore EOS

anmarques · anmarques · commit 43f14d4febc4 · 2024-12-04T18:07:22.000Z
diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py
@@ -92,6 +92,7 @@ async def make_request(
                 {
                     "max_tokens": request.output_token_count,
                     "stop": None,
+                    "ignore_eos": True,
                 }
             )
         elif settings.openai.max_gen_tokens and settings.openai.max_gen_tokens > 0:

Original file line number	Diff line number	Diff line change
`@@ -92,6 +92,7 @@ async def make_request(`
`92`	`92`	`{`
`93`	`93`	`"max_tokens": request.output_token_count,`
`94`	`94`	`"stop": None,`
	`95`	`+ "ignore_eos": True,`
`95`	`96`	`}`
`96`	`97`	`)`
`97`	`98`	`elif settings.openai.max_gen_tokens and settings.openai.max_gen_tokens > 0:`