Ignore EOS

anmarques · anmarques · commit d9819e94e87c · 2024-12-04T18:47:05.000Z
diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py
@@ -92,7 +92,9 @@ async def make_request(
                 {
                     "max_tokens": request.output_token_count,
                     "stop": None,
-                    "ignore_eos": True,
+                    "extra_body": {
+                        "ignore_eos": True,
+                    }
                 }
             )
         elif settings.openai.max_gen_tokens and settings.openai.max_gen_tokens > 0:

Original file line number	Diff line number	Diff line change
`@@ -92,7 +92,9 @@ async def make_request(`
`92`	`92`	`{`
`93`	`93`	`"max_tokens": request.output_token_count,`
`94`	`94`	`"stop": None,`
`95`		`- "ignore_eos": True,`
	`95`	`+ "extra_body": {`
	`96`	`+ "ignore_eos": True,`
	`97`	`+ }`
`96`	`98`	`}`
`97`	`99`	`)`
`98`	`100`	`elif settings.openai.max_gen_tokens and settings.openai.max_gen_tokens > 0:`