From 265c19754bfb5fcf78d2a926395d85e523111ca7 Mon Sep 17 00:00:00 2001 From: Celina Hanouti Date: Tue, 1 Jul 2025 14:35:47 +0200 Subject: [PATCH 1/5] fix structured output for sambanova --- packages/inference/src/providers/sambanova.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/inference/src/providers/sambanova.ts b/packages/inference/src/providers/sambanova.ts index d198ff53c6..847ab74d12 100644 --- a/packages/inference/src/providers/sambanova.ts +++ b/packages/inference/src/providers/sambanova.ts @@ -24,6 +24,30 @@ export class SambanovaConversationalTask extends BaseConversationalTask { constructor() { super("sambanova", "https://api.sambanova.ai"); } + + override preparePayload(params: BodyParams): Record { + const payload = super.preparePayload(params) as Record; + + const responseFormat = (params.args as Record)["response_format"]; + if ( + responseFormat && + typeof responseFormat === "object" && + "type" in responseFormat && + (responseFormat as { type?: unknown }).type === "json_schema" + ) { + const jsonSchemaConfig = (responseFormat as Record)["json_schema"] as + | Record + | undefined; + if (jsonSchemaConfig && typeof jsonSchemaConfig === "object") { + const strict = jsonSchemaConfig["strict"]; + if (strict === true || strict === undefined) { + jsonSchemaConfig["strict"] = false; + } + } + } + + return payload; + } } export class SambanovaFeatureExtractionTask extends TaskProviderHelper implements FeatureExtractionTaskHelper { From db94e92bd778ec96da6a7932d78ebf59fe189925 Mon Sep 17 00:00:00 2001 From: Celina Hanouti Date: Tue, 1 Jul 2025 14:42:50 +0200 Subject: [PATCH 2/5] fix structured output for nebius --- packages/inference/src/providers/nebius.ts | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/inference/src/providers/nebius.ts b/packages/inference/src/providers/nebius.ts index a5b31c6cf6..04ff59f0db 100644 --- a/packages/inference/src/providers/nebius.ts +++ b/packages/inference/src/providers/nebius.ts @@ -44,6 +44,29 @@ export class NebiusConversationalTask extends BaseConversationalTask { constructor() { super("nebius", NEBIUS_API_BASE_URL); } + + override preparePayload(params: BodyParams): Record { + const payload = super.preparePayload(params) as Record; + + const responseFormat = (params.args as Record)["response_format"]; + if ( + responseFormat && + typeof responseFormat === "object" && + "type" in responseFormat && + (responseFormat as { type?: unknown }).type === "json_schema" + ) { + const jsonSchemaDetails = (responseFormat as Record)["json_schema"]; + if ( + jsonSchemaDetails && + typeof jsonSchemaDetails === "object" && + "schema" in (jsonSchemaDetails as Record) + ) { + payload["guided_json"] = (jsonSchemaDetails as Record)["schema"]; + } + } + + return payload; + } } export class NebiusTextGenerationTask extends BaseTextGenerationTask { From bf05b9b259101a4b38af30c9c14997633bbb6053 Mon Sep 17 00:00:00 2001 From: Celina Hanouti Date: Tue, 1 Jul 2025 19:44:14 +0200 Subject: [PATCH 3/5] fix together structured output --- packages/inference/src/providers/together.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/inference/src/providers/together.ts b/packages/inference/src/providers/together.ts index 6777fdb169..bb395731ac 100644 --- a/packages/inference/src/providers/together.ts +++ b/packages/inference/src/providers/together.ts @@ -47,6 +47,22 @@ export class TogetherConversationalTask extends BaseConversationalTask { constructor() { super("together", TOGETHER_API_BASE_URL); } + + override preparePayload(params: BodyParams): Record { + const payload = super.preparePayload(params); + const response_format = payload.response_format as + | { type: "json_schema"; json_schema: { schema: unknown } } + | undefined; + + if (response_format?.type === "json_schema" && response_format?.json_schema?.schema) { + payload.response_format = { + type: "json_object", + schema: response_format.json_schema.schema, + }; + } + + return payload; + } } export class TogetherTextGenerationTask extends BaseTextGenerationTask { From e86c4993e799abcd2bbe2c450764292c371ea5c5 Mon Sep 17 00:00:00 2001 From: Celina Hanouti Date: Wed, 2 Jul 2025 13:06:11 +0200 Subject: [PATCH 4/5] fix --- packages/inference/src/providers/together.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/inference/src/providers/together.ts b/packages/inference/src/providers/together.ts index bb395731ac..1974885098 100644 --- a/packages/inference/src/providers/together.ts +++ b/packages/inference/src/providers/together.ts @@ -56,7 +56,7 @@ export class TogetherConversationalTask extends BaseConversationalTask { if (response_format?.type === "json_schema" && response_format?.json_schema?.schema) { payload.response_format = { - type: "json_object", + type: "json_schema", schema: response_format.json_schema.schema, }; } From fb164ba90158a464c4ea41d662e396ecf23dbc77 Mon Sep 17 00:00:00 2001 From: SBrandeis Date: Thu, 3 Jul 2025 11:33:07 +0200 Subject: [PATCH 5/5] use the type system a bit more --- packages/inference/src/providers/nebius.ts | 21 ++++------------ packages/inference/src/providers/sambanova.ts | 24 ++++++------------- packages/inference/src/providers/together.ts | 3 ++- 3 files changed, 14 insertions(+), 34 deletions(-) diff --git a/packages/inference/src/providers/nebius.ts b/packages/inference/src/providers/nebius.ts index 04ff59f0db..93376f9170 100644 --- a/packages/inference/src/providers/nebius.ts +++ b/packages/inference/src/providers/nebius.ts @@ -25,6 +25,7 @@ import { type TextToImageTaskHelper, } from "./providerHelper.js"; import { InferenceClientProviderOutputError } from "../errors.js"; +import type { ChatCompletionInput } from "../../../tasks/dist/commonjs/index.js"; const NEBIUS_API_BASE_URL = "https://api.studio.nebius.ai"; @@ -45,24 +46,12 @@ export class NebiusConversationalTask extends BaseConversationalTask { super("nebius", NEBIUS_API_BASE_URL); } - override preparePayload(params: BodyParams): Record { + override preparePayload(params: BodyParams): Record { const payload = super.preparePayload(params) as Record; - const responseFormat = (params.args as Record)["response_format"]; - if ( - responseFormat && - typeof responseFormat === "object" && - "type" in responseFormat && - (responseFormat as { type?: unknown }).type === "json_schema" - ) { - const jsonSchemaDetails = (responseFormat as Record)["json_schema"]; - if ( - jsonSchemaDetails && - typeof jsonSchemaDetails === "object" && - "schema" in (jsonSchemaDetails as Record) - ) { - payload["guided_json"] = (jsonSchemaDetails as Record)["schema"]; - } + const responseFormat = params.args.response_format; + if (responseFormat?.type === "json_schema" && responseFormat.json_schema?.schema) { + payload["guided_json"] = responseFormat.json_schema.schema; } return payload; diff --git a/packages/inference/src/providers/sambanova.ts b/packages/inference/src/providers/sambanova.ts index 847ab74d12..7c66f81733 100644 --- a/packages/inference/src/providers/sambanova.ts +++ b/packages/inference/src/providers/sambanova.ts @@ -19,32 +19,22 @@ import type { BodyParams } from "../types.js"; import type { FeatureExtractionTaskHelper } from "./providerHelper.js"; import { BaseConversationalTask, TaskProviderHelper } from "./providerHelper.js"; import { InferenceClientProviderOutputError } from "../errors.js"; +import type { ChatCompletionInput } from "../../../tasks/dist/commonjs/index.js"; export class SambanovaConversationalTask extends BaseConversationalTask { constructor() { super("sambanova", "https://api.sambanova.ai"); } - override preparePayload(params: BodyParams): Record { - const payload = super.preparePayload(params) as Record; + override preparePayload(params: BodyParams): Record { + const responseFormat = params.args.response_format; - const responseFormat = (params.args as Record)["response_format"]; - if ( - responseFormat && - typeof responseFormat === "object" && - "type" in responseFormat && - (responseFormat as { type?: unknown }).type === "json_schema" - ) { - const jsonSchemaConfig = (responseFormat as Record)["json_schema"] as - | Record - | undefined; - if (jsonSchemaConfig && typeof jsonSchemaConfig === "object") { - const strict = jsonSchemaConfig["strict"]; - if (strict === true || strict === undefined) { - jsonSchemaConfig["strict"] = false; - } + if (responseFormat?.type === "json_schema" && responseFormat.json_schema) { + if (responseFormat.json_schema.strict ?? true) { + responseFormat.json_schema.strict = false; } } + const payload = super.preparePayload(params) as Record; return payload; } diff --git a/packages/inference/src/providers/together.ts b/packages/inference/src/providers/together.ts index 1974885098..bec666186c 100644 --- a/packages/inference/src/providers/together.ts +++ b/packages/inference/src/providers/together.ts @@ -24,6 +24,7 @@ import { type TextToImageTaskHelper, } from "./providerHelper.js"; import { InferenceClientProviderOutputError } from "../errors.js"; +import type { ChatCompletionInput } from "../../../tasks/dist/commonjs/index.js"; const TOGETHER_API_BASE_URL = "https://api.together.xyz"; @@ -48,7 +49,7 @@ export class TogetherConversationalTask extends BaseConversationalTask { super("together", TOGETHER_API_BASE_URL); } - override preparePayload(params: BodyParams): Record { + override preparePayload(params: BodyParams): Record { const payload = super.preparePayload(params); const response_format = payload.response_format as | { type: "json_schema"; json_schema: { schema: unknown } }