Skip to content

[Inference Providers] Fix structured output #1579

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jul 4, 2025
Merged

[Inference Providers] Fix structured output #1579

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
265c197
fix structured output for sambanova
hanouticelina Jul 1, 2025
db94e92
fix structured output for nebius
hanouticelina Jul 1, 2025
bf05b9b
fix together structured output
hanouticelina Jul 1, 2025
e86c499
fix
hanouticelina Jul 2, 2025
fb164ba
use the type system a bit more
SBrandeis Jul 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions packages/inference/src/providers/nebius.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
type TextToImageTaskHelper,
} from "./providerHelper.js";
import { InferenceClientProviderOutputError } from "../errors.js";
import type { ChatCompletionInput } from "../../../tasks/dist/commonjs/index.js";

const NEBIUS_API_BASE_URL = "https://api.studio.nebius.ai";

Expand All @@ -44,6 +45,17 @@ export class NebiusConversationalTask extends BaseConversationalTask {
constructor() {
super("nebius", NEBIUS_API_BASE_URL);
}

override preparePayload(params: BodyParams<ChatCompletionInput>): Record<string, unknown> {
const payload = super.preparePayload(params) as Record<string, unknown>;

const responseFormat = params.args.response_format;
if (responseFormat?.type === "json_schema" && responseFormat.json_schema?.schema) {
payload["guided_json"] = responseFormat.json_schema.schema;
}

return payload;
}
}

export class NebiusTextGenerationTask extends BaseTextGenerationTask {
Expand Down
14 changes: 14 additions & 0 deletions packages/inference/src/providers/sambanova.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,25 @@ import type { BodyParams } from "../types.js";
import type { FeatureExtractionTaskHelper } from "./providerHelper.js";
import { BaseConversationalTask, TaskProviderHelper } from "./providerHelper.js";
import { InferenceClientProviderOutputError } from "../errors.js";
import type { ChatCompletionInput } from "../../../tasks/dist/commonjs/index.js";

export class SambanovaConversationalTask extends BaseConversationalTask {
constructor() {
super("sambanova", "https://api.sambanova.ai");
}

override preparePayload(params: BodyParams<ChatCompletionInput>): Record<string, unknown> {
const responseFormat = params.args.response_format;

if (responseFormat?.type === "json_schema" && responseFormat.json_schema) {
if (responseFormat.json_schema.strict ?? true) {
responseFormat.json_schema.strict = false;
}
}
const payload = super.preparePayload(params) as Record<string, unknown>;

return payload;
}
}

export class SambanovaFeatureExtractionTask extends TaskProviderHelper implements FeatureExtractionTaskHelper {
Expand Down
17 changes: 17 additions & 0 deletions packages/inference/src/providers/together.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
type TextToImageTaskHelper,
} from "./providerHelper.js";
import { InferenceClientProviderOutputError } from "../errors.js";
import type { ChatCompletionInput } from "../../../tasks/dist/commonjs/index.js";

const TOGETHER_API_BASE_URL = "https://api.together.xyz";

Expand All @@ -47,6 +48,22 @@ export class TogetherConversationalTask extends BaseConversationalTask {
constructor() {
super("together", TOGETHER_API_BASE_URL);
}

override preparePayload(params: BodyParams<ChatCompletionInput>): Record<string, unknown> {
const payload = super.preparePayload(params);
const response_format = payload.response_format as
| { type: "json_schema"; json_schema: { schema: unknown } }
| undefined;

if (response_format?.type === "json_schema" && response_format?.json_schema?.schema) {
payload.response_format = {
type: "json_schema",
schema: response_format.json_schema.schema,
};
}

return payload;
}
}

export class TogetherTextGenerationTask extends BaseTextGenerationTask {
Expand Down