Skip to content

Commit 1435c14

Browse files
authored
Fix misleading error message while streaming (#1591)
In `innerRequest`/`innerStreamingRequest`, if an error occur we raise a `InferenceClientProviderApiError` with a HTTP Request object in it. To avoid bloating the logs, a `requestArgsToJson` helper is applied to clean-up the body. However we do not pass the body to its but the `requestArgs` instead i.e. **before** provider-specific logic has been applied. This is can be really misleading since the body shown in the error is not necessarily the one sent in the request, making it hard to investigate. I also took the opportunity to make sure `accessToken` is never printed in the logs (it is currently the case as part of `requestArgs`). (also added a console.log for auto-provider)
1 parent 7763879 commit 1435c14

File tree

2 files changed

+27
-20
lines changed

2 files changed

+27
-20
lines changed

packages/inference/src/lib/getInferenceProviderMapping.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ export async function resolveProvider(
160160
}
161161
const mappings = await fetchInferenceProviderMappingForModel(modelId);
162162
provider = mappings[0]?.provider as InferenceProvider | undefined;
163+
console.log("Auto selected provider:", provider);
163164
}
164165
if (!provider) {
165166
throw new InferenceClientInputError(`No Inference Provider available for model ${modelId}.`);

packages/inference/src/utils/request.ts

Lines changed: 26 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,21 @@ export interface ResponseWrapper<T> {
1414
};
1515
}
1616

17-
function requestArgsToJson(args: RequestArgs): JsonObject {
18-
// Convert the entire args object to a JSON-serializable format
19-
const argsWithData = args as RequestArgs & { data?: Blob | ArrayBuffer };
20-
return JSON.parse(
21-
JSON.stringify({
22-
...argsWithData,
23-
data: argsWithData.data ? "[Blob or ArrayBuffer]" : null,
24-
})
25-
) as JsonObject;
17+
function bodyToJson(body?: BodyInit | null): JsonObject {
18+
let data = null;
19+
if (body instanceof Blob || body instanceof ArrayBuffer) {
20+
data = "[Blob or ArrayBuffer]";
21+
} else if (typeof body === "string") {
22+
try {
23+
data = JSON.parse(body);
24+
} catch {
25+
data = body;
26+
}
27+
}
28+
if (data.accessToken) {
29+
data.accessToken = "[REDACTED]";
30+
}
31+
return data as JsonObject;
2632
}
2733

2834
/**
@@ -60,7 +66,7 @@ export async function innerRequest<T>(
6066
url,
6167
method: info.method ?? "GET",
6268
headers: info.headers as Record<string, string>,
63-
body: requestArgsToJson(args),
69+
body: bodyToJson(info.body),
6470
},
6571
{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
6672
);
@@ -72,7 +78,7 @@ export async function innerRequest<T>(
7278
url,
7379
method: info.method ?? "GET",
7480
headers: info.headers as Record<string, string>,
75-
body: requestArgsToJson(args),
81+
body: bodyToJson(info.body),
7682
},
7783
{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
7884
);
@@ -83,7 +89,7 @@ export async function innerRequest<T>(
8389
url,
8490
method: info.method ?? "GET",
8591
headers: info.headers as Record<string, string>,
86-
body: requestArgsToJson(args),
92+
body: bodyToJson(info.body),
8793
},
8894
{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
8995
);
@@ -96,7 +102,7 @@ export async function innerRequest<T>(
96102
url,
97103
method: info.method ?? "GET",
98104
headers: info.headers as Record<string, string>,
99-
body: requestArgsToJson(args),
105+
body: bodyToJson(info.body),
100106
},
101107
{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: message ?? "" }
102108
);
@@ -142,7 +148,7 @@ export async function* innerStreamingRequest<T>(
142148
url,
143149
method: info.method ?? "GET",
144150
headers: info.headers as Record<string, string>,
145-
body: requestArgsToJson(args),
151+
body: bodyToJson(info.body),
146152
},
147153
{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
148154
);
@@ -154,7 +160,7 @@ export async function* innerStreamingRequest<T>(
154160
url,
155161
method: info.method ?? "GET",
156162
headers: info.headers as Record<string, string>,
157-
body: requestArgsToJson(args),
163+
body: bodyToJson(info.body),
158164
},
159165
{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
160166
);
@@ -167,7 +173,7 @@ export async function* innerStreamingRequest<T>(
167173
url,
168174
method: info.method ?? "GET",
169175
headers: info.headers as Record<string, string>,
170-
body: requestArgsToJson(args),
176+
body: bodyToJson(info.body),
171177
},
172178
{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
173179
);
@@ -180,7 +186,7 @@ export async function* innerStreamingRequest<T>(
180186
url,
181187
method: info.method ?? "GET",
182188
headers: info.headers as Record<string, string>,
183-
body: requestArgsToJson(args),
189+
body: bodyToJson(info.body),
184190
},
185191
{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: output }
186192
);
@@ -193,7 +199,7 @@ export async function* innerStreamingRequest<T>(
193199
url,
194200
method: info.method ?? "GET",
195201
headers: info.headers as Record<string, string>,
196-
body: requestArgsToJson(args),
202+
body: bodyToJson(info.body),
197203
},
198204
{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: "" }
199205
);
@@ -206,7 +212,7 @@ export async function* innerStreamingRequest<T>(
206212
url,
207213
method: info.method ?? "GET",
208214
headers: info.headers as Record<string, string>,
209-
body: requestArgsToJson(args),
215+
body: bodyToJson(info.body),
210216
},
211217
{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: "" }
212218
);
@@ -261,7 +267,7 @@ export async function* innerStreamingRequest<T>(
261267
url,
262268
method: info.method ?? "GET",
263269
headers: info.headers as Record<string, string>,
264-
body: requestArgsToJson(args),
270+
body: bodyToJson(info.body),
265271
},
266272
{ requestId: response.headers.get("x-request-id") ?? "", status: response.status, body: data }
267273
);

0 commit comments

Comments
 (0)