Skip to content

Commit 62ad84f

Browse files
krrishdholakiakrrishdholakia
authored
UI (new_usage.tsx): Report 'total_tokens' + report success/failure calls (#9675)
* feat(internal_user_endpoints.py): return 'total_tokens' in `/user/daily/analytics` * test(test_internal_user_endpoints.py): add unit test to assert spend metrics and dailyspend metadata always report the same fields * build(schema.prisma): record success + failure calls to daily user table allows understanding why model requests might exceed provider requests (e.g. user hit rate limit error) * fix(internal_user_endpoints.py): report success / failure requests in API * fix(proxy/utils.py): default to success status can be missing or none at times for successful requests * feat(new_usage.tsx): show success/failure calls on UI * style(new_usage.tsx): ui cleanup * fix: fix linting error * fix: fix linting error * feat(litellm-proxy-extras/): add new migration files
1 parent f2a7eda commit 62ad84f

File tree

16 files changed

+240
-137
lines changed

16 files changed

+240
-137
lines changed

‎litellm-proxy-extras/dist/litellm_proxy_extras-0.1.2-py3-none-any.whl

8.19 KB
Binary file not shown.

‎litellm-proxy-extras/dist/litellm_proxy_extras-0.1.2.tar.gz

5.58 KB
Binary file not shown.

‎litellm-proxy-extras/litellm_proxy_extras/migrations/20250331215456_track_success_and_failed_requests_daily_agg_table/migration.sql

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
-- AlterTable
2+
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN "failed_requests" INTEGER NOT NULL DEFAULT 0,
3+
ADD COLUMN "successful_requests" INTEGER NOT NULL DEFAULT 0;
4+

‎litellm-proxy-extras/poetry.lock

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎litellm-proxy-extras/pyproject.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "litellm-proxy-extras"
3-
version = "0.1.1"
3+
version = "0.1.2"
44
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
55
authors = ["BerriAI"]
66
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
2222
build-backend = "poetry.core.masonry.api"
2323

2424
[tool.commitizen]
25-
version = "0.1.1"
25+
version = "0.1.2"
2626
version_files = [
2727
"pyproject.toml:version",
2828
"../requirements.txt:litellm-proxy-extras==",

‎litellm/proxy/_new_secret_config.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ litellm_settings:
3232
callbacks: ["prometheus"]
3333
# json_logs: true
3434

35-
# router_settings:
36-
# routing_strategy: usage-based-routing-v2 # 👈 KEY CHANGE
37-
# redis_host: os.environ/REDIS_HOST
38-
# redis_password: os.environ/REDIS_PASSWORD
39-
# redis_port: os.environ/REDIS_PORT
35+
router_settings:
36+
routing_strategy: usage-based-routing-v2 # 👈 KEY CHANGE
37+
redis_host: os.environ/REDIS_HOST
38+
redis_password: os.environ/REDIS_PASSWORD
39+
redis_port: os.environ/REDIS_PORT

‎litellm/proxy/_types.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2736,6 +2736,8 @@ class DailyUserSpendTransaction(TypedDict):
27362736
completion_tokens: int
27372737
spend: float
27382738
api_requests: int
2739+
successful_requests: int
2740+
failed_requests: int
27392741

27402742

27412743
class DBSpendUpdateTransactions(TypedDict):

‎litellm/proxy/management_endpoints/internal_user_endpoints.py

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1259,6 +1259,8 @@ class SpendMetrics(BaseModel):
12591259
prompt_tokens: int = Field(default=0)
12601260
completion_tokens: int = Field(default=0)
12611261
total_tokens: int = Field(default=0)
1262+
successful_requests: int = Field(default=0)
1263+
failed_requests: int = Field(default=0)
12621264
api_requests: int = Field(default=0)
12631265

12641266

@@ -1284,7 +1286,10 @@ class DailySpendMetadata(BaseModel):
12841286
total_spend: float = Field(default=0.0)
12851287
total_prompt_tokens: int = Field(default=0)
12861288
total_completion_tokens: int = Field(default=0)
1289+
total_tokens: int = Field(default=0)
12871290
total_api_requests: int = Field(default=0)
1291+
total_successful_requests: int = Field(default=0)
1292+
total_failed_requests: int = Field(default=0)
12881293
page: int = Field(default=1)
12891294
total_pages: int = Field(default=1)
12901295
has_more: bool = Field(default=False)
@@ -1307,6 +1312,8 @@ class LiteLLM_DailyUserSpend(BaseModel):
13071312
completion_tokens: int = 0
13081313
spend: float = 0.0
13091314
api_requests: int = 0
1315+
successful_requests: int = 0
1316+
failed_requests: int = 0
13101317

13111318

13121319
class GroupedData(TypedDict):
@@ -1322,6 +1329,8 @@ def update_metrics(
13221329
group_metrics.completion_tokens += record.completion_tokens
13231330
group_metrics.total_tokens += record.prompt_tokens + record.completion_tokens
13241331
group_metrics.api_requests += record.api_requests
1332+
group_metrics.successful_requests += record.successful_requests
1333+
group_metrics.failed_requests += record.failed_requests
13251334
return group_metrics
13261335

13271336

@@ -1443,14 +1452,18 @@ async def get_user_daily_activity(
14431452
take=page_size,
14441453
)
14451454

1455+
daily_spend_data_pydantic_list = [
1456+
LiteLLM_DailyUserSpend(**record.model_dump()) for record in daily_spend_data
1457+
]
1458+
14461459
# Process results
14471460
results = []
14481461
total_metrics = SpendMetrics()
14491462

14501463
# Group data by date and other dimensions
14511464

14521465
grouped_data: Dict[str, Dict[str, Any]] = {}
1453-
for record in daily_spend_data:
1466+
for record in daily_spend_data_pydantic_list:
14541467
date_str = record.date
14551468
if date_str not in grouped_data:
14561469
grouped_data[date_str] = {
@@ -1474,7 +1487,9 @@ async def get_user_daily_activity(
14741487
total_metrics.total_tokens += (
14751488
record.prompt_tokens + record.completion_tokens
14761489
)
1477-
total_metrics.api_requests += 1
1490+
total_metrics.api_requests += record.api_requests
1491+
total_metrics.successful_requests += record.successful_requests
1492+
total_metrics.failed_requests += record.failed_requests
14781493

14791494
# Convert grouped data to response format
14801495
for date_str, data in grouped_data.items():
@@ -1495,7 +1510,10 @@ async def get_user_daily_activity(
14951510
total_spend=total_metrics.spend,
14961511
total_prompt_tokens=total_metrics.prompt_tokens,
14971512
total_completion_tokens=total_metrics.completion_tokens,
1513+
total_tokens=total_metrics.total_tokens,
14981514
total_api_requests=total_metrics.api_requests,
1515+
total_successful_requests=total_metrics.successful_requests,
1516+
total_failed_requests=total_metrics.failed_requests,
14991517
page=page,
15001518
total_pages=-(-total_count // page_size), # Ceiling division
15011519
has_more=(page * page_size) < total_count,

‎litellm/proxy/schema.prisma

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,8 @@ model LiteLLM_DailyUserSpend {
327327
completion_tokens Int @default(0)
328328
spend Float @default(0.0)
329329
api_requests Int @default(0)
330+
successful_requests Int @default(0)
331+
failed_requests Int @default(0)
330332
created_at DateTime @default(now())
331333
updated_at DateTime @updatedAt
332334
@@ -352,4 +354,3 @@ enum JobStatus {
352354
INACTIVE
353355
}
354356

355-

‎litellm/proxy/utils.py

Lines changed: 72 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,25 @@
1010
from datetime import datetime, timedelta
1111
from email.mime.multipart import MIMEMultipart
1212
from email.mime.text import MIMEText
13-
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, overload
13+
from typing import (
14+
TYPE_CHECKING,
15+
Any,
16+
Dict,
17+
List,
18+
Literal,
19+
Optional,
20+
Union,
21+
cast,
22+
overload,
23+
)
1424

1525
from litellm.proxy._types import (
1626
DB_CONNECTION_ERROR_TYPES,
1727
CommonProxyErrors,
1828
DailyUserSpendTransaction,
1929
ProxyErrorTypes,
2030
ProxyException,
31+
SpendLogsMetadata,
2132
SpendLogsPayload,
2233
)
2334
from litellm.types.guardrails import GuardrailEventHooks
@@ -1145,6 +1156,41 @@ def __init__(
11451156
) # Client to connect to Prisma db
11461157
verbose_proxy_logger.debug("Success - Created Prisma Client")
11471158

1159+
def get_request_status(
1160+
self, payload: Union[dict, SpendLogsPayload]
1161+
) -> Literal["success", "failure"]:
1162+
"""
1163+
Determine if a request was successful or failed based on payload metadata.
1164+
1165+
Args:
1166+
payload (Union[dict, SpendLogsPayload]): Request payload containing metadata
1167+
1168+
Returns:
1169+
Literal["success", "failure"]: Request status
1170+
"""
1171+
try:
1172+
# Get metadata and convert to dict if it's a JSON string
1173+
payload_metadata: Union[Dict, SpendLogsMetadata, str] = payload.get(
1174+
"metadata", {}
1175+
)
1176+
if isinstance(payload_metadata, str):
1177+
payload_metadata_json: Union[Dict, SpendLogsMetadata] = cast(
1178+
Dict, json.loads(payload_metadata)
1179+
)
1180+
else:
1181+
payload_metadata_json = payload_metadata
1182+
1183+
# Check status in metadata dict
1184+
return (
1185+
"failure"
1186+
if payload_metadata_json.get("status") == "failure"
1187+
else "success"
1188+
)
1189+
1190+
except (json.JSONDecodeError, AttributeError):
1191+
# Default to success if metadata parsing fails
1192+
return "success"
1193+
11481194
def add_spend_log_transaction_to_daily_user_transaction(
11491195
self, payload: Union[dict, SpendLogsPayload]
11501196
):
@@ -1156,12 +1202,15 @@ def add_spend_log_transaction_to_daily_user_transaction(
11561202
If key exists, update the transaction with the new spend and usage
11571203
"""
11581204
expected_keys = ["user", "startTime", "api_key", "model", "custom_llm_provider"]
1205+
11591206
if not all(key in payload for key in expected_keys):
11601207
verbose_proxy_logger.debug(
11611208
f"Missing expected keys: {expected_keys}, in payload, skipping from daily_user_spend_transactions"
11621209
)
11631210
return
11641211

1212+
request_status = self.get_request_status(payload)
1213+
verbose_proxy_logger.info(f"Logged request status: {request_status}")
11651214
if isinstance(payload["startTime"], datetime):
11661215
start_time = payload["startTime"].isoformat()
11671216
date = start_time.split("T")[0]
@@ -1174,6 +1223,7 @@ def add_spend_log_transaction_to_daily_user_transaction(
11741223
return
11751224
try:
11761225
daily_transaction_key = f"{payload['user']}_{date}_{payload['api_key']}_{payload['model']}_{payload['custom_llm_provider']}"
1226+
11771227
if daily_transaction_key in self.daily_user_spend_transactions:
11781228
daily_transaction = self.daily_user_spend_transactions[
11791229
daily_transaction_key
@@ -1182,6 +1232,11 @@ def add_spend_log_transaction_to_daily_user_transaction(
11821232
daily_transaction["prompt_tokens"] += payload["prompt_tokens"]
11831233
daily_transaction["completion_tokens"] += payload["completion_tokens"]
11841234
daily_transaction["api_requests"] += 1
1235+
1236+
if request_status == "success":
1237+
daily_transaction["successful_requests"] += 1
1238+
else:
1239+
daily_transaction["failed_requests"] += 1
11851240
else:
11861241
daily_transaction = DailyUserSpendTransaction(
11871242
user_id=payload["user"],
@@ -1194,6 +1249,8 @@ def add_spend_log_transaction_to_daily_user_transaction(
11941249
completion_tokens=payload["completion_tokens"],
11951250
spend=payload["spend"],
11961251
api_requests=1,
1252+
successful_requests=1 if request_status == "success" else 0,
1253+
failed_requests=1 if request_status != "success" else 0,
11971254
)
11981255

11991256
self.daily_user_spend_transactions[
@@ -2603,6 +2660,12 @@ async def update_daily_user_spend(
26032660
],
26042661
"spend": transaction["spend"],
26052662
"api_requests": transaction["api_requests"],
2663+
"successful_requests": transaction[
2664+
"successful_requests"
2665+
],
2666+
"failed_requests": transaction[
2667+
"failed_requests"
2668+
],
26062669
},
26072670
"update": {
26082671
"prompt_tokens": {
@@ -2617,6 +2680,14 @@ async def update_daily_user_spend(
26172680
"api_requests": {
26182681
"increment": transaction["api_requests"]
26192682
},
2683+
"successful_requests": {
2684+
"increment": transaction[
2685+
"successful_requests"
2686+
]
2687+
},
2688+
"failed_requests": {
2689+
"increment": transaction["failed_requests"]
2690+
},
26202691
},
26212692
},
26222693
)

0 commit comments

Comments
 (0)