Skip to content

Commit 8cdcdd7

Browse files
waldekmastykarzgarrytrinder
waldekmastykarz
and
garrytrinder
authored
Extends OpenAITelemetryPlugin with token report. Closes #1308 (#1318)
Co-authored-by: Garry Trinder <garry@trinder365.co.uk>
1 parent d65774c commit 8cdcdd7

File tree

2 files changed

+198
-1
lines changed

2 files changed

+198
-1
lines changed

‎DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
using OpenTelemetry.Metrics;
1616
using OpenTelemetry.Resources;
1717
using OpenTelemetry.Trace;
18+
using System.Collections.Concurrent;
1819
using System.Diagnostics;
1920
using System.Diagnostics.Metrics;
2021
using System.Text.Json;
@@ -46,7 +47,7 @@ public sealed class OpenAITelemetryPlugin(
4647
ISet<UrlToWatch> urlsToWatch,
4748
IProxyConfiguration proxyConfiguration,
4849
IConfigurationSection pluginConfigurationSection) :
49-
BasePlugin<OpenAITelemetryPluginConfiguration>(
50+
BaseReportingPlugin<OpenAITelemetryPluginConfiguration>(
5051
httpClient,
5152
logger,
5253
urlsToWatch,
@@ -65,6 +66,7 @@ public sealed class OpenAITelemetryPlugin(
6566
private LanguageModelPricesLoader? _loader;
6667
private MeterProvider? _meterProvider;
6768
private TracerProvider? _tracerProvider;
69+
private readonly ConcurrentDictionary<string, List<OpenAITelemetryPluginReportModelUsageInformation>> _modelUsage = [];
6870

6971
public override string Name => nameof(OpenAITelemetryPlugin);
7072

@@ -189,6 +191,26 @@ public override Task AfterResponseAsync(ProxyResponseArgs e, CancellationToken c
189191
return Task.CompletedTask;
190192
}
191193

194+
public override Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken cancellationToken)
195+
{
196+
Logger.LogTrace("{Method} called", nameof(AfterRecordingStopAsync));
197+
198+
var report = new OpenAITelemetryPluginReport
199+
{
200+
Application = Configuration.Application,
201+
Environment = Configuration.Environment,
202+
Currency = Configuration.Currency,
203+
IncludeCosts = Configuration.IncludeCosts,
204+
ModelUsage = _modelUsage.ToDictionary()
205+
};
206+
207+
StoreReport(report, e);
208+
_modelUsage.Clear();
209+
210+
Logger.LogTrace("Left {Name}", nameof(AfterRecordingStopAsync));
211+
return Task.CompletedTask;
212+
}
213+
192214
private void InitializeOpenTelemetryExporter()
193215
{
194216
Logger.LogTrace("InitializeOpenTelemetryExporter() called");
@@ -811,6 +833,15 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI
811833
.SetTag(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, usage.CompletionTokens)
812834
.SetTag(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS, usage.TotalTokens);
813835

836+
var reportModelUsageInformation = new OpenAITelemetryPluginReportModelUsageInformation
837+
{
838+
Model = response.Model,
839+
PromptTokens = usage.PromptTokens,
840+
CompletionTokens = usage.CompletionTokens
841+
};
842+
var usagePerModel = _modelUsage.GetOrAdd(response.Model, model => []);
843+
usagePerModel.Add(reportModelUsageInformation);
844+
814845
if (!Configuration.IncludeCosts || Configuration.Prices is null)
815846
{
816847
Logger.LogDebug("Cost tracking is disabled or prices data is not available");
@@ -847,6 +878,7 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI
847878
new(SemanticConvention.GEN_AI_REQUEST_MODEL, request.Model),
848879
new(SemanticConvention.GEN_AI_RESPONSE_MODEL, response.Model)
849880
]);
881+
reportModelUsageInformation.Cost = totalCost;
850882
}
851883
else
852884
{

‎DevProxy.Plugins/Inspection/OpenAITelemetryPluginReport.cs

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
using DevProxy.Abstractions.Plugins;
2+
using DevProxy.Abstractions.Utils;
3+
using System.Globalization;
4+
using System.Text;
5+
using System.Text.Json;
6+
using System.Text.Json.Serialization;
7+
8+
namespace DevProxy.Plugins.Inspection;
9+
10+
public class OpenAITelemetryPluginReportModelUsageInformation
11+
{
12+
public required long CompletionTokens { get; init; }
13+
public double Cost { get; set; }
14+
public required string Model { get; init; }
15+
public required long PromptTokens { get; init; }
16+
}
17+
18+
public class OpenAITelemetryPluginReport : IMarkdownReport, IPlainTextReport, IJsonReport
19+
{
20+
public required string Application { get; init; }
21+
public required string Currency { get; init; }
22+
public required string Environment { get; init; }
23+
[JsonIgnore]
24+
public bool IncludeCosts { get; set; }
25+
public required Dictionary<string, List<OpenAITelemetryPluginReportModelUsageInformation>> ModelUsage { get; init; } = [];
26+
27+
public string FileExtension => ".json";
28+
29+
public object ToJson() => JsonSerializer.Serialize(this, ProxyUtils.JsonSerializerOptions);
30+
31+
public string? ToMarkdown()
32+
{
33+
var totalTokens = 0L;
34+
var totalPromptTokens = 0L;
35+
var totalCompletionTokens = 0L;
36+
var totalCost = 0.0;
37+
var totalRequests = 0;
38+
39+
var sb = new StringBuilder();
40+
_ = sb
41+
.AppendLine(CultureInfo.InvariantCulture, $"# LLM usage report for {Application} in {Environment}")
42+
.AppendLine()
43+
.Append("Model|Requests|Prompt Tokens|Completion Tokens|Total Tokens");
44+
45+
if (IncludeCosts)
46+
{
47+
_ = sb.Append("|Total Cost");
48+
}
49+
50+
_ = sb
51+
.AppendLine()
52+
.Append(":----|-------:|------------:|----------------:|-----------:");
53+
54+
if (IncludeCosts)
55+
{
56+
_ = sb.Append("|---------:");
57+
}
58+
59+
_ = sb.AppendLine();
60+
61+
foreach (var modelUsage in ModelUsage.OrderBy(m => m.Key))
62+
{
63+
var promptTokens = modelUsage.Value.Sum(u => u.PromptTokens);
64+
var completionTokens = modelUsage.Value.Sum(u => u.CompletionTokens);
65+
var tokens = promptTokens + completionTokens;
66+
67+
totalPromptTokens += promptTokens;
68+
totalCompletionTokens += completionTokens;
69+
totalTokens += tokens;
70+
totalRequests += modelUsage.Value.Count;
71+
72+
_ = sb
73+
.Append(modelUsage.Key)
74+
.Append('|').Append(totalRequests)
75+
.Append('|').Append(promptTokens)
76+
.Append('|').Append(completionTokens)
77+
.Append('|').Append(tokens);
78+
79+
if (IncludeCosts)
80+
{
81+
var cost = modelUsage.Value.Sum(u => u.Cost);
82+
totalCost += cost;
83+
_ = sb.Append('|').Append(FormatCost(cost, Currency));
84+
}
85+
86+
_ = sb.AppendLine();
87+
}
88+
89+
_ = sb
90+
.Append("**Total**")
91+
.Append('|').Append(CultureInfo.CurrentCulture, $"**{totalRequests}**")
92+
.Append('|').Append(CultureInfo.CurrentCulture, $"**{totalPromptTokens}**")
93+
.Append('|').Append(CultureInfo.CurrentCulture, $"**{totalCompletionTokens}**")
94+
.Append('|').Append(CultureInfo.CurrentCulture, $"**{totalTokens}**");
95+
96+
if (IncludeCosts)
97+
{
98+
_ = sb.Append('|').Append(CultureInfo.CurrentCulture, $"**{FormatCost(totalCost, Currency)}**");
99+
}
100+
101+
_ = sb.AppendLine();
102+
103+
return sb.ToString();
104+
}
105+
106+
public string? ToPlainText()
107+
{
108+
var totalTokens = 0L;
109+
var totalPromptTokens = 0L;
110+
var totalCompletionTokens = 0L;
111+
var totalCost = 0.0;
112+
113+
var sb = new StringBuilder();
114+
_ = sb
115+
.AppendLine(CultureInfo.InvariantCulture, $"LLM USAGE REPORT FOR {Application} IN {Environment}")
116+
.AppendLine()
117+
.AppendLine("PER MODEL USAGE")
118+
.AppendLine();
119+
120+
foreach (var modelUsage in ModelUsage.OrderBy(m => m.Key))
121+
{
122+
var promptTokens = modelUsage.Value.Sum(u => u.PromptTokens);
123+
var completionTokens = modelUsage.Value.Sum(u => u.CompletionTokens);
124+
var tokens = promptTokens + completionTokens;
125+
126+
totalPromptTokens += promptTokens;
127+
totalCompletionTokens += completionTokens;
128+
totalTokens += tokens;
129+
130+
_ = sb.AppendLine(CultureInfo.InvariantCulture, $"MODEL: {modelUsage.Key}")
131+
.AppendLine()
132+
.AppendLine(CultureInfo.InvariantCulture, $"Requests: {modelUsage.Value.Count}")
133+
.AppendLine(CultureInfo.InvariantCulture, $"Prompt Tokens: {promptTokens}")
134+
.AppendLine(CultureInfo.InvariantCulture, $"Completion Tokens: {completionTokens}")
135+
.AppendLine(CultureInfo.InvariantCulture, $"Total Tokens: {tokens}");
136+
137+
if (IncludeCosts)
138+
{
139+
var cost = modelUsage.Value.Sum(u => u.Cost);
140+
totalCost += cost;
141+
_ = sb.AppendLine(CultureInfo.InvariantCulture, $"Total Cost: {FormatCost(cost, Currency)}");
142+
}
143+
144+
_ = sb.AppendLine();
145+
}
146+
147+
_ = sb
148+
.AppendLine("TOTALS")
149+
.AppendLine()
150+
.AppendLine(CultureInfo.InvariantCulture, $"Prompt Tokens: {totalPromptTokens}")
151+
.AppendLine(CultureInfo.InvariantCulture, $"Completion Tokens: {totalCompletionTokens}")
152+
.AppendLine(CultureInfo.InvariantCulture, $"Total Tokens: {totalTokens}");
153+
154+
if (IncludeCosts)
155+
{
156+
_ = sb.AppendLine(CultureInfo.InvariantCulture, $"Total Cost: {FormatCost(totalCost, Currency)}");
157+
}
158+
return sb.ToString();
159+
}
160+
161+
private static string FormatCost(double cost, string currency)
162+
{
163+
return $"{cost.ToString("#,##0.00########", CultureInfo.InvariantCulture)} {currency}";
164+
}
165+
}

0 commit comments

Comments
 (0)