Skip to content

Commit 37deebf

Browse files
jsimpherjsimpher
committed
first run through
1 parent aaae9c2 commit 37deebf

File tree

2 files changed

+1
-194
lines changed

2 files changed

+1
-194
lines changed

‎packages/datadog-plugin-openai/src/tracing.js

Lines changed: 0 additions & 145 deletions
Original file line numberDiff line numberDiff line change
@@ -78,27 +78,12 @@ class OpenAiTracingPlugin extends TracingPlugin {
7878
kind: 'client',
7979
meta: {
8080
[MEASURED]: 1,
81-
// Data that is always available with a request
82-
'openai.user.api_key': truncateApiKey(apiKey),
83-
'openai.api_base': basePath,
8481
// The openai.api_type (openai|azure) is present in Python but not in Node.js
8582
// Add support once https://github.com/openai/openai-node/issues/53 is closed
8683

8784
// Data that is common across many requests
88-
'openai.request.best_of': payload.best_of,
89-
'openai.request.echo': payload.echo,
90-
'openai.request.logprobs': payload.logprobs,
91-
'openai.request.max_tokens': payload.max_tokens,
9285
'openai.request.model': payload.model, // vague model
93-
'openai.request.n': payload.n,
94-
'openai.request.presence_penalty': payload.presence_penalty,
95-
'openai.request.frequency_penalty': payload.frequency_penalty,
96-
'openai.request.stop': payload.stop,
9786
'openai.request.suffix': payload.suffix,
98-
'openai.request.temperature': payload.temperature,
99-
'openai.request.top_p': payload.top_p,
100-
'openai.request.user': payload.user,
101-
'openai.request.file_id': payload.file_id // deleteFile, retrieveFile, downloadFile
10287
}
10388
}, false)
10489

@@ -110,34 +95,14 @@ class OpenAiTracingPlugin extends TracingPlugin {
11095
if (payload.prompt) {
11196
const prompt = payload.prompt
11297
openaiStore.prompt = prompt
113-
if (typeof prompt === 'string' || (Array.isArray(prompt) && typeof prompt[0] === 'number')) {
114-
// This is a single prompt, either String or [Number]
115-
tags['openai.request.prompt'] = normalizeStringOrTokenArray(prompt, true)
116-
} else if (Array.isArray(prompt)) {
117-
// This is multiple prompts, either [String] or [[Number]]
118-
for (let i = 0; i < prompt.length; i++) {
119-
tags[`openai.request.prompt.${i}`] = normalizeStringOrTokenArray(prompt[i], true)
120-
}
121-
}
12298
}
12399

124100
// createEdit, createEmbedding, createModeration
125101
if (payload.input) {
126102
const normalized = normalizeStringOrTokenArray(payload.input, false)
127-
tags['openai.request.input'] = normalize(normalized)
128103
openaiStore.input = normalized
129104
}
130105

131-
// createChatCompletion, createCompletion
132-
if (payload.logit_bias !== null && typeof payload.logit_bias === 'object') {
133-
for (const [tokenId, bias] of Object.entries(payload.logit_bias)) {
134-
tags[`openai.request.logit_bias.${tokenId}`] = bias
135-
}
136-
}
137-
138-
if (payload.stream) {
139-
tags['openai.request.stream'] = payload.stream
140-
}
141106

142107
switch (normalizedMethodName) {
143108
case 'createFineTune':
@@ -230,17 +195,8 @@ class OpenAiTracingPlugin extends TracingPlugin {
230195
'openai.request.endpoint': endpoint,
231196
'openai.request.method': method.toUpperCase(),
232197

233-
'openai.organization.id': body.organization_id, // only available in fine-tunes endpoints
234-
'openai.organization.name': headers['openai-organization'],
235-
236198
'openai.response.model': headers['openai-model'] || body.model, // specific model, often undefined
237-
'openai.response.id': body.id, // common creation value, numeric epoch
238199
'openai.response.deleted': body.deleted, // common boolean field in delete responses
239-
240-
// The OpenAI API appears to use both created and created_at in different places
241-
// Here we're conciously choosing to surface this inconsistency instead of normalizing
242-
'openai.response.created': body.created,
243-
'openai.response.created_at': body.created_at
244200
}
245201

246202
responseDataExtractionByMethod(normalizedMethodName, tags, body, openaiStore)
@@ -510,13 +466,6 @@ function createChatCompletionRequestExtraction (tags, payload, openaiStore) {
510466
if (!defensiveArrayLength(messages)) return
511467

512468
openaiStore.messages = payload.messages
513-
for (let i = 0; i < payload.messages.length; i++) {
514-
const message = payload.messages[i]
515-
tagChatCompletionRequestContent(message.content, i, tags)
516-
tags[`openai.request.messages.${i}.role`] = message.role
517-
tags[`openai.request.messages.${i}.name`] = message.name
518-
tags[`openai.request.messages.${i}.finish_reason`] = message.finish_reason
519-
}
520469
}
521470

522471
function commonCreateImageRequestExtraction (tags, payload, openaiStore) {
@@ -728,13 +677,6 @@ function createRetrieveFileResponseExtraction (tags, body) {
728677

729678
function createEmbeddingResponseExtraction (tags, body, openaiStore) {
730679
usageExtraction(tags, body, openaiStore)
731-
732-
if (!body.data) return
733-
734-
tags['openai.response.embeddings_count'] = body.data.length
735-
for (let i = 0; i < body.data.length; i++) {
736-
tags[`openai.response.embedding.${i}.embedding_length`] = body.data[i].embedding.length
737-
}
738680
}
739681

740682
function commonListCountResponseExtraction (tags, body) {
@@ -766,101 +708,14 @@ function commonCreateResponseExtraction (tags, body, openaiStore, methodName) {
766708

767709
if (!body.choices) return
768710

769-
tags['openai.response.choices_count'] = body.choices.length
770-
771711
openaiStore.choices = body.choices
772-
773-
for (let choiceIdx = 0; choiceIdx < body.choices.length; choiceIdx++) {
774-
const choice = body.choices[choiceIdx]
775-
776-
// logprobs can be null and we still want to tag it as 'returned' even when set to 'null'
777-
const specifiesLogProb = Object.keys(choice).includes('logprobs')
778-
779-
tags[`openai.response.choices.${choiceIdx}.finish_reason`] = choice.finish_reason
780-
tags[`openai.response.choices.${choiceIdx}.logprobs`] = specifiesLogProb ? 'returned' : undefined
781-
tags[`openai.response.choices.${choiceIdx}.text`] = normalize(choice.text)
782-
783-
// createChatCompletion only
784-
const message = choice.message || choice.delta // delta for streamed responses
785-
if (message) {
786-
tags[`openai.response.choices.${choiceIdx}.message.role`] = message.role
787-
tags[`openai.response.choices.${choiceIdx}.message.content`] = normalize(message.content)
788-
tags[`openai.response.choices.${choiceIdx}.message.name`] = normalize(message.name)
789-
if (message.tool_calls) {
790-
const toolCalls = message.tool_calls
791-
for (let toolIdx = 0; toolIdx < toolCalls.length; toolIdx++) {
792-
tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.function.name`] =
793-
toolCalls[toolIdx].function.name
794-
tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.function.arguments`] =
795-
toolCalls[toolIdx].function.arguments
796-
tags[`openai.response.choices.${choiceIdx}.message.tool_calls.${toolIdx}.id`] =
797-
toolCalls[toolIdx].id
798-
}
799-
}
800-
}
801-
}
802712
}
803713

804-
// createCompletion, createChatCompletion, createEdit, createEmbedding
805-
function usageExtraction (tags, body, methodName, openaiStore) {
806-
let promptTokens = 0
807-
let completionTokens = 0
808-
let totalTokens = 0
809-
if (body && body.usage) {
810-
promptTokens = body.usage.prompt_tokens
811-
completionTokens = body.usage.completion_tokens
812-
totalTokens = body.usage.total_tokens
813-
} else if (body.model && ['createChatCompletion', 'createCompletion'].includes(methodName)) {
814-
// estimate tokens based on method name for completions and chat completions
815-
const { model } = body
816-
817-
// prompt tokens
818-
const payload = openaiStore
819-
const promptTokensCount = countPromptTokens(methodName, payload, model)
820-
promptTokens = promptTokensCount.promptTokens
821-
const promptEstimated = promptTokensCount.promptEstimated
822-
823-
// completion tokens
824-
const completionTokensCount = countCompletionTokens(body, model)
825-
completionTokens = completionTokensCount.completionTokens
826-
const completionEstimated = completionTokensCount.completionEstimated
827-
828-
// total tokens
829-
totalTokens = promptTokens + completionTokens
830-
if (promptEstimated) tags['openai.response.usage.prompt_tokens_estimated'] = true
831-
if (completionEstimated) tags['openai.response.usage.completion_tokens_estimated'] = true
832-
}
833-
834-
if (promptTokens != null) tags['openai.response.usage.prompt_tokens'] = promptTokens
835-
if (completionTokens != null) tags['openai.response.usage.completion_tokens'] = completionTokens
836-
if (totalTokens != null) tags['openai.response.usage.total_tokens'] = totalTokens
837-
}
838714

839715
function truncateApiKey (apiKey) {
840716
return apiKey && `sk-...${apiKey.slice(-4)}`
841717
}
842718

843-
function tagChatCompletionRequestContent (contents, messageIdx, tags) {
844-
if (typeof contents === 'string') {
845-
tags[`openai.request.messages.${messageIdx}.content`] = normalize(contents)
846-
} else if (Array.isArray(contents)) {
847-
// content can also be an array of objects
848-
// which represent text input or image url
849-
for (const contentIdx in contents) {
850-
const content = contents[contentIdx]
851-
const type = content.type
852-
tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.type`] = content.type
853-
if (type === 'text') {
854-
tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.text`] = normalize(content.text)
855-
} else if (type === 'image_url') {
856-
tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.image_url.url`] =
857-
normalize(content.image_url.url)
858-
}
859-
// unsupported type otherwise, won't be tagged
860-
}
861-
}
862-
// unsupported type otherwise, won't be tagged
863-
}
864719

865720
// The server almost always responds with JSON
866721
function coerceResponseBody (body, methodName) {

‎packages/datadog-plugin-openai/test/index.spec.js

Lines changed: 1 addition & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -240,20 +240,8 @@ describe('Plugin', () => {
240240
expect(traces[0][0].meta).to.have.property('component', 'openai')
241241
expect(traces[0][0].meta).to.have.property('_dd.integration', 'openai')
242242
expect(traces[0][0].meta).to.have.property('openai.api_base', 'http://127.0.0.1:9126/vcr/openai')
243-
expect(traces[0][0].meta).to.have.property('openai.organization.name', 'datadog-staging')
244243
expect(traces[0][0].meta).to.have.property('openai.request.model', 'gpt-3.5-turbo-instruct')
245-
expect(traces[0][0].meta).to.have.property('openai.request.prompt', 'Hello, OpenAI!')
246-
expect(traces[0][0].meta).to.have.property('openai.response.choices.0.finish_reason')
247-
expect(traces[0][0].meta).to.have.property('openai.response.choices.0.logprobs')
248-
expect(traces[0][0].meta).to.have.property('openai.response.choices.0.text')
249244
expect(traces[0][0].meta).to.have.property('openai.response.model')
250-
expect(traces[0][0].meta).to.have.property('openai.user.api_key', 'sk-...ESTS')
251-
expect(traces[0][0].metrics).to.have.property('openai.request.max_tokens', 100)
252-
expect(traces[0][0].metrics).to.have.property('openai.request.temperature', 0.5)
253-
expect(traces[0][0].metrics).to.have.property('openai.request.n', 1)
254-
expect(traces[0][0].metrics).to.have.property('openai.response.usage.completion_tokens')
255-
expect(traces[0][0].metrics).to.have.property('openai.response.usage.prompt_tokens')
256-
expect(traces[0][0].metrics).to.have.property('openai.response.usage.total_tokens')
257245
})
258246

259247
const params = {
@@ -321,16 +309,7 @@ describe('Plugin', () => {
321309
it('makes a successful call', async () => {
322310
const checkTraces = agent
323311
.assertSomeTraces(traces => {
324-
expect(traces[0][0].meta).to.have.property(
325-
'openai.response.choices.0.text', '\\n\\nHello! How can I assist you?'
326-
)
327-
expect(traces[0][0].meta).to.have.property('openai.response.choices.0.finish_reason', 'stop')
328-
expect(traces[0][0].meta).to.have.property('openai.response.choices.0.logprobs')
329312
expect(traces[0][0].meta).to.have.property('openai.response.model')
330-
331-
expect(traces[0][0].metrics).to.have.property('openai.response.usage.completion_tokens')
332-
expect(traces[0][0].metrics).to.have.property('openai.response.usage.prompt_tokens')
333-
expect(traces[0][0].metrics).to.have.property('openai.response.usage.total_tokens')
334313
})
335314

336315
const params = {
@@ -356,19 +335,10 @@ describe('Plugin', () => {
356335
expect(metricStub).to.have.been.calledWith('openai.tokens.total')
357336
})
358337

359-
it('makes a successful call with usage included', async () => {
338+
it('makes a successful call', async () => {
360339
const checkTraces = agent
361340
.assertSomeTraces(traces => {
362-
expect(traces[0][0].meta).to.have.property(
363-
'openai.response.choices.0.text', '\\n\\nHello! How can I assist you?'
364-
)
365-
expect(traces[0][0].meta).to.have.property('openai.response.choices.0.finish_reason', 'stop')
366-
expect(traces[0][0].meta).to.have.property('openai.response.choices.0.logprobs')
367341
expect(traces[0][0].meta).to.have.property('openai.response.model')
368-
369-
expect(traces[0][0].metrics).to.have.property('openai.response.usage.completion_tokens')
370-
expect(traces[0][0].metrics).to.have.property('openai.response.usage.prompt_tokens')
371-
expect(traces[0][0].metrics).to.have.property('openai.response.usage.total_tokens')
372342
})
373343

374344
const params = {
@@ -456,8 +426,6 @@ describe('Plugin', () => {
456426
expect(traces[0][0].meta).to.have.property('openai.request.input', 'hello world')
457427
expect(traces[0][0].meta).to.have.property('openai.request.model', 'text-embedding-ada-002')
458428
expect(traces[0][0].meta).to.have.property('openai.response.model')
459-
expect(traces[0][0].metrics).to.have.property('openai.response.embeddings_count', 1)
460-
expect(traces[0][0].metrics).to.have.property('openai.response.embedding.0.embedding_length')
461429
})
462430

463431
const params = {
@@ -1304,30 +1272,14 @@ describe('Plugin', () => {
13041272
expect(traces[0][0]).to.have.property('resource', 'createChatCompletion')
13051273
}
13061274
expect(traces[0][0]).to.have.property('error', 0)
1307-
expect(traces[0][0].meta).to.have.property('openai.organization.name', 'datadog-staging')
13081275

13091276
expect(traces[0][0].meta).to.have.property('openai.request.method', 'POST')
13101277
expect(traces[0][0].meta).to.have.property(
13111278
'openai.request.endpoint', '/vcr/openai/chat/completions'
13121279
)
13131280

1314-
expect(traces[0][0].meta).to.have.property('openai.request.messages.0.content',
1315-
'You are a helpful assistant.')
1316-
expect(traces[0][0].meta).to.have.property('openai.request.messages.0.role', 'system')
1317-
expect(traces[0][0].meta).to.have.property('openai.request.messages.1.content', 'Hello, OpenAI!')
1318-
expect(traces[0][0].meta).to.have.property('openai.request.messages.1.role', 'user')
13191281
expect(traces[0][0].meta).to.have.property('openai.request.model', 'gpt-3.5-turbo')
1320-
expect(traces[0][0].meta).to.have.property('openai.request.user', 'dd-trace-test')
1321-
expect(traces[0][0].meta).to.have.property('openai.response.choices.0.finish_reason')
1322-
expect(traces[0][0].meta).to.have.property('openai.response.choices.0.message.content')
1323-
expect(traces[0][0].meta).to.have.property('openai.response.choices.0.message.role', 'assistant')
13241282
expect(traces[0][0].meta).to.have.property('openai.response.model')
1325-
expect(traces[0][0].metrics).to.have.property('openai.request.max_tokens', 100)
1326-
expect(traces[0][0].metrics).to.have.property('openai.request.n', 1)
1327-
expect(traces[0][0].metrics).to.have.property('openai.request.temperature', 0.5)
1328-
expect(traces[0][0].metrics).to.have.property('openai.response.usage.completion_tokens')
1329-
expect(traces[0][0].metrics).to.have.property('openai.response.usage.prompt_tokens')
1330-
expect(traces[0][0].metrics).to.have.property('openai.response.usage.total_tokens')
13311283
})
13321284

13331285
const params = {

0 commit comments

Comments
 (0)