diff --git a/.gitignore b/.gitignore index a30d642..c177bb3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +literalai-client-*.tgz + # Logs logs *.log diff --git a/package.json b/package.json index 9b71ed5..60507cd 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "types": "./dist/index.d.ts", "scripts": { "build": "tsup ./src", - "test": "jest --runInBand --watchAll=false", + "test": "jest --detectOpenHandles --runInBand --watchAll=false", "prepare": "husky install" }, "author": "Literal AI", diff --git a/src/api.ts b/src/api.ts index c9a68ce..13bed68 100644 --- a/src/api.ts +++ b/src/api.ts @@ -3,6 +3,7 @@ import FormData from 'form-data'; import { createReadStream } from 'fs'; import { v4 as uuidv4 } from 'uuid'; +import { LiteralClient } from '.'; import { GenerationsFilter, GenerationsOrderBy, @@ -327,6 +328,8 @@ function addGenerationsToDatasetQueryBuilder(generationIds: string[]) { } export class API { + /** @ignore */ + private client: LiteralClient; /** @ignore */ private apiKey: string; /** @ignore */ @@ -339,7 +342,13 @@ export class API { public disabled: boolean; /** @ignore */ - constructor(apiKey: string, url: string, disabled?: boolean) { + constructor( + client: LiteralClient, + apiKey: string, + url: string, + disabled?: boolean + ) { + this.client = client; this.apiKey = apiKey; this.url = url; this.graphqlEndpoint = `${url}/api/graphql`; @@ -509,7 +518,9 @@ export class API { const response = result.data.steps; - response.data = response.edges.map((x: any) => new Step(this, x.node)); + response.data = response.edges.map( + (x: any) => new Step(this.client, x.node, true) + ); delete response.edges; return response; @@ -541,7 +552,7 @@ export class API { return null; } - return new Step(this, result.data.step); + return new Step(this.client, result.data.step, true); } /** @@ -878,7 +889,7 @@ export class API { }; const response = await this.makeGqlCall(query, variables); - return new Thread(this, response.data.upsertThread); + return new Thread(this.client, response.data.upsertThread); } /** @@ -943,7 +954,9 @@ export class API { const response = result.data.threads; - response.data = response.edges.map((x: any) => new Thread(this, x.node)); + response.data = response.edges.map( + (x: any) => new Thread(this.client, x.node) + ); delete response.edges; return response; @@ -967,12 +980,11 @@ export class API { const variables = { id }; const response = await this.makeGqlCall(query, variables); - if (!response.data.threadDetail) { return null; } - return new Thread(this, response.data.threadDetail); + return new Thread(this.client, response.data.threadDetail); } /** diff --git a/src/index.ts b/src/index.ts index 4469813..9750dfa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,5 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; + import { API } from './api'; import instrumentation from './instrumentation'; import openai from './openai'; @@ -8,10 +10,18 @@ export * from './generation'; export type * from './instrumentation'; +type StoredContext = { + currentThread: Thread | null; + currentStep: Step | null; +}; + +const storage = new AsyncLocalStorage(); + export class LiteralClient { api: API; openai: ReturnType; instrumentation: ReturnType; + store: AsyncLocalStorage = storage; constructor(apiKey?: string, apiUrl?: string, disabled?: boolean) { if (!apiKey) { @@ -22,21 +32,54 @@ export class LiteralClient { apiUrl = process.env.LITERAL_API_URL || 'https://cloud.getliteral.ai'; } - this.api = new API(apiKey!, apiUrl!, disabled); + this.api = new API(this, apiKey!, apiUrl!, disabled); this.openai = openai(this); this.instrumentation = instrumentation(this); } thread(data?: ThreadConstructor) { - return new Thread(this.api, data); + return new Thread(this, data); } step(data: StepConstructor) { - return new Step(this.api, data); + return new Step(this, data); } run(data: Omit) { - const runData = { ...data, type: 'run' as const }; - return new Step(this.api, runData); + return this.step({ ...data, type: 'run' }); + } + + /** + * Gets the current thread from the context. + * WARNING : this will throw if run outside of a thread context. + * @returns The current thread, if any. + */ + getCurrentThread(): Thread { + const store = storage.getStore(); + + if (!store?.currentThread) { + throw new Error( + 'Literal AI SDK : tried to access current thread outside of a thread context.' + ); + } + + return store.currentThread; + } + + /** + * Gets the current step from the context. + * WARNING : this will throw if run outside of a step context. + * @returns The current step, if any. + */ + getCurrentStep(): Step { + const store = storage.getStore(); + + if (!store?.currentStep) { + throw new Error( + 'Literal AI SDK : tried to access current step outside of a context.' + ); + } + + return store.currentStep; } } diff --git a/src/types.ts b/src/types.ts index b22fabf..536f0d3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -5,6 +5,7 @@ import { } from 'openai/resources'; import { v4 as uuidv4 } from 'uuid'; +import { LiteralClient } from '.'; import { API } from './api'; import { Generation, GenerationType, IGenerationMessage } from './generation'; import { CustomChatPromptTemplate } from './instrumentation/langchain'; @@ -24,6 +25,15 @@ export type PaginatedResponse = { pageInfo: PageInfo; }; +function isPlainObject(value: unknown): value is Record { + if (typeof value !== 'object' || value === null) { + return false; + } + + const prototype = Object.getPrototypeOf(value); + return prototype === null || prototype === Object.prototype; +} + /** * Represents a utility class with serialization capabilities. */ @@ -37,7 +47,7 @@ export class Utils { serialize(): any { const dict: any = {}; Object.keys(this as any).forEach((key) => { - if (key === 'api') { + if (['api', 'client'].includes(key)) { return; } if ((this as any)[key] !== undefined) { @@ -135,20 +145,24 @@ export type ThreadConstructor = Omit & */ export class Thread extends ThreadFields { api: API; + client: LiteralClient; /** * Constructs a new Thread instance. * @param api - The API instance to interact with backend services. * @param data - Optional initial data for the thread, with an auto-generated ID if not provided. */ - constructor(api: API, data?: ThreadConstructor) { + constructor(client: LiteralClient, data?: ThreadConstructor) { super(); - this.api = api; + this.api = client.api; + this.client = client; + if (!data) { data = { id: uuidv4() }; } else if (!data.id) { data.id = uuidv4(); } + Object.assign(this, data); } @@ -158,12 +172,21 @@ export class Thread extends ThreadFields { * @returns A new Step instance linked to this thread. */ step(data: Omit) { - return new Step(this.api, { + return new Step(this.client, { ...data, threadId: this.id }); } + /** + * Creates a new Run step associated with this thread. + * @param data - The data for the new step, excluding the thread ID and the type + * @returns A new Step instance linked to this thread. + */ + run(data: Omit) { + return this.step({ ...data, type: 'run' }); + } + /** * Upserts the thread data to the backend, creating or updating as necessary. * @returns The updated Thread instance. @@ -182,6 +205,42 @@ export class Thread extends ThreadFields { }); return this; } + + /** + * Sends the thread to the API, handling disabled state and setting the end time if not already set. + * @param cb The callback function to run within the context of the thread. + * @param updateThread Optional update function to modify the thread after the callback. + * @returns The output of the wrapped callback function. + */ + async wrap( + cb: (thread: Thread) => Output | Promise, + updateThread?: + | ThreadConstructor + | ((output: Output) => ThreadConstructor) + | ((output: Output) => Promise) + ) { + const output = await this.client.store.run( + { currentThread: this, currentStep: null }, + () => cb(this) + ); + + if (updateThread) { + const updatedThread = + typeof updateThread === 'function' + ? await updateThread(output) + : updateThread; + + this.participantId = updatedThread.participantId ?? this.participantId; + this.environment = updatedThread.environment ?? this.environment; + this.name = updatedThread.name ?? this.name; + this.metadata = updatedThread.metadata ?? this.metadata; + this.tags = updatedThread.tags ?? this.tags; + } + + this.upsert().catch(console.error); + + return output; + } } export type StepType = @@ -222,15 +281,22 @@ export type StepConstructor = OmitUtils; */ export class Step extends StepFields { api: API; + client: LiteralClient; /** * Constructs a new Step instance. * @param api The API instance to be used for sending and managing steps. * @param data The initial data for the step, excluding utility properties. */ - constructor(api: API, data: StepConstructor) { + constructor( + client: LiteralClient, + data: StepConstructor, + ignoreContext?: true + ) { super(); - this.api = api; + this.api = client.api; + this.client = client; + Object.assign(this, data); // Automatically generate an ID if not provided. @@ -238,6 +304,20 @@ export class Step extends StepFields { this.id = uuidv4(); } + if (ignoreContext) { + return; + } + + // Automatically assign parent thread & step if there are any in the store. + const store = this.client.store.getStore(); + + if (store?.currentThread) { + this.threadId = store.currentThread.id; + } + if (store?.currentStep) { + this.parentId = store.currentStep.id; + } + // Set the creation and start time to the current time if not provided. if (!this.createdAt) { this.createdAt = new Date().toISOString(); @@ -284,7 +364,7 @@ export class Step extends StepFields { * @returns A new Step instance. */ step(data: Omit) { - return new Step(this.api, { + return new Step(this.client, { ...data, threadId: this.threadId, parentId: this.id @@ -306,6 +386,59 @@ export class Step extends StepFields { await this.api.sendSteps([this]); return this; } + + /** + * Sends the step to the API, handling disabled state and setting the end time if not already set. + * @param cb The callback function to run within the context of the step. + * @param updateStep Optional update function to modify the step after the callback. + * @returns The output of the wrapped callback function. + */ + async wrap( + cb: (step: Step) => Output | Promise, + updateStep?: + | Partial + | ((output: Output) => Partial) + | ((output: Output) => Promise>) + ) { + const startTime = new Date(); + this.startTime = startTime.toISOString(); + const currentStore = this.client.store.getStore(); + + const output = await this.client.store.run( + { currentThread: currentStore?.currentThread ?? null, currentStep: this }, + () => cb(this) + ); + + this.output = isPlainObject(output) ? output : { output }; + this.endTime = new Date().toISOString(); + + if (updateStep) { + const updatedStep = + typeof updateStep === 'function' + ? await updateStep(output) + : updateStep; + + this.name = updatedStep.name ?? this.name; + this.type = updatedStep.type ?? this.type; + this.threadId = updatedStep.threadId ?? this.threadId; + this.createdAt = updatedStep.createdAt ?? this.createdAt; + this.startTime = updatedStep.startTime ?? this.startTime; + this.error = updatedStep.error ?? this.error; + this.input = updatedStep.input ?? this.input; + this.output = updatedStep.output ?? this.output; + this.metadata = updatedStep.metadata ?? this.metadata; + this.tags = updatedStep.tags ?? this.tags; + this.parentId = updatedStep.parentId ?? this.parentId; + this.endTime = updatedStep.endTime ?? this.endTime; + this.generation = updatedStep.generation ?? this.generation; + this.scores = updatedStep.scores ?? this.scores; + this.attachments = updatedStep.attachments ?? this.attachments; + } + + this.send().catch(console.error); + + return output; + } } /** diff --git a/tests/integration/api.test.ts b/tests/api.test.ts similarity index 99% rename from tests/integration/api.test.ts rename to tests/api.test.ts index fc7f0ad..490573f 100644 --- a/tests/integration/api.test.ts +++ b/tests/api.test.ts @@ -8,7 +8,7 @@ import { Dataset, LiteralClient, Score -} from '../../src'; +} from '../src'; describe('End to end tests for the SDK', function () { let client: LiteralClient; @@ -339,9 +339,7 @@ describe('End to end tests for the SDK', function () { it('should test attachment', async function () { const thread = await client.thread({ id: uuidv4() }); // Upload an attachment - const fileStream = createReadStream( - './tests/integration/chainlit-logo.png' - ); + const fileStream = createReadStream('./tests/chainlit-logo.png'); const mime = 'image/png'; const { objectKey } = await client.api.uploadFile({ diff --git a/tests/async.test.ts b/tests/async.test.ts new file mode 100644 index 0000000..0c93cb4 --- /dev/null +++ b/tests/async.test.ts @@ -0,0 +1,12 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; + +const storage = new AsyncLocalStorage(); + +describe('Async Local Storage', () => { + it('is supported on this environment', () => { + storage.run('This is good', async () => { + const store = await storage.getStore(); + expect(store).toEqual('This is good'); + }); + }); +}); diff --git a/tests/integration/chainlit-logo.png b/tests/chainlit-logo.png similarity index 100% rename from tests/integration/chainlit-logo.png rename to tests/chainlit-logo.png diff --git a/tests/wrappers.test.ts b/tests/wrappers.test.ts new file mode 100644 index 0000000..35cbfc0 --- /dev/null +++ b/tests/wrappers.test.ts @@ -0,0 +1,388 @@ +import 'dotenv/config'; + +import { LiteralClient, Maybe, Step } from '../src'; + +const url = process.env.LITERAL_API_URL; +const apiKey = process.env.LITERAL_API_KEY; + +if (!url || !apiKey) { + throw new Error('Missing environment variables'); +} + +const client = new LiteralClient(apiKey, url); + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +describe('Wrapper', () => { + it('handles failing step', async () => { + let threadId: Maybe; + let stepId: Maybe; + + try { + await client.thread({ name: 'Test Wrappers Thread' }).wrap(async () => { + threadId = client.getCurrentThread()!.id; + + return client + .step({ name: 'Test Wrappers Step', type: 'assistant_message' }) + .wrap(async () => { + stepId = client.getCurrentStep()!.id; + + throw new Error('Something bad happened'); + }); + }); + } catch (error) { + expect((error as Error).message).toBe('Something bad happened'); + } + + const thread = await client.api.getThread(threadId!); + const step = await client.api.getStep(stepId!); + + expect(thread).toBeNull(); + expect(step).toBeNull(); + }); + + describe('Wrapping', () => { + it('handles simple use case', async () => { + let threadId: Maybe; + let stepId: Maybe; + + const result = await client + .thread({ name: 'Test Wrappers Thread' }) + .wrap(async () => { + threadId = client.getCurrentThread()!.id; + + return client + .step({ name: 'Test Wrappers Step', type: 'assistant_message' }) + .wrap(async () => { + stepId = client.getCurrentStep()!.id; + + return 'Paris is a city in Europe'; + }); + }); + + await sleep(1000); + const thread = await client.api.getThread(threadId!); + const step = await client.api.getStep(stepId!); + + expect(result).toBe('Paris is a city in Europe'); + + expect(thread!.name).toEqual('Test Wrappers Thread'); + + expect(step!.name).toEqual('Test Wrappers Step'); + expect(step!.threadId).toEqual(thread!.id); + expect(step!.parentId).toBeNull(); + expect(step!.output).toEqual({ output: 'Paris is a city in Europe' }); + }); + + it('handles nested steps', async () => { + let threadId: Maybe; + let runId: Maybe; + let retrieveStepId: Maybe; + let completionStepId: Maybe; + + const retrievedDocuments = [ + { score: 0.8, text: 'France is a country in Europe' }, + { score: 0.7, text: 'Paris is the capital of France' } + ]; + + const retrieve = async (_query: string) => + client.step({ name: 'Retrieve', type: 'retrieval' }).wrap(async () => { + retrieveStepId = client.getCurrentStep()!.id; + + return retrievedDocuments; + }); + + const completion = async (_query: string, _augmentations: string[]) => + client.step({ name: 'Completion', type: 'llm' }).wrap(async () => { + completionStepId = client.getCurrentStep()!.id; + + return { content: 'Paris is a city in Europe' }; + }); + + const query = 'France'; + + const result = await client + .thread({ name: 'Test Wrappers Thread' }) + .wrap(async () => { + threadId = client.getCurrentThread()!.id; + + return client.run({ name: 'Test Wrappers Run' }).wrap(async () => { + runId = client.getCurrentStep()!.id; + + const results = await retrieve(query); + const augmentations = results.map((result) => result.text); + const completionText = await completion(query, augmentations); + return completionText.content; + }); + }); + + await sleep(1000); + const thread = await client.api.getThread(threadId!); + const run = await client.api.getStep(runId!); + const retrieveStep = await client.api.getStep(retrieveStepId!); + const completionStep = await client.api.getStep(completionStepId!); + + expect(result).toBe('Paris is a city in Europe'); + + expect(run!.threadId).toEqual(thread!.id); + expect(run!.parentId).toBeNull(); + expect(run!.output).toEqual({ output: 'Paris is a city in Europe' }); + + expect(retrieveStep!.threadId).toEqual(thread!.id); + expect(retrieveStep!.parentId).toEqual(run!.id); + expect(retrieveStep!.output).toEqual({ output: retrievedDocuments }); + + expect(completionStep!.threadId).toEqual(thread!.id); + expect(completionStep!.parentId).toEqual(run!.id); + expect(completionStep!.output).toEqual({ + content: 'Paris is a city in Europe' + }); + }); + + it('handles steps outside of a thread', async () => { + let runId: Maybe; + let stepId: Maybe; + + const result = await client + .run({ name: 'Test Wrappers Run' }) + .wrap(async () => { + runId = client.getCurrentStep()!.id; + + return client + .step({ name: 'Test Wrappers Step', type: 'assistant_message' }) + .wrap(async () => { + stepId = client.getCurrentStep()!.id; + + return 'Paris is a city in Europe'; + }); + }); + + await sleep(1000); + const run = await client.api.getStep(runId!); + const step = await client.api.getStep(stepId!); + + expect(result).toBe('Paris is a city in Europe'); + + expect(run!.name).toEqual('Test Wrappers Run'); + + expect(step!.name).toEqual('Test Wrappers Step'); + expect(step!.threadId).toBeNull(); + expect(step!.parentId).toEqual(run!.id); + }); + + it("doesn't leak the current store when getting entities from the API", async () => { + let fetchedOldStep: Maybe; + + const oldStep = await client + .step({ name: 'Test Old Step', type: 'run' }) + .send(); + + await client.thread({ name: 'Test Wrappers Thread' }).wrap(async () => { + return client + .step({ name: 'Test Wrappers Step', type: 'assistant_message' }) + .wrap(async () => { + fetchedOldStep = await client.api.getStep(oldStep!.id!); + + return 'Paris is a city in Europe'; + }); + }); + + expect(fetchedOldStep!.parentId).toBeNull(); + }); + }); + + describe('Updating the thread / step after the wrap', () => { + it('updates the thread / step with a static object', async () => { + let threadId: Maybe; + let stepId: Maybe; + + await client.thread({ name: 'Test Wrappers Thread' }).wrap( + async () => { + threadId = client.getCurrentThread()!.id; + + return client + .step({ name: 'Test Wrappers Step', type: 'assistant_message' }) + .wrap( + async () => { + stepId = client.getCurrentStep()!.id; + client.getCurrentStep()!.name = 'Edited Test Wrappers Step'; + + return { content: 'Paris is a city in Europe' }; + }, + { metadata: { key: 'step-value' } } + ); + }, + { metadata: { key: 'thread-value' } } + ); + + await sleep(1000); + const thread = await client.api.getThread(threadId!); + const step = await client.api.getStep(stepId!); + + expect(thread!.metadata!.key).toEqual('thread-value'); + expect(step!.metadata!.key).toEqual('step-value'); + }); + + it('updates the thread / step based on the output of the wrap', async () => { + let threadId: Maybe; + let stepId: Maybe; + + await client.thread({ name: 'Test Wrappers Thread' }).wrap( + async () => { + threadId = client.getCurrentThread()!.id; + + return client + .step({ name: 'Test Wrappers Step', type: 'assistant_message' }) + .wrap( + async () => { + stepId = client.getCurrentStep()!.id; + client.getCurrentStep()!.name = 'Edited Test Wrappers Step'; + + return { content: 'Paris is a city in Europe' }; + }, + (output) => ({ + output: { type: 'assistant', message: output.content } + }) + ); + }, + (output) => ({ metadata: { assistantMessage: output.content } }) + ); + + await sleep(1000); + const thread = await client.api.getThread(threadId!); + const step = await client.api.getStep(stepId!); + + expect(thread!.metadata!.assistantMessage).toEqual( + 'Paris is a city in Europe' + ); + expect(step!.output!.type).toEqual('assistant'); + expect(step!.output!.message).toEqual('Paris is a city in Europe'); + }); + }); + + describe('Editing current thread / step', () => { + it('handles edition using the `getCurrentXXX` helpers', async () => { + let threadId: Maybe; + let stepId: Maybe; + + await client.thread({ name: 'Test Wrappers Thread' }).wrap(async () => { + threadId = client.getCurrentThread()!.id; + client.getCurrentThread()!.name = 'Edited Test Wrappers Thread'; + + return client + .step({ name: 'Test Wrappers Step', type: 'assistant_message' }) + .wrap(async () => { + stepId = client.getCurrentStep()!.id; + client.getCurrentStep()!.name = 'Edited Test Wrappers Step'; + + return 'Paris is a city in Europe'; + }); + }); + + await sleep(1000); + const thread = await client.api.getThread(threadId!); + const step = await client.api.getStep(stepId!); + + expect(thread!.name).toEqual('Edited Test Wrappers Thread'); + expect(step!.name).toEqual('Edited Test Wrappers Step'); + }); + + it('handles edition using the variable provided to the callback', async () => { + let threadId: Maybe; + let stepId: Maybe; + + await client + .thread({ name: 'Test Wrappers Thread' }) + .wrap(async (thread) => { + threadId = thread.id; + thread.name = 'Edited Test Wrappers Thread'; + + return client + .step({ name: 'Test Wrappers Step', type: 'assistant_message' }) + .wrap(async (step) => { + stepId = step.id; + step.name = 'Edited Test Wrappers Step'; + + return 'Paris is a city in Europe'; + }); + }); + + await sleep(1000); + const thread = await client.api.getThread(threadId!); + const step = await client.api.getStep(stepId!); + + expect(thread!.name).toEqual('Edited Test Wrappers Thread'); + expect(step!.name).toEqual('Edited Test Wrappers Step'); + }); + }); + + describe('Wrapping existing steps and threads', () => { + it('wraps an existing thread', async () => { + const { id: threadId } = await client + .thread({ name: 'Test Wrappers Thread' }) + .upsert(); + + await sleep(1000); + const thread = await client.api.getThread(threadId); + + const wrappedThreadId = await thread!.wrap(async () => { + return client.getCurrentThread()!.id; + }); + + expect(wrappedThreadId).toEqual(threadId); + }); + + it('wraps an existing step', async () => { + const { id: stepId } = await client + .run({ name: 'Test Wrappers Thread' }) + .send(); + + await sleep(1000); + const step = await client.api.getStep(stepId!); + + const wrappedStepId = await step!.wrap(async () => { + return client.getCurrentStep()!.id; + }); + + expect(wrappedStepId).toEqual(stepId); + }); + }); + + describe('Concurrency', () => { + it("doesn't mix up threads and steps", async () => { + let firstThreadId: Maybe; + let secondThreadId: Maybe; + let firstStep: Maybe; + let secondStep: Maybe; + + await Promise.all([ + client.thread({ name: 'Thread 1' }).wrap(async () => { + firstThreadId = client.getCurrentThread()!.id; + + return client + .step({ name: 'Step 1', type: 'assistant_message' }) + .wrap(async () => { + firstStep = client.getCurrentStep(); + return 'Paris is a city in Europe'; + }); + }), + client.thread({ name: 'Thread 2' }).wrap(async () => { + secondThreadId = client.getCurrentThread()!.id; + + return client + .step({ name: 'Step 2', type: 'assistant_message' }) + .wrap(async () => { + secondStep = client.getCurrentStep(); + return 'London is a city in Europe'; + }); + }) + ]); + + expect(firstThreadId).not.toEqual(secondThreadId); + expect(firstStep?.threadId).toEqual(firstThreadId); + expect(secondStep?.threadId).toEqual(secondThreadId); + }); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 928c264..90476a6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,14 +12,14 @@ }, "typedocOptions": { "plugin": "typedoc-plugin-markdown", - "entryPoints": ["./src/api.ts"], /*, "./src/index.ts" */ + "entryPoints": ["./src/api.ts"], "disableSources": true, "sort": "source-order", "readme": "none", "outputFileStrategy": "modules", "hideInPageTOC": true, "githubPages": false, - "excludePrivate": true, + "excludePrivate": true }, "exclude": ["eval-promptfoo/*"] }