From de963f8cfa65e2e7c43f16acea9d861407aa4130 Mon Sep 17 00:00:00 2001 From: Damien BUTY Date: Tue, 30 Jul 2024 11:17:51 +0200 Subject: [PATCH 1/9] refactor: reorganize code structure --- docs/README.md | 2933 +++++++++++++++++++++++++ docs/api.md | 942 -------- src/api.ts | 48 +- src/evaluation/dataset.ts | 237 ++ src/evaluation/experiment-run.ts | 61 + src/evaluation/score.ts | 29 + src/index.ts | 55 +- src/instrumentation/langchain.ts | 2 +- src/instrumentation/llamaindex.ts | 4 +- src/instrumentation/openai.ts | 4 +- src/instrumentation/vercel-sdk.ts | 6 +- src/observability/attachment.ts | 24 + src/{ => observability}/generation.ts | 2 +- src/observability/step.ts | 211 ++ src/observability/thread.ts | 134 ++ src/openai.ts | 5 +- src/prompt-engineering/prompt.ts | 141 ++ src/types.ts | 889 -------- src/utils.ts | 81 + tests/api.test.ts | 4 +- tests/attachments.test.ts | 3 +- tests/integration/openai.test.ts | 9 +- tests/wrappers.test.ts | 4 +- 23 files changed, 3948 insertions(+), 1880 deletions(-) create mode 100644 docs/README.md delete mode 100644 docs/api.md create mode 100644 src/evaluation/dataset.ts create mode 100644 src/evaluation/experiment-run.ts create mode 100644 src/evaluation/score.ts create mode 100644 src/observability/attachment.ts rename src/{ => observability}/generation.ts (97%) create mode 100644 src/observability/step.ts create mode 100644 src/observability/thread.ts create mode 100644 src/prompt-engineering/prompt.ts delete mode 100644 src/types.ts create mode 100644 src/utils.ts diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..e9e924b --- /dev/null +++ b/docs/README.md @@ -0,0 +1,2933 @@ +**@literalai/client** + +*** + +# @literalai/client + +## Classes + +### BaseGeneration + +Represents a utility class with serialization capabilities. + +#### Extends + +- [`Utils`](README.md#utils) + +#### Constructors + +##### new BaseGeneration() + +> **new BaseGeneration**(): [`BaseGeneration`](README.md#basegeneration) + +###### Returns + +[`BaseGeneration`](README.md#basegeneration) + +###### Inherited from + +[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) + +#### Properties + +##### promptId + +> **promptId**?: `string` + +##### provider + +> **provider**?: [`Maybe`](README.md#maybet)\<`string`\> + +##### model + +> **model**?: [`Maybe`](README.md#maybet)\<`string`\> + +##### id + +> **id**?: [`Maybe`](README.md#maybet)\<`string`\> + +##### tags + +> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> + +##### error + +> **error**?: [`Maybe`](README.md#maybet)\<`string`\> + +##### variables + +> **variables**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +##### settings + +> **settings**?: [`Maybe`](README.md#maybet)\<[`ILLMSettings`](README.md#illmsettings)\> + +##### tools + +> **tools**?: [`Maybe`](README.md#maybet)\<[`ITool`](README.md#itool)[]\> + +##### tokenCount + +> **tokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> + +##### inputTokenCount + +> **inputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> + +##### outputTokenCount + +> **outputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> + +##### ttFirstToken + +> **ttFirstToken**?: [`Maybe`](README.md#maybet)\<`number`\> + +##### tokenThroughputInSeconds + +> **tokenThroughputInSeconds**?: [`Maybe`](README.md#maybet)\<`number`\> + +##### duration + +> **duration**?: [`Maybe`](README.md#maybet)\<`number`\> + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. + +###### Returns + +`any` + +A dictionary representing the serialized properties of the object. + +###### Inherited from + +[`Utils`](README.md#utils).[`serialize`](README.md#serialize-3) + +*** + +### CompletionGeneration + +Represents a utility class with serialization capabilities. + +#### Extends + +- [`BaseGeneration`](README.md#basegeneration) + +#### Constructors + +##### new CompletionGeneration(data) + +> **new CompletionGeneration**(`data`): [`CompletionGeneration`](README.md#completiongeneration) + +###### Parameters + +▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`CompletionGeneration`](README.md#completiongeneration)\> + +###### Returns + +[`CompletionGeneration`](README.md#completiongeneration) + +###### Overrides + +[`BaseGeneration`](README.md#basegeneration).[`constructor`](README.md#constructors) + +#### Properties + +##### promptId + +> **promptId**?: `string` + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`promptId`](README.md#promptid) + +##### provider + +> **provider**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`provider`](README.md#provider) + +##### model + +> **model**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`model`](README.md#model) + +##### id + +> **id**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`id`](README.md#id) + +##### tags + +> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`tags`](README.md#tags) + +##### error + +> **error**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`error`](README.md#error) + +##### variables + +> **variables**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`variables`](README.md#variables) + +##### settings + +> **settings**?: [`Maybe`](README.md#maybet)\<[`ILLMSettings`](README.md#illmsettings)\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`settings`](README.md#settings) + +##### tools + +> **tools**?: [`Maybe`](README.md#maybet)\<[`ITool`](README.md#itool)[]\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`tools`](README.md#tools) + +##### tokenCount + +> **tokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`tokenCount`](README.md#tokencount) + +##### inputTokenCount + +> **inputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`inputTokenCount`](README.md#inputtokencount) + +##### outputTokenCount + +> **outputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`outputTokenCount`](README.md#outputtokencount) + +##### ttFirstToken + +> **ttFirstToken**?: [`Maybe`](README.md#maybet)\<`number`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`ttFirstToken`](README.md#ttfirsttoken) + +##### tokenThroughputInSeconds + +> **tokenThroughputInSeconds**?: [`Maybe`](README.md#maybet)\<`number`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`tokenThroughputInSeconds`](README.md#tokenthroughputinseconds) + +##### duration + +> **duration**?: [`Maybe`](README.md#maybet)\<`number`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`duration`](README.md#duration) + +##### type + +> **type**?: [`GenerationType`](README.md#generationtype) = `'COMPLETION'` + +##### prompt + +> **prompt**?: [`Maybe`](README.md#maybet)\<`string`\> + +##### completion + +> **completion**?: [`Maybe`](README.md#maybet)\<`string`\> + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. + +###### Returns + +`any` + +A dictionary representing the serialized properties of the object. + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`serialize`](README.md#serialize) + +*** + +### ChatGeneration + +Represents a utility class with serialization capabilities. + +#### Extends + +- [`BaseGeneration`](README.md#basegeneration) + +#### Constructors + +##### new ChatGeneration(data) + +> **new ChatGeneration**(`data`): [`ChatGeneration`](README.md#chatgeneration) + +###### Parameters + +▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`ChatGeneration`](README.md#chatgeneration)\> + +###### Returns + +[`ChatGeneration`](README.md#chatgeneration) + +###### Overrides + +[`BaseGeneration`](README.md#basegeneration).[`constructor`](README.md#constructors) + +#### Properties + +##### promptId + +> **promptId**?: `string` + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`promptId`](README.md#promptid) + +##### provider + +> **provider**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`provider`](README.md#provider) + +##### model + +> **model**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`model`](README.md#model) + +##### id + +> **id**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`id`](README.md#id) + +##### tags + +> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`tags`](README.md#tags) + +##### error + +> **error**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`error`](README.md#error) + +##### variables + +> **variables**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`variables`](README.md#variables) + +##### settings + +> **settings**?: [`Maybe`](README.md#maybet)\<[`ILLMSettings`](README.md#illmsettings)\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`settings`](README.md#settings) + +##### tools + +> **tools**?: [`Maybe`](README.md#maybet)\<[`ITool`](README.md#itool)[]\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`tools`](README.md#tools) + +##### tokenCount + +> **tokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`tokenCount`](README.md#tokencount) + +##### inputTokenCount + +> **inputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`inputTokenCount`](README.md#inputtokencount) + +##### outputTokenCount + +> **outputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`outputTokenCount`](README.md#outputtokencount) + +##### ttFirstToken + +> **ttFirstToken**?: [`Maybe`](README.md#maybet)\<`number`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`ttFirstToken`](README.md#ttfirsttoken) + +##### tokenThroughputInSeconds + +> **tokenThroughputInSeconds**?: [`Maybe`](README.md#maybet)\<`number`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`tokenThroughputInSeconds`](README.md#tokenthroughputinseconds) + +##### duration + +> **duration**?: [`Maybe`](README.md#maybet)\<`number`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`duration`](README.md#duration) + +##### type + +> **type**?: [`GenerationType`](README.md#generationtype) = `'CHAT'` + +##### messages + +> **messages**?: [`Maybe`](README.md#maybet)\<[`IGenerationMessage`](README.md#igenerationmessage)[]\> = `[]` + +##### messageCompletion + +> **messageCompletion**?: [`Maybe`](README.md#maybet)\<[`IGenerationMessage`](README.md#igenerationmessage)\> + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. + +###### Returns + +`any` + +A dictionary representing the serialized properties of the object. + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`serialize`](README.md#serialize) + +*** + +### LiteralClient + +#### Constructors + +##### new LiteralClient(__namedParameters) + +> **new LiteralClient**(`__namedParameters`): [`LiteralClient`](README.md#literalclient) + +###### Parameters + +▪ **\_\_namedParameters**: `object`= `{}` + +▪ **\_\_namedParameters.apiKey?**: `string` + +▪ **\_\_namedParameters.apiUrl?**: `string` + +▪ **\_\_namedParameters.environment?**: [`Environment`](README.md#environment-3) + +▪ **\_\_namedParameters.disabled?**: `boolean` + +###### Returns + +[`LiteralClient`](README.md#literalclient) + +#### Properties + +##### api + +> **api**: `API` + +##### openai + +> **openai**: (`openai`) => `object` + +###### Parameters + +▪ **openai**: `OpenAI` + +###### Returns + +`object` + +> ###### assistant +> +> > **assistant**: `object` +> +> ###### assistant.syncer +> +> > **assistant.syncer**: `OpenAIAssistantSyncer` +> + +##### instrumentation + +> **instrumentation**: `object` + +###### Type declaration + +###### openai + +> **openai**: (`options`?) => `object` + +###### Parameters + +▪ **options?**: [`OpenAIGlobalOptions`](README.md#openaiglobaloptions) + +###### Returns + +`object` + +> ###### chat +> +> > **chat**: `object` +> +> ###### chat.completions +> +> > **chat.completions**: `object` +> +> ###### chat.completions.create +> +> > **chat.completions.create**: (`this`, `body`, `callOptions`?) => `Promise`\<`Stream`\<`ChatCompletionChunk`\> \| `ChatCompletion`\> = `wrappedChatCompletionsCreate` +> +> ###### Parameters +> +> ▪ **this**: `any` +> +> ▪ **body**: `any` +> +> ▪ **callOptions?**: `RequestOptions` & `OpenAICallOptions` +> +> ###### Returns +> +> `Promise`\<`Stream`\<`ChatCompletionChunk`\> \| `ChatCompletion`\> +> +> ###### completions +> +> > **completions**: `object` +> +> ###### completions.create +> +> > **completions.create**: (`this`, `body`, `callOptions`?) => `Promise`\<`Completion` \| `Stream`\<`Completion`\>\> = `wrappedCompletionsCreate` +> +> ###### Parameters +> +> ▪ **this**: `any` +> +> ▪ **body**: `any` +> +> ▪ **callOptions?**: `RequestOptions` & `OpenAICallOptions` +> +> ###### Returns +> +> `Promise`\<`Completion` \| `Stream`\<`Completion`\>\> +> +> ###### images +> +> > **images**: `object` +> +> ###### images.generate +> +> > **images.generate**: (`this`, `body`, `callOptions`?) => `Promise`\<`ImagesResponse`\> = `wrappedImagesGenerate` +> +> ###### Parameters +> +> ▪ **this**: `any` +> +> ▪ **body**: `any` +> +> ▪ **callOptions?**: `RequestOptions` & `OpenAICallOptions` +> +> ###### Returns +> +> `Promise`\<`ImagesResponse`\> +> + +###### langchain + +> **langchain**: `object` + +###### langchain.literalCallback + +> **langchain.literalCallback**: (`threadId`?) => `LiteralCallbackHandler` + +###### Parameters + +▪ **threadId?**: `string` + +###### Returns + +`LiteralCallbackHandler` + +###### vercel + +> **vercel**: `object` + +###### vercel.instrument + +> **vercel.instrument**: `InstrumentationVercelMethod` + +###### llamaIndex + +> **llamaIndex**: `object` + +###### llamaIndex.instrument + +> **llamaIndex.instrument**: () => `void` + +###### Returns + +`void` + +###### llamaIndex.withThread + +> **llamaIndex.withThread**: \<`R`\>(`thread`, `callback`) => `R` + +###### Type parameters + +▪ **R** + +###### Parameters + +▪ **thread**: [`Thread`](README.md#thread-1) + +▪ **callback**: () => `R` + +###### Returns + +`R` + +##### store + +> **store**: `AsyncLocalStorage`\<`StoredContext`\> = `storage` + +#### Methods + +##### thread() + +> **thread**(`data`?): [`Thread`](README.md#thread-1) + +###### Parameters + +▪ **data?**: [`ThreadConstructor`](README.md#threadconstructor) + +###### Returns + +[`Thread`](README.md#thread-1) + +##### step() + +> **step**(`data`): [`Step`](README.md#step-2) + +###### Parameters + +▪ **data**: [`StepConstructor`](README.md#stepconstructor) + +###### Returns + +[`Step`](README.md#step-2) + +##### run() + +> **run**(`data`): [`Step`](README.md#step-2) + +###### Parameters + +▪ **data**: `Omit`\<[`StepConstructor`](README.md#stepconstructor), `"type"`\> + +###### Returns + +[`Step`](README.md#step-2) + +##### experimentRun() + +> **experimentRun**(`data`?): [`ExperimentRun`](README.md#experimentrun-1) + +###### Parameters + +▪ **data?**: `Omit`\<[`StepConstructor`](README.md#stepconstructor), `"name"` \| `"type"`\> + +###### Returns + +[`ExperimentRun`](README.md#experimentrun-1) + +##### \_currentThread() + +> **\_currentThread**(): `null` \| [`Thread`](README.md#thread-1) + +###### Returns + +`null` \| [`Thread`](README.md#thread-1) + +##### \_currentStep() + +> **\_currentStep**(): `null` \| [`Step`](README.md#step-2) + +###### Returns + +`null` \| [`Step`](README.md#step-2) + +##### getCurrentThread() + +> **getCurrentThread**(): [`Thread`](README.md#thread-1) + +Gets the current thread from the context. +WARNING : this will throw if run outside of a thread context. + +###### Returns + +[`Thread`](README.md#thread-1) + +The current thread, if any. + +##### getCurrentStep() + +> **getCurrentStep**(): [`Step`](README.md#step-2) + +Gets the current step from the context. +WARNING : this will throw if run outside of a step context. + +###### Returns + +[`Step`](README.md#step-2) + +The current step, if any. + +*** + +### Utils + +Represents a utility class with serialization capabilities. + +#### Extended By + +- [`Attachment`](README.md#attachment) +- [`User`](README.md#user) +- [`DatasetItem`](README.md#datasetitem) +- [`DatasetExperiment`](README.md#datasetexperiment) +- [`BaseGeneration`](README.md#basegeneration) + +#### Constructors + +##### new Utils() + +> **new Utils**(): [`Utils`](README.md#utils) + +###### Returns + +[`Utils`](README.md#utils) + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. + +###### Returns + +`any` + +A dictionary representing the serialized properties of the object. + +*** + +### Score + +Represents a score entity with properties to track various aspects of scoring. +It extends the `Utils` class for serialization capabilities. + +#### Extends + +- `ScoreFields` + +#### Constructors + +##### new Score(data) + +> **new Score**(`data`): [`Score`](README.md#score) + +###### Parameters + +▪ **data**: [`ScoreConstructor`](README.md#scoreconstructor) + +###### Returns + +[`Score`](README.md#score) + +###### Overrides + +ScoreFields.constructor + +#### Properties + +##### id + +> **id**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +ScoreFields.id + +##### stepId + +> **stepId**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +ScoreFields.stepId + +##### generationId + +> **generationId**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +ScoreFields.generationId + +##### datasetExperimentItemId + +> **datasetExperimentItemId**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +ScoreFields.datasetExperimentItemId + +##### name + +> **name**: `string` = `'user-feedback'` + +###### Inherited from + +ScoreFields.name + +##### value + +> **value**: `number` = `0` + +###### Inherited from + +ScoreFields.value + +##### type + +> **type**: [`ScoreType`](README.md#scoretype) = `'AI'` + +###### Inherited from + +ScoreFields.type + +##### scorer + +> **scorer**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +ScoreFields.scorer + +##### comment + +> **comment**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +ScoreFields.comment + +##### tags + +> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> + +###### Inherited from + +ScoreFields.tags + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. + +###### Returns + +`any` + +A dictionary representing the serialized properties of the object. + +###### Inherited from + +ScoreFields.serialize + +*** + +### Attachment + +Represents an attachment with optional metadata, MIME type, and other properties. +It extends the `Utils` class for serialization capabilities. + +#### Extends + +- [`Utils`](README.md#utils) + +#### Constructors + +##### new Attachment(data) + +> **new Attachment**(`data`): [`Attachment`](README.md#attachment) + +###### Parameters + +▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`Attachment`](README.md#attachment)\> + +###### Returns + +[`Attachment`](README.md#attachment) + +###### Overrides + +[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) + +#### Properties + +##### id + +> **id**?: [`Maybe`](README.md#maybet)\<`string`\> + +##### metadata + +> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +##### mime + +> **mime**?: [`Maybe`](README.md#maybet)\<`string`\> + +##### name + +> **name**: [`Maybe`](README.md#maybet)\<`string`\> + +##### objectKey + +> **objectKey**?: [`Maybe`](README.md#maybet)\<`string`\> + +##### url + +> **url**?: [`Maybe`](README.md#maybet)\<`string`\> + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. + +###### Returns + +`any` + +A dictionary representing the serialized properties of the object. + +###### Inherited from + +[`Utils`](README.md#utils).[`serialize`](README.md#serialize-3) + +*** + +### Thread + +Represents a thread in the system, extending the properties and methods from `ThreadFields`. +This class manages thread-specific operations such as creation and updates via the API. + +#### Extends + +- `ThreadFields` + +#### Constructors + +##### new Thread(client, data) + +> **new Thread**(`client`, `data`?): [`Thread`](README.md#thread-1) + +Constructs a new Thread instance. + +###### Parameters + +▪ **client**: [`LiteralClient`](README.md#literalclient) + +▪ **data?**: [`ThreadConstructor`](README.md#threadconstructor) + +Optional initial data for the thread, with an auto-generated ID if not provided. + +###### Returns + +[`Thread`](README.md#thread-1) + +###### Overrides + +ThreadFields.constructor + +#### Properties + +##### id + +> **id**: `string` + +###### Inherited from + +ThreadFields.id + +##### participantId + +> **participantId**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +ThreadFields.participantId + +##### environment + +> **environment**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +ThreadFields.environment + +##### name + +> **name**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +ThreadFields.name + +##### metadata + +> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +###### Inherited from + +ThreadFields.metadata + +##### steps + +> **steps**?: [`Maybe`](README.md#maybet)\<[`Step`](README.md#step-2)[]\> + +###### Inherited from + +ThreadFields.steps + +##### tags + +> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> + +###### Inherited from + +ThreadFields.tags + +##### api + +> **api**: `API` + +##### client + +> **client**: [`LiteralClient`](README.md#literalclient) + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. + +###### Returns + +`any` + +A dictionary representing the serialized properties of the object. + +###### Inherited from + +ThreadFields.serialize + +##### step() + +> **step**(`data`): [`Step`](README.md#step-2) + +Creates a new step associated with this thread. + +###### Parameters + +▪ **data**: `Omit`\<[`StepConstructor`](README.md#stepconstructor), `"threadId"`\> + +The data for the new step, excluding the thread ID. + +###### Returns + +[`Step`](README.md#step-2) + +A new Step instance linked to this thread. + +##### run() + +> **run**(`data`): [`Step`](README.md#step-2) + +Creates a new Run step associated with this thread. + +###### Parameters + +▪ **data**: `Omit`\<[`StepConstructor`](README.md#stepconstructor), `"type"` \| `"threadId"`\> + +The data for the new step, excluding the thread ID and the type + +###### Returns + +[`Step`](README.md#step-2) + +A new Step instance linked to this thread. + +##### upsert() + +> **upsert**(): `Promise`\<[`Thread`](README.md#thread-1)\> + +Upserts the thread data to the backend, creating or updating as necessary. + +###### Returns + +`Promise`\<[`Thread`](README.md#thread-1)\> + +The updated Thread instance. + +##### wrap() + +> **wrap**\<`Output`\>(`cb`, `updateThread`?): `Promise`\<`Output`\> + +Sends the thread to the API, handling disabled state and setting the end time if not already set. + +###### Type parameters + +▪ **Output** + +###### Parameters + +▪ **cb**: (`thread`) => `Output` \| `Promise`\<`Output`\> + +The callback function to run within the context of the thread. + +▪ **updateThread?**: [`ThreadConstructor`](README.md#threadconstructor) \| (`output`) => [`ThreadConstructor`](README.md#threadconstructor) \| (`output`) => `Promise`\<[`ThreadConstructor`](README.md#threadconstructor)\> + +Optional update function to modify the thread after the callback. + +###### Returns + +`Promise`\<`Output`\> + +The output of the wrapped callback function. + +*** + +### Step + +Represents a step in a process or workflow, extending the fields and methods from StepFields. + +#### Extends + +- `StepFields` + +#### Constructors + +##### new Step(client, data, ignoreContext) + +> **new Step**(`client`, `data`, `ignoreContext`?): [`Step`](README.md#step-2) + +Constructs a new Step instance. + +###### Parameters + +▪ **client**: [`LiteralClient`](README.md#literalclient) + +▪ **data**: [`StepConstructor`](README.md#stepconstructor) + +The initial data for the step, excluding utility properties. + +▪ **ignoreContext?**: `true` + +###### Returns + +[`Step`](README.md#step-2) + +###### Overrides + +StepFields.constructor + +#### Properties + +##### name + +> **name**: `string` + +###### Inherited from + +StepFields.name + +##### type + +> **type**: [`StepType`](README.md#steptype) + +###### Inherited from + +StepFields.type + +##### threadId + +> **threadId**?: `string` + +###### Inherited from + +StepFields.threadId + +##### createdAt + +> **createdAt**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +StepFields.createdAt + +##### startTime + +> **startTime**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +StepFields.startTime + +##### id + +> **id**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +StepFields.id + +##### environment + +> **environment**?: [`Maybe`](README.md#maybet)\<[`Environment`](README.md#environment-3)\> + +###### Inherited from + +StepFields.environment + +##### error + +> **error**?: [`Maybe`](README.md#maybet)\<`string` \| `Record`\<`string`, `any`\>\> + +###### Inherited from + +StepFields.error + +##### input + +> **input**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +###### Inherited from + +StepFields.input + +##### output + +> **output**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +###### Inherited from + +StepFields.output + +##### metadata + +> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +###### Inherited from + +StepFields.metadata + +##### tags + +> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> + +###### Inherited from + +StepFields.tags + +##### parentId + +> **parentId**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +StepFields.parentId + +##### endTime + +> **endTime**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +StepFields.endTime + +##### generation + +> **generation**?: [`Maybe`](README.md#maybet)\<[`Generation`](README.md#generation-2)\> + +###### Inherited from + +StepFields.generation + +##### scores + +> **scores**?: [`Maybe`](README.md#maybet)\<[`Score`](README.md#score)[]\> + +###### Inherited from + +StepFields.scores + +##### attachments + +> **attachments**?: [`Maybe`](README.md#maybet)\<[`Attachment`](README.md#attachment)[]\> + +###### Inherited from + +StepFields.attachments + +##### api + +> **api**: `API` + +##### client + +> **client**: [`LiteralClient`](README.md#literalclient) + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the step instance, converting complex objects to strings as necessary. + +###### Returns + +`any` + +A serialized representation of the step. + +###### Overrides + +StepFields.serialize + +##### isMessage() + +> **isMessage**(): `boolean` + +Determines if the step is a type of message. + +###### Returns + +`boolean` + +True if the step is a user, assistant, or system message. + +##### step() + +> **step**(`data`): [`Step`](README.md#step-2) + +Creates a new step instance linked to the current step as a parent. + +###### Parameters + +▪ **data**: `Omit`\<[`StepConstructor`](README.md#stepconstructor), `"threadId"`\> + +The data for the new step, excluding the threadId which is inherited. + +###### Returns + +[`Step`](README.md#step-2) + +A new Step instance. + +##### send() + +> **send**(): `Promise`\<[`Step`](README.md#step-2)\> + +Sends the step to the API, handling disabled state and setting the end time if not already set. + +###### Returns + +`Promise`\<[`Step`](README.md#step-2)\> + +The current Step instance after potentially sending to the API. + +##### wrap() + +> **wrap**\<`Output`\>(`cb`, `updateStep`?): `Promise`\<`Output`\> + +Sends the step to the API, handling disabled state and setting the end time if not already set. + +###### Type parameters + +▪ **Output** + +###### Parameters + +▪ **cb**: (`step`) => `Output` \| `Promise`\<`Output`\> + +The callback function to run within the context of the step. + +▪ **updateStep?**: `Partial`\<[`StepConstructor`](README.md#stepconstructor)\> \| (`output`) => `Partial`\<[`StepConstructor`](README.md#stepconstructor)\> \| (`output`) => `Promise`\<`Partial`\<[`StepConstructor`](README.md#stepconstructor)\>\> + +Optional update function to modify the step after the callback. + +###### Returns + +`Promise`\<`Output`\> + +The output of the wrapped callback function. + +*** + +### ExperimentRun + +Represents a step in a process or workflow, extending the fields and methods from StepFields. + +#### Extends + +- [`Step`](README.md#step-2) + +#### Constructors + +##### new ExperimentRun(client, data, ignoreContext) + +> **new ExperimentRun**(`client`, `data`, `ignoreContext`?): [`ExperimentRun`](README.md#experimentrun-1) + +Constructs a new ExperimentRun instance. + +###### Parameters + +▪ **client**: [`LiteralClient`](README.md#literalclient) + +▪ **data**: [`StepConstructor`](README.md#stepconstructor) + +The initial data for the step, excluding utility properties. + +▪ **ignoreContext?**: `true` + +###### Returns + +[`ExperimentRun`](README.md#experimentrun-1) + +###### Overrides + +[`Step`](README.md#step-2).[`constructor`](README.md#constructors-8) + +#### Properties + +##### name + +> **name**: `string` + +###### Inherited from + +[`Step`](README.md#step-2).[`name`](README.md#name-3) + +##### type + +> **type**: [`StepType`](README.md#steptype) + +###### Inherited from + +[`Step`](README.md#step-2).[`type`](README.md#type-3) + +##### threadId + +> **threadId**?: `string` + +###### Inherited from + +[`Step`](README.md#step-2).[`threadId`](README.md#threadid) + +##### createdAt + +> **createdAt**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +[`Step`](README.md#step-2).[`createdAt`](README.md#createdat) + +##### startTime + +> **startTime**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +[`Step`](README.md#step-2).[`startTime`](README.md#starttime) + +##### id + +> **id**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +[`Step`](README.md#step-2).[`id`](README.md#id-6) + +##### environment + +> **environment**?: [`Maybe`](README.md#maybet)\<[`Environment`](README.md#environment-3)\> + +###### Inherited from + +[`Step`](README.md#step-2).[`environment`](README.md#environment-1) + +##### error + +> **error**?: [`Maybe`](README.md#maybet)\<`string` \| `Record`\<`string`, `any`\>\> + +###### Inherited from + +[`Step`](README.md#step-2).[`error`](README.md#error-3) + +##### input + +> **input**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +###### Inherited from + +[`Step`](README.md#step-2).[`input`](README.md#input) + +##### output + +> **output**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +###### Inherited from + +[`Step`](README.md#step-2).[`output`](README.md#output) + +##### metadata + +> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +###### Inherited from + +[`Step`](README.md#step-2).[`metadata`](README.md#metadata-2) + +##### tags + +> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> + +###### Inherited from + +[`Step`](README.md#step-2).[`tags`](README.md#tags-5) + +##### parentId + +> **parentId**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +[`Step`](README.md#step-2).[`parentId`](README.md#parentid) + +##### endTime + +> **endTime**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +[`Step`](README.md#step-2).[`endTime`](README.md#endtime) + +##### generation + +> **generation**?: [`Maybe`](README.md#maybet)\<[`Generation`](README.md#generation-2)\> + +###### Inherited from + +[`Step`](README.md#step-2).[`generation`](README.md#generation) + +##### scores + +> **scores**?: [`Maybe`](README.md#maybet)\<[`Score`](README.md#score)[]\> + +###### Inherited from + +[`Step`](README.md#step-2).[`scores`](README.md#scores) + +##### attachments + +> **attachments**?: [`Maybe`](README.md#maybet)\<[`Attachment`](README.md#attachment)[]\> + +###### Inherited from + +[`Step`](README.md#step-2).[`attachments`](README.md#attachments) + +##### api + +> **api**: `API` + +###### Overrides + +[`Step`](README.md#step-2).[`api`](README.md#api-2) + +##### client + +> **client**: [`LiteralClient`](README.md#literalclient) + +###### Overrides + +[`Step`](README.md#step-2).[`client`](README.md#client-1) + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the step instance, converting complex objects to strings as necessary. + +###### Returns + +`any` + +A serialized representation of the step. + +###### Inherited from + +[`Step`](README.md#step-2).[`serialize`](README.md#serialize-7) + +##### isMessage() + +> **isMessage**(): `boolean` + +Determines if the step is a type of message. + +###### Returns + +`boolean` + +True if the step is a user, assistant, or system message. + +###### Inherited from + +[`Step`](README.md#step-2).[`isMessage`](README.md#ismessage) + +##### step() + +> **step**(`data`): [`Step`](README.md#step-2) + +Creates a new step instance linked to the current step as a parent. + +###### Parameters + +▪ **data**: `Omit`\<[`StepConstructor`](README.md#stepconstructor), `"threadId"`\> + +The data for the new step, excluding the threadId which is inherited. + +###### Returns + +[`Step`](README.md#step-2) + +A new Step instance. + +###### Inherited from + +[`Step`](README.md#step-2).[`step`](README.md#step-3) + +##### send() + +> **send**(): `Promise`\<[`ExperimentRun`](README.md#experimentrun-1)\> + +Sends the step to the API, handling disabled state and setting the end time if not already set. + +###### Returns + +`Promise`\<[`ExperimentRun`](README.md#experimentrun-1)\> + +The current Step instance after potentially sending to the API. + +###### Inherited from + +[`Step`](README.md#step-2).[`send`](README.md#send) + +##### wrap() + +> **wrap**\<`Output`\>(`cb`, `updateStep`?): `Promise`\<`Output`\> + +Sends the step to the API, handling disabled state and setting the end time if not already set. + +###### Type parameters + +▪ **Output** + +###### Parameters + +▪ **cb**: (`step`) => `Output` \| `Promise`\<`Output`\> + +The callback function to run within the context of the step. + +▪ **updateStep?**: `Partial`\<[`StepConstructor`](README.md#stepconstructor)\> \| (`output`) => `Partial`\<[`StepConstructor`](README.md#stepconstructor)\> \| (`output`) => `Promise`\<`Partial`\<[`StepConstructor`](README.md#stepconstructor)\>\> + +Optional update function to modify the step after the callback. + +###### Returns + +`Promise`\<`Output`\> + +The output of the wrapped callback function. + +###### Overrides + +[`Step`](README.md#step-2).[`wrap`](README.md#wrap-1) + +*** + +### User + +Represents a user with optional metadata and identifier. + +#### Extends + +- [`Utils`](README.md#utils) + +#### Constructors + +##### new User(data) + +> **new User**(`data`): [`User`](README.md#user) + +###### Parameters + +▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`User`](README.md#user)\> + +###### Returns + +[`User`](README.md#user) + +###### Overrides + +[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) + +#### Properties + +##### id + +> **id**?: [`Maybe`](README.md#maybet)\<`string`\> + +##### identifier + +> **identifier**: `string` + +##### metadata + +> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. + +###### Returns + +`any` + +A dictionary representing the serialized properties of the object. + +###### Inherited from + +[`Utils`](README.md#utils).[`serialize`](README.md#serialize-3) + +*** + +### Dataset + +#### Extends + +- `DatasetFields` + +#### Constructors + +##### new Dataset(api, data) + +> **new Dataset**(`api`, `data`): [`Dataset`](README.md#dataset) + +Constructs a new Dataset instance. + +###### Parameters + +▪ **api**: `API` + +The API instance to interact with backend services. + +▪ **data**: [`DatasetConstructor`](README.md#datasetconstructor) + +The initial data for the dataset. + +###### Returns + +[`Dataset`](README.md#dataset) + +###### Overrides + +DatasetFields.constructor + +#### Properties + +##### id + +> **id**: `string` + +###### Inherited from + +DatasetFields.id + +##### createdAt + +> **createdAt**: `string` + +###### Inherited from + +DatasetFields.createdAt + +##### name + +> **name**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +DatasetFields.name + +##### description + +> **description**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +DatasetFields.description + +##### metadata + +> **metadata**: `Record`\<`string`, `any`\> + +###### Inherited from + +DatasetFields.metadata + +##### items + +> **items**: [`OmitUtils`](README.md#omitutilst)\<[`DatasetItem`](README.md#datasetitem)\>[] + +###### Inherited from + +DatasetFields.items + +##### type + +> **type**?: [`DatasetType`](README.md#datasettype) + +###### Inherited from + +DatasetFields.type + +##### api + +> **api**: `API` + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. + +###### Returns + +`any` + +A dictionary representing the serialized properties of the object. + +###### Inherited from + +DatasetFields.serialize + +##### update() + +> **update**(`dataset`): `Promise`\<`void`\> + +Updates the dataset with new data. + +###### Parameters + +▪ **dataset**: `object` + +The dataset data to update. + +▪ **dataset.name?**: [`Maybe`](README.md#maybet)\<`string`\> + +▪ **dataset.description?**: [`Maybe`](README.md#maybet)\<`string`\> + +▪ **dataset.metadata?**: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +###### Returns + +`Promise`\<`void`\> + +The updated dataset instance. + +##### delete() + +> **delete**(): `Promise`\<[`Dataset`](README.md#dataset)\> + +Deletes the dataset. + +###### Returns + +`Promise`\<[`Dataset`](README.md#dataset)\> + +A promise that resolves when the dataset is deleted. + +##### createItem() + +> **createItem**(`datasetItem`): `Promise`\<[`DatasetItem`](README.md#datasetitem)\> + +Creates a new item in the dataset. + +###### Parameters + +▪ **datasetItem**: `object` + +The new item to be added to the dataset. + +▪ **datasetItem.input**: `Record`\<`string`, `any`\> + +▪ **datasetItem.expectedOutput?**: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +▪ **datasetItem.metadata?**: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +###### Returns + +`Promise`\<[`DatasetItem`](README.md#datasetitem)\> + +The newly created dataset item. + +##### deleteItem() + +> **deleteItem**(`id`): `Promise`\<[`DatasetItem`](README.md#datasetitem)\> + +Deletes an item from the dataset. + +###### Parameters + +▪ **id**: `string` + +The ID of the item to delete. + +###### Returns + +`Promise`\<[`DatasetItem`](README.md#datasetitem)\> + +The deleted dataset item. + +##### createExperiment() + +> **createExperiment**(`experiment`): `Promise`\<[`DatasetExperiment`](README.md#datasetexperiment)\> + +Creates a new experiment associated with the dataset. + +###### Parameters + +▪ **experiment**: `object` + +The experiment details including name, optional prompt ID, and parameters. + +▪ **experiment.name**: `string` + +▪ **experiment.promptId?**: `string` + +▪ **experiment.params?**: `Record`\<`string`, `any`\> \| `Record`\<`string`, `any`\>[] + +###### Returns + +`Promise`\<[`DatasetExperiment`](README.md#datasetexperiment)\> + +A new instance of DatasetExperiment containing the created experiment. + +##### addStep() + +> **addStep**(`stepId`, `metadata`?): `Promise`\<[`DatasetItem`](README.md#datasetitem)\> + +Adds a step to the dataset. + +###### Parameters + +▪ **stepId**: `string` + +The ID of the step to add. + +▪ **metadata?**: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `unknown`\>\> + +Optional metadata for the step. + +###### Returns + +`Promise`\<[`DatasetItem`](README.md#datasetitem)\> + +The added dataset item. + +###### Throws + +Error if the dataset type is 'generation'. + +##### addGeneration() + +> **addGeneration**(`generationId`, `metadata`?): `Promise`\<[`DatasetItem`](README.md#datasetitem)\> + +Adds a generation to the dataset. + +###### Parameters + +▪ **generationId**: `string` + +The ID of the generation to add. + +▪ **metadata?**: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `unknown`\>\> + +Optional metadata for the generation. + +###### Returns + +`Promise`\<[`DatasetItem`](README.md#datasetitem)\> + +The added dataset item. + +##### addGenerations() + +> **addGenerations**(`generationIds`?): `Promise`\<[`DatasetItem`](README.md#datasetitem)[]\> + +###### Parameters + +▪ **generationIds?**: `string`[] + +###### Returns + +`Promise`\<[`DatasetItem`](README.md#datasetitem)[]\> + +*** + +### DatasetItem + +Represents a utility class with serialization capabilities. + +#### Extends + +- [`Utils`](README.md#utils) + +#### Constructors + +##### new DatasetItem(data) + +> **new DatasetItem**(`data`): [`DatasetItem`](README.md#datasetitem) + +###### Parameters + +▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`DatasetItem`](README.md#datasetitem)\> + +###### Returns + +[`DatasetItem`](README.md#datasetitem) + +###### Overrides + +[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) + +#### Properties + +##### id + +> **id**: `string` + +##### createdAt + +> **createdAt**: `string` + +##### datasetId + +> **datasetId**: `string` + +##### metadata + +> **metadata**: `Record`\<`string`, `any`\> + +##### input + +> **input**: `Record`\<`string`, `any`\> + +##### expectedOutput + +> **expectedOutput**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +##### intermediarySteps + +> **intermediarySteps**: `Record`\<`string`, `any`\>[] + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. + +###### Returns + +`any` + +A dictionary representing the serialized properties of the object. + +###### Inherited from + +[`Utils`](README.md#utils).[`serialize`](README.md#serialize-3) + +*** + +### DatasetExperiment + +Represents a utility class with serialization capabilities. + +#### Extends + +- [`Utils`](README.md#utils) + +#### Constructors + +##### new DatasetExperiment(api, data) + +> **new DatasetExperiment**(`api`, `data`): [`DatasetExperiment`](README.md#datasetexperiment) + +###### Parameters + +▪ **api**: `API` + +▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`DatasetExperiment`](README.md#datasetexperiment)\> + +###### Returns + +[`DatasetExperiment`](README.md#datasetexperiment) + +###### Overrides + +[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) + +#### Properties + +##### id + +> **id**: `string` + +##### createdAt + +> **createdAt**: `string` + +##### name + +> **name**: `string` + +##### datasetId + +> **datasetId**?: `string` + +##### promptId + +> **promptId**?: `string` + +##### api + +> **api**: `API` + +##### params + +> **params**: `Record`\<`string`, `any`\> \| `Record`\<`string`, `any`\>[] + +##### items + +> **items**: [`DatasetExperimentItem`](README.md#datasetexperimentitem)[] + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. + +###### Returns + +`any` + +A dictionary representing the serialized properties of the object. + +###### Inherited from + +[`Utils`](README.md#utils).[`serialize`](README.md#serialize-3) + +##### log() + +> **log**(`itemFields`): `Promise`\<[`DatasetExperimentItem`](README.md#datasetexperimentitem)\> + +###### Parameters + +▪ **itemFields**: `Omit`\<[`OmitUtils`](README.md#omitutilst)\<`DatasetExperimentItemFields`\>, `"id"` \| `"datasetExperimentId"`\> + +###### Returns + +`Promise`\<[`DatasetExperimentItem`](README.md#datasetexperimentitem)\> + +*** + +### DatasetExperimentItem + +#### Extends + +- `DatasetExperimentItemFields` + +#### Constructors + +##### new DatasetExperimentItem(data) + +> **new DatasetExperimentItem**(`data`): [`DatasetExperimentItem`](README.md#datasetexperimentitem) + +###### Parameters + +▪ **data**: [`DatasetExperimentItemConstructor`](README.md#datasetexperimentitemconstructor) + +###### Returns + +[`DatasetExperimentItem`](README.md#datasetexperimentitem) + +###### Overrides + +DatasetExperimentItemFields.constructor + +#### Properties + +##### id + +> **id**?: `string` + +###### Inherited from + +DatasetExperimentItemFields.id + +##### datasetExperimentId + +> **datasetExperimentId**: `string` + +###### Inherited from + +DatasetExperimentItemFields.datasetExperimentId + +##### datasetItemId + +> **datasetItemId**?: `string` + +###### Inherited from + +DatasetExperimentItemFields.datasetItemId + +##### experimentRunId + +> **experimentRunId**?: `string` + +###### Inherited from + +DatasetExperimentItemFields.experimentRunId + +##### scores + +> **scores**: [`ScoreConstructor`](README.md#scoreconstructor)[] + +###### Inherited from + +DatasetExperimentItemFields.scores + +##### input + +> **input**?: `Record`\<`string`, `any`\> + +###### Inherited from + +DatasetExperimentItemFields.input + +##### output + +> **output**?: `Record`\<`string`, `any`\> + +###### Inherited from + +DatasetExperimentItemFields.output + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. + +###### Returns + +`any` + +A dictionary representing the serialized properties of the object. + +###### Inherited from + +DatasetExperimentItemFields.serialize + +*** + +### Prompt + +#### Extends + +- `PromptFields` + +#### Constructors + +##### new Prompt(api, data) + +> **new Prompt**(`api`, `data`): [`Prompt`](README.md#prompt-1) + +Constructs a new Prompt instance. + +###### Parameters + +▪ **api**: `API` + +The API instance to interact with backend services. + +▪ **data**: [`PromptConstructor`](README.md#promptconstructor) + +The initial data for the prompt. + +###### Returns + +[`Prompt`](README.md#prompt-1) + +###### Overrides + +PromptFields.constructor + +#### Properties + +##### id + +> **id**: `string` + +###### Inherited from + +PromptFields.id + +##### type + +> **type**: [`GenerationType`](README.md#generationtype) + +###### Inherited from + +PromptFields.type + +##### createdAt + +> **createdAt**: `string` + +###### Inherited from + +PromptFields.createdAt + +##### name + +> **name**: `string` + +###### Inherited from + +PromptFields.name + +##### version + +> **version**: `number` + +###### Inherited from + +PromptFields.version + +##### versionDesc + +> **versionDesc**?: [`Maybe`](README.md#maybet)\<`string`\> + +###### Inherited from + +PromptFields.versionDesc + +##### metadata + +> **metadata**: `Record`\<`string`, `any`\> + +###### Inherited from + +PromptFields.metadata + +##### items + +> **items**: [`OmitUtils`](README.md#omitutilst)\<[`DatasetItem`](README.md#datasetitem)\>[] + +###### Inherited from + +PromptFields.items + +##### variablesDefaultValues + +> **variablesDefaultValues**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +###### Inherited from + +PromptFields.variablesDefaultValues + +##### templateMessages + +> **templateMessages**: [`IGenerationMessage`](README.md#igenerationmessage)[] + +###### Inherited from + +PromptFields.templateMessages + +##### tools + +> **tools**?: `ChatCompletionTool`[] + +###### Inherited from + +PromptFields.tools + +##### provider + +> **provider**: `string` + +###### Inherited from + +PromptFields.provider + +##### settings + +> **settings**: [`IProviderSettings`](README.md#iprovidersettings) + +###### Inherited from + +PromptFields.settings + +##### variables + +> **variables**: [`IPromptVariableDefinition`](README.md#ipromptvariabledefinition)[] + +###### Inherited from + +PromptFields.variables + +##### api + +> **api**: `API` + +#### Methods + +##### serialize() + +> **serialize**(): `any` + +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. + +###### Returns + +`any` + +A dictionary representing the serialized properties of the object. + +###### Inherited from + +PromptFields.serialize + +##### formatMessages() + +> **formatMessages**(`variables`?): `ChatCompletionMessageParam`[] + +Formats the prompt's template messages with the given variables. + +###### Parameters + +▪ **variables?**: `Record`\<`string`, `any`\> + +Optional variables to resolve in the template messages. + +###### Returns + +`ChatCompletionMessageParam`[] + +An array of formatted chat completion messages. + +##### format() + +> **format**(`variables`?): `ChatCompletionMessageParam`[] + +###### Parameters + +▪ **variables?**: `Record`\<`string`, `any`\> + +###### Returns + +`ChatCompletionMessageParam`[] + +###### Deprecated + +Please use `formatMessages` instead. + +##### toLangchainChatPromptTemplate() + +> **toLangchainChatPromptTemplate**(): `CustomChatPromptTemplate` + +Converts the prompt's template messages into a Langchain chat prompt template. + +###### Returns + +`CustomChatPromptTemplate` + +A custom chat prompt template configured with the prompt's data. + +## Interfaces + +### ITextContent + +#### Properties + +##### type + +> **type**: `"text"` + +##### text + +> **text**: `string` + +*** + +### IImageUrlContent + +#### Properties + +##### type + +> **type**: `"image_url"` + +##### image\_url + +> **image\_url**: `string` + +*** + +### IGenerationMessage + +#### Properties + +##### uuid + +> **uuid**?: `string` + +##### templated + +> **templated**?: `boolean` + +##### content + +> **content**: `null` \| `string` \| ([`ITextContent`](README.md#itextcontent) \| [`IImageUrlContent`](README.md#iimageurlcontent))[] + +##### role + +> **role**: [`GenerationMessageRole`](README.md#generationmessagerole) + +##### name + +> **name**?: `string` + +##### function\_call + +> **function\_call**?: `Record`\<`string`, `any`\> + +##### tool\_calls + +> **tool\_calls**?: `Record`\<`string`, `any`\>[] + +##### tool\_call\_id + +> **tool\_call\_id**?: `string` + +*** + +### IFunction + +#### Properties + +##### name + +> **name**: `string` + +##### description + +> **description**: `string` + +##### parameters + +> **parameters**: `object` + +###### Type declaration + +###### required + +> **required**: `string`[] + +###### properties + +> **properties**: `Record`\<`string`, `object`\> + +*** + +### ITool + +#### Properties + +##### type + +> **type**: `string` + +##### function + +> **function**: [`IFunction`](README.md#ifunction) + +*** + +### IPromptVariableDefinition + +#### Properties + +##### name + +> **name**: `string` + +##### language + +> **language**: `"json"` \| `"plaintext"` + +*** + +### IProviderSettings + +#### Properties + +##### provider + +> **provider**: `string` + +##### model + +> **model**: `string` + +##### frequency\_penalty + +> **frequency\_penalty**: `number` + +##### max\_tokens + +> **max\_tokens**: `number` + +##### presence\_penalty + +> **presence\_penalty**: `number` + +##### stop + +> **stop**?: `string`[] + +##### temperature + +> **temperature**: `number` + +##### top\_p + +> **top\_p**: `number` + +## Type Aliases + +### GenerationMessageRole + +> **GenerationMessageRole**: `"system"` \| `"assistant"` \| `"user"` \| `"function"` \| `"tool"` + +*** + +### ILLMSettings + +> **ILLMSettings**: `Record`\<`string`, `string` \| `string`[] \| `number` \| `boolean`\> + +*** + +### GenerationType + +> **GenerationType**: `"COMPLETION"` \| `"CHAT"` + +*** + +### Generation + +> **Generation**: [`OmitUtils`](README.md#omitutilst)\<[`CompletionGeneration`](README.md#completiongeneration)\> \| [`OmitUtils`](README.md#omitutilst)\<[`ChatGeneration`](README.md#chatgeneration)\> + +*** + +### PersistedGeneration + +> **PersistedGeneration**: [`Generation`](README.md#generation-2) & `object` + +#### Type declaration + +##### id + +> **id**: `string` + +*** + +### OpenAIGlobalOptions + +> **OpenAIGlobalOptions**: `object` + +#### Type declaration + +##### tags + +> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> + +##### metadata + +> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> + +*** + +### Maybe`` + +> **Maybe**\<`T`\>: `T` \| `null` \| `undefined` + +#### Type parameters + +| Parameter | +| :------ | +| `T` | + +*** + +### OmitUtils`` + +> **OmitUtils**\<`T`\>: `Omit`\<`T`, keyof [`Utils`](README.md#utils)\> + +#### Type parameters + +| Parameter | +| :------ | +| `T` | + +*** + +### Environment + +> **Environment**: `"dev"` \| `"staging"` \| `"prod"` \| `"experiment"` + +*** + +### PageInfo + +> **PageInfo**: `object` + +#### Type declaration + +##### hasNextPage + +> **hasNextPage**: `boolean` + +##### startCursor + +> **startCursor**: `string` + +##### endCursor + +> **endCursor**: `string` + +*** + +### PaginatedResponse`` + +> **PaginatedResponse**\<`T`\>: `object` + +#### Type parameters + +| Parameter | +| :------ | +| `T` | + +#### Type declaration + +##### data + +> **data**: `T`[] + +##### pageInfo + +> **pageInfo**: [`PageInfo`](README.md#pageinfo) + +*** + +### ScoreType + +> **ScoreType**: `"HUMAN"` \| `"AI"` + +*** + +### ScoreConstructor + +> **ScoreConstructor**: [`OmitUtils`](README.md#omitutilst)\<`ScoreFields`\> + +*** + +### CleanThreadFields + +> **CleanThreadFields**: [`OmitUtils`](README.md#omitutilst)\<`ThreadFields`\> + +*** + +### ThreadConstructor + +> **ThreadConstructor**: `Omit`\<[`CleanThreadFields`](README.md#cleanthreadfields), `"id"`\> & `Partial`\<`Pick`\<[`CleanThreadFields`](README.md#cleanthreadfields), `"id"`\>\> + +*** + +### StepType + +> **StepType**: `"assistant_message"` \| `"embedding"` \| `"llm"` \| `"rerank"` \| `"retrieval"` \| `"run"` \| `"system_message"` \| `"tool"` \| `"undefined"` \| `"user_message"` + +*** + +### StepConstructor + +> **StepConstructor**: [`OmitUtils`](README.md#omitutilst)\<`StepFields`\> + +*** + +### DatasetType + +> **DatasetType**: `"key_value"` \| `"generation"` + +*** + +### DatasetConstructor + +> **DatasetConstructor**: [`OmitUtils`](README.md#omitutilst)\<`DatasetFields`\> + +*** + +### DatasetExperimentItemConstructor + +> **DatasetExperimentItemConstructor**: [`OmitUtils`](README.md#omitutilst)\<`DatasetExperimentItemFields`\> + +*** + +### PromptConstructor + +> **PromptConstructor**: [`OmitUtils`](README.md#omitutilst)\<`PromptFields`\> + +*** + +Generated using [typedoc-plugin-markdown](https://www.npmjs.com/package/typedoc-plugin-markdown) and [TypeDoc](https://typedoc.org/) diff --git a/docs/api.md b/docs/api.md deleted file mode 100644 index 61b9f1e..0000000 --- a/docs/api.md +++ /dev/null @@ -1,942 +0,0 @@ -**@literalai/client** - -*** - -# @literalai/client - -## Classes - -### API - -#### Methods - -##### sendSteps() - -> **sendSteps**(`steps`): `Promise`\<`any`\> - -Sends a collection of steps to the GraphQL endpoint. - -This method constructs a GraphQL query using the provided steps, then executes the query. - -###### Parameters - -▪ **steps**: `Step`[] - -An array of Step objects to be sent. - -###### Returns - -`Promise`\<`any`\> - -The response from the GraphQL call. - -##### getStep() - -> **getStep**(`id`): `Promise`\<`Maybe`\<`Step`\>\> - -Retrieves a step by its ID. - -This method constructs a GraphQL query to fetch a step by its unique identifier. -It then executes the query and returns the step if found. - -###### Parameters - -▪ **id**: `string` - -The unique identifier of the step to retrieve. - -###### Returns - -`Promise`\<`Maybe`\<`Step`\>\> - -A `Promise` that resolves to the step if found, or `null` if not found. - -##### deleteStep() - -> **deleteStep**(`id`): `Promise`\<`string`\> - -Deletes a Step from the platform by its unique identifier. - -This method constructs a GraphQL mutation to delete a step by its unique identifier. -It then executes the mutation and returns the ID of the deleted step. - -###### Parameters - -▪ **id**: `string` - -The unique identifier of the step to delete. - -###### Returns - -`Promise`\<`string`\> - -A `Promise` that resolves to the ID of the deleted step. - -##### uploadFile() - -> **uploadFile**(`params`): `Promise`\<`object`\> - -Uploads a file to a specified thread. This method supports uploading either through direct content or via a file path. -It first signs the upload through a pre-configured endpoint and then proceeds to upload the file using the signed URL. - -###### Parameters - -▪ **params**: `object` - -The parameters for uploading a file, including: - -▪ **params.content?**: `any` - -The content of the file to upload. Optional if `path` is provided. - -▪ **params.path?**: `Maybe`\<`string`\> - -The path to the file to upload. Optional if `content` is provided. - -▪ **params.id?**: `Maybe`\<`string`\> - -The unique identifier for the file. If not provided, a UUID will be generated. - -▪ **params.threadId**: `string` - -The unique identifier of the thread to which the file is being uploaded. - -▪ **params.mime?**: `Maybe`\<`string`\> - -The MIME type of the file. Defaults to 'application/octet-stream' if not provided. - -###### Returns - -`Promise`\<`object`\> - -An object containing the `objectKey` of the uploaded file and the signed `url`, or `null` values if the upload fails. - -###### Throws - -Throws an error if neither `content` nor `path` is provided, or if the server response is invalid. - -##### getGenerations() - -> **getGenerations**(`variables`): `Promise`\<`PaginatedResponse`\<`PersistedGeneration`\>\> - -Retrieves a paginated list of Generations based on the provided filters and sorting order. - -###### Parameters - -▪ **variables**: `object` - -The variables to filter and sort the Generations. It includes: -- `first`: The number of items to return. -- `after`: The cursor to fetch items after. -- `before`: The cursor to fetch items before. -- `filters`: The filters applied to the Generations. -- `orderBy`: The order in which the Generations are sorted. - -▪ **variables.first?**: `Maybe`\<`number`\> - -▪ **variables.after?**: `Maybe`\<`string`\> - -▪ **variables.before?**: `Maybe`\<`string`\> - -▪ **variables.filters?**: `GenerationsFilter`[] - -▪ **variables.orderBy?**: `GenerationsOrderBy` - -###### Returns - -`Promise`\<`PaginatedResponse`\<`PersistedGeneration`\>\> - -A `Promise` that resolves to a `PaginatedResponse` object containing the filtered and sorted Generations. - -##### createGeneration() - -> **createGeneration**(`generation`): `Promise`\<`PersistedGeneration`\> - -Creates a new generation entity and sends it to the platform. - -###### Parameters - -▪ **generation**: `Generation` - -The `Generation` object to be created and sent to the platform. - -###### Returns - -`Promise`\<`PersistedGeneration`\> - -A Promise resolving to the newly created `Generation` object. - -##### upsertThread() - -###### upsertThread(options) - -> **upsertThread**(`options`): `Promise`\<`CleanThreadFields`\> - -Upserts a Thread with new information. - -###### Parameters - -▪ **options**: `object` - -The parameters to upsert a thread. - -▪ **options.threadId**: `string` - -The unique identifier of the thread. (Required) - -▪ **options.name?**: `Maybe`\<`string`\> - -The name of the thread. (Optional) - -▪ **options.metadata?**: `Maybe`\<`Record`\<`string`, `any`\>\> - -Additional metadata for the thread as a key-value pair object. (Optional) - -▪ **options.participantId?**: `Maybe`\<`string`\> - -The unique identifier of the participant. (Optional) - -▪ **options.environment?**: `Maybe`\<`string`\> - -The environment where the thread is being upserted. (Optional) - -▪ **options.tags?**: `Maybe`\<`string`[]\> - -An array of tags associated with the thread. (Optional) - -###### Returns - -`Promise`\<`CleanThreadFields`\> - -The upserted thread object. - -###### upsertThread(threadId, name, metadata, participantId, environment, tags) - -> **upsertThread**(`threadId`, `name`?, `metadata`?, `participantId`?, `environment`?, `tags`?): `Promise`\<`CleanThreadFields`\> - -Upserts a Thread with new information. - -###### Parameters - -▪ **threadId**: `string` - -The unique identifier of the thread. (Required) - -▪ **name?**: `Maybe`\<`string`\> - -The name of the thread. (Optional) - -▪ **metadata?**: `Maybe`\<`Record`\<`string`, `any`\>\> - -Additional metadata for the thread as a key-value pair object. (Optional) - -▪ **participantId?**: `Maybe`\<`string`\> - -The unique identifier of the participant. (Optional) - -▪ **environment?**: `Maybe`\<`string`\> - -The environment where the thread is being upserted. (Optional) - -▪ **tags?**: `Maybe`\<`string`[]\> - -An array of tags associated with the thread. (Optional) - -###### Returns - -`Promise`\<`CleanThreadFields`\> - -The upserted thread object. - -###### Deprecated - -Use one single object attribute instead of multiple parameters. - -##### getThreads() - -> **getThreads**(`variables`): `Promise`\<`PaginatedResponse`\<`CleanThreadFields`\>\> - -Retrieves a paginated list of threads (conversations) based on the provided criteria. - -###### Parameters - -▪ **variables**: `object` - -The parameters to filter and paginate the threads. - -▪ **variables.first?**: `Maybe`\<`number`\> - -The number of threads to retrieve after the cursor. (Optional) - -▪ **variables.after?**: `Maybe`\<`string`\> - -The cursor to start retrieving threads after. (Optional) - -▪ **variables.before?**: `Maybe`\<`string`\> - -The cursor to start retrieving threads before. (Optional) - -▪ **variables.filters?**: `ThreadsFilter`[] - -The filters to apply on the threads retrieval. (Optional) - -▪ **variables.orderBy?**: `ThreadsOrderBy` - -The order in which to retrieve the threads. (Optional) - -▪ **variables.stepTypesToKeep?**: `StepType`[] - -###### Returns - -`Promise`\<`PaginatedResponse`\<`CleanThreadFields`\>\> - -A promise that resolves to a paginated response of threads. - -##### getThread() - -> **getThread**(`id`): `Promise`\<`any`\> - -Retrieves information from a single Thread. - -###### Parameters - -▪ **id**: `string` - -The unique identifier of the thread. This parameter is required. - -###### Returns - -`Promise`\<`any`\> - -The detailed information of the specified thread. - -##### deleteThread() - -> **deleteThread**(`id`): `Promise`\<`any`\> - -Deletes a single Thread by its unique identifier. - -###### Parameters - -▪ **id**: `string` - -The unique identifier of the thread to be deleted. This parameter is required. - -###### Returns - -`Promise`\<`any`\> - -The ID of the deleted thread. - -##### getUsers() - -> **getUsers**(`variables`): `Promise`\<`PaginatedResponse`\<`OmitUtils`\<`User`\>\>\> - -Retrieves a list of users with optional filters. - -###### Parameters - -▪ **variables**: `object` - -The parameters used to filter and paginate the user list. - -▪ **variables.first?**: `Maybe`\<`number`\> - -Optional. The number of items to return. - -▪ **variables.after?**: `Maybe`\<`string`\> - -Optional. The cursor after which to start fetching data. - -▪ **variables.before?**: `Maybe`\<`string`\> - -Optional. The cursor before which to start fetching data. - -▪ **variables.filters?**: `ParticipantsFilter`[] - -Optional. Array of filters to apply to the user query. - -###### Returns - -`Promise`\<`PaginatedResponse`\<`OmitUtils`\<`User`\>\>\> - -A `PaginatedResponse` containing a list of users without utility types. - -##### createUser() - -> **createUser**(`identifier`, `metadata`?): `Promise`\<`User`\> - -Creates a new user and sends it to the platform. - -###### Parameters - -▪ **identifier**: `string` - -The unique identifier for the user. This parameter is required. - -▪ **metadata?**: `Maybe`\<`Record`\<`string`, `any`\>\> - -Optional metadata for the user. This parameter is optional. - -###### Returns - -`Promise`\<`User`\> - -A promise that resolves with the newly created User object. - -##### updateUser() - -> **updateUser**(`id`, `identifier`?, `metadata`?): `Promise`\<`User`\> - -Updates an existing user's details in the platform. - -###### Parameters - -▪ **id**: `string` - -The unique identifier of the user to update. This parameter is required. - -▪ **identifier?**: `string` - -A new identifier for the user. This parameter is optional. - -▪ **metadata?**: `Maybe`\<`Record`\<`string`, `any`\>\> - -Additional metadata for the user. This parameter is optional. - -###### Returns - -`Promise`\<`User`\> - -A promise that resolves with the updated User object. - -##### getOrCreateUser() - -> **getOrCreateUser**(`identifier`, `metadata`?): `Promise`\<`string`\> - -Retrieves an existing user by their identifier or creates a new one if they do not exist. - -###### Parameters - -▪ **identifier**: `string` - -The unique identifier for the user. This parameter is required. - -▪ **metadata?**: `Maybe`\<`Record`\<`string`, `any`\>\> - -Additional metadata for the user. This parameter is optional. - -###### Returns - -`Promise`\<`string`\> - -The ID of the existing or newly created user. - -##### getUser() - -> **getUser**(`identifier`): `Promise`\<`Maybe`\<`User`\>\> - -Retrieves a user by their unique identifier. - -###### Parameters - -▪ **identifier**: `string` - -The unique identifier for the user. This parameter is required. - -###### Returns - -`Promise`\<`Maybe`\<`User`\>\> - -A `Promise` that resolves to a `User` object if found, otherwise `undefined`. - -##### deleteUser() - -> **deleteUser**(`id`): `Promise`\<`string`\> - -Deletes a single user by their unique identifier. - -###### Parameters - -▪ **id**: `string` - -The unique identifier of the user to be deleted. This parameter is required. - -###### Returns - -`Promise`\<`string`\> - -A `Promise` that resolves to the ID of the deleted user. - -##### getScores() - -> **getScores**(`variables`): `Promise`\<`PaginatedResponse`\<`OmitUtils`\<`Score`\>\>\> - -Get all scores connected to the platform. - -###### Parameters - -▪ **variables**: `object` - -The parameters for querying scores. - -▪ **variables.first?**: `Maybe`\<`number`\> - -Optional. The number of scores to retrieve. - -▪ **variables.after?**: `Maybe`\<`string`\> - -Optional. The cursor after which to start fetching scores. - -▪ **variables.before?**: `Maybe`\<`string`\> - -Optional. The cursor before which to start fetching scores. - -▪ **variables.filters?**: `ScoresFilter`[] - -Optional. Filters to apply to the score query. - -▪ **variables.orderBy?**: `ScoresOrderBy` - -Optional. The order in which to sort the scores. - -###### Returns - -`Promise`\<`PaginatedResponse`\<`OmitUtils`\<`Score`\>\>\> - -A `Promise` that resolves to a paginated response of scores, excluding certain utility fields. - -##### createScores() - -> **createScores**(`scores`): `Promise`\<`Score`[]\> - -Creates multiple scores in the database using the provided array of scores. -Each score in the array is transformed into a GraphQL mutation call. - -###### Parameters - -▪ **scores**: `Score`[] - -An array of `Score` objects to be created. - -###### Returns - -`Promise`\<`Score`[]\> - -A promise that resolves to an array of `Score` instances populated with the created scores' data. - -##### createScore() - -> **createScore**(`variables`): `Promise`\<`Score`\> - -Creates a new score in the database using the provided parameters. - -###### Parameters - -▪ **variables**: `OmitUtils`\<`Score`\> - -The score details to be used in the creation process. This includes: - -###### Returns - -`Promise`\<`Score`\> - -A new `Score` instance populated with the created score's data. - -##### updateScore() - -> **updateScore**(`id`, `updateParams`): `Promise`\<`Score`\> - -Updates an existing score in the database. - -###### Parameters - -▪ **id**: `string` - -The unique identifier of the score to update. (required) - -▪ **updateParams**: `object` - -The parameters to update in the score. (required) - -▪ **updateParams.comment?**: `Maybe`\<`string`\> - -A new or updated comment for the score. (optional) - -▪ **updateParams.value**: `number` - -The new value to set for the score. (required) - -###### Returns - -`Promise`\<`Score`\> - -A `Score` instance representing the updated score. - -##### deleteScore() - -> **deleteScore**(`id`): `Promise`\<`any`\> - -Deletes a single score from the database. - -###### Parameters - -▪ **id**: `string` - -The unique identifier of the score to delete. (required) - -###### Returns - -`Promise`\<`any`\> - -The ID of the deleted score. - -##### createDataset() - -> **createDataset**(`dataset`): `Promise`\<`Dataset`\> - -Creates a new dataset in the database. - -###### Parameters - -▪ **dataset**: `object` - -The dataset details to be created. - -▪ **dataset.name**: `string` - -The name of the dataset. (required) - -▪ **dataset.description?**: `Maybe`\<`string`\> - -The description of the dataset. (optional) - -▪ **dataset.metadata?**: `Maybe`\<`Record`\<`string`, `any`\>\> - -Additional metadata for the dataset as a key-value pair object. (optional) - -▪ **dataset.type?**: `DatasetType` - -The type of the dataset, defined by the DatasetType enum. (optional) - -###### Returns - -`Promise`\<`Dataset`\> - -A new Dataset instance populated with the created dataset's data. - -##### getDataset() - -> **getDataset**(`variables`): `Promise`\<`null` \| `Dataset`\> - -Retrieves a dataset based on provided ID or name. - -###### Parameters - -▪ **variables**: `object` - -An object containing optional `id` and `name` properties to specify which dataset to retrieve. - -▪ **variables.id?**: `string` - -▪ **variables.name?**: `string` - -###### Returns - -`Promise`\<`null` \| `Dataset`\> - -A `Dataset` instance populated with the retrieved dataset's data, or `null` if no data is found. - -##### updateDataset() - -> **updateDataset**(`id`, `dataset`): `Promise`\<`Dataset`\> - -Updates a dataset with new information. - -###### Parameters - -▪ **id**: `string` - -The unique identifier of the dataset to update. This parameter is required. - -▪ **dataset**: `object` - -An object containing the new dataset information. - -▪ **dataset.name?**: `Maybe`\<`string`\> - -The new name of the dataset. (optional) - -▪ **dataset.description?**: `Maybe`\<`string`\> - -The new description of the dataset. (optional) - -▪ **dataset.metadata?**: `Maybe`\<`Record`\<`string`, `any`\>\> - -Additional metadata for the dataset as a key-value pair object. (optional) - -###### Returns - -`Promise`\<`Dataset`\> - -A new `Dataset` instance populated with the updated dataset's data. - -##### deleteDataset() - -> **deleteDataset**(`id`): `Promise`\<`Dataset`\> - -Deletes a single dataset by its unique identifier. - -###### Parameters - -▪ **id**: `string` - -The unique identifier of the dataset to delete. This parameter is required. - -###### Returns - -`Promise`\<`Dataset`\> - -A new `Dataset` instance populated with the deleted dataset's data. - -##### createDatasetItem() - -> **createDatasetItem**(`datasetId`, `datasetItem`): `Promise`\<`DatasetItem`\> - -Creates a new item in a dataset. - -###### Parameters - -▪ **datasetId**: `string` - -The unique identifier of the dataset. This parameter is required. - -▪ **datasetItem**: `object` - -The data for the new dataset item. This parameter is required. - -▪ **datasetItem.input**: `Record`\<`string`, `any`\> - -The input data for the dataset item. This field is required. - -▪ **datasetItem.expectedOutput?**: `Maybe`\<`Record`\<`string`, `any`\>\> - -The expected output data for the dataset item. This field is optional. - -▪ **datasetItem.metadata?**: `Maybe`\<`Record`\<`string`, `any`\>\> - -Additional metadata for the dataset item. This field is optional. - -###### Returns - -`Promise`\<`DatasetItem`\> - -A new `DatasetItem` instance populated with the created dataset item's data. - -##### getDatasetItem() - -> **getDatasetItem**(`id`): `Promise`\<`DatasetItem`\> - -Retrieves a single item from a dataset by its unique identifier. - -###### Parameters - -▪ **id**: `string` - -The unique identifier of the dataset item. This parameter is required. - -###### Returns - -`Promise`\<`DatasetItem`\> - -A `DatasetItem` instance populated with the retrieved dataset item's data. - -##### deleteDatasetItem() - -> **deleteDatasetItem**(`id`): `Promise`\<`DatasetItem`\> - -Deletes a single item from a dataset by its unique identifier. - -###### Parameters - -▪ **id**: `string` - -The unique identifier of the dataset item to be deleted. This parameter is required. - -###### Returns - -`Promise`\<`DatasetItem`\> - -A `DatasetItem` instance populated with the data of the deleted dataset item. - -##### addStepToDataset() - -> **addStepToDataset**(`datasetId`, `stepId`, `metadata`?): `Promise`\<`DatasetItem`\> - -Adds a single step item to a dataset. - -###### Parameters - -▪ **datasetId**: `string` - -The unique identifier of the dataset. This parameter is required. - -▪ **stepId**: `string` - -The unique identifier of the step to be added. This parameter is required. - -▪ **metadata?**: `Maybe`\<`Record`\<`string`, `unknown`\>\> - -Additional metadata for the step as a JSON object. This parameter is optional. - -###### Returns - -`Promise`\<`DatasetItem`\> - -A `DatasetItem` instance populated with the data of the newly added step. - -##### addGenerationToDataset() - -> **addGenerationToDataset**(`datasetId`, `generationId`, `metadata`?): `Promise`\<`DatasetItem`\> - -Adds a generation item to a dataset. - -###### Parameters - -▪ **datasetId**: `string` - -The unique identifier of the dataset. This parameter is required. - -▪ **generationId**: `string` - -The unique identifier of the generation to be added. This parameter is required. - -▪ **metadata?**: `Maybe`\<`Record`\<`string`, `unknown`\>\> - -Additional metadata for the generation as a JSON object. This parameter is optional. - -###### Returns - -`Promise`\<`DatasetItem`\> - -A `DatasetItem` instance populated with the data of the newly added generation. - -##### addGenerationsToDataset() - -> **addGenerationsToDataset**(`datasetId`, `generationIds`): `Promise`\<`DatasetItem`[]\> - -###### Parameters - -▪ **datasetId**: `string` - -▪ **generationIds**: `string`[] - -###### Returns - -`Promise`\<`DatasetItem`[]\> - -##### createExperiment() - -> **createExperiment**(`datasetExperiment`): `Promise`\<`DatasetExperiment`\> - -###### Parameters - -▪ **datasetExperiment**: `object` - -▪ **datasetExperiment.name**: `string` - -▪ **datasetExperiment.datasetId**: `string` - -▪ **datasetExperiment.promptId?**: `string` - -▪ **datasetExperiment.params?**: `Record`\<`string`, `any`\> \| `Record`\<`string`, `any`\>[] - -###### Returns - -`Promise`\<`DatasetExperiment`\> - -##### createExperimentItem() - -> **createExperimentItem**(`__namedParameters`): `Promise`\<`DatasetExperimentItem`\> - -###### Parameters - -▪ **\_\_namedParameters**: `DatasetExperimentItem` - -###### Returns - -`Promise`\<`DatasetExperimentItem`\> - -##### createPromptLineage() - -> **createPromptLineage**(`name`, `description`?): `Promise`\<`any`\> - -Create a new prompt lineage. - -###### Parameters - -▪ **name**: `string` - -The name of the prompt lineage. This parameter is required. - -▪ **description?**: `string` - -A description for the prompt lineage. This parameter is optional. - -###### Returns - -`Promise`\<`any`\> - -The newly created prompt lineage object, or null if creation failed. - -##### createPrompt() - -> **createPrompt**(`name`, `templateMessages`, `settings`?): `Promise`\<`Prompt`\> - -Create a new prompt. - -###### Parameters - -▪ **name**: `string` - -The name of the prompt lineage. This parameter is required. - -▪ **templateMessages**: `IGenerationMessage`[] - -An array of template messages defining the prompt. This parameter is required. - -▪ **settings?**: `Maybe`\<`Record`\<`string`, `any`\>\> - -Optional settings for the prompt creation, provided as a record of string keys to any type of values. - -###### Returns - -`Promise`\<`Prompt`\> - -A new Prompt instance containing the created prompt data. - -##### getPrompt() - -> **getPrompt**(`name`, `version`?): `Promise`\<`null` \| `Prompt`\> - -Retrieves a prompt by its name and optionally by its version. - -###### Parameters - -▪ **name**: `string` - -The name of the prompt to retrieve. - -▪ **version?**: `number` - -The version number of the prompt (optional). - -###### Returns - -`Promise`\<`null` \| `Prompt`\> - -An instance of `Prompt` containing the prompt data, or `null` if not found. - -*** - -Generated using [typedoc-plugin-markdown](https://www.npmjs.com/package/typedoc-plugin-markdown) and [TypeDoc](https://typedoc.org/) diff --git a/src/api.ts b/src/api.ts index f2f2084..c1e1baa 100644 --- a/src/api.ts +++ b/src/api.ts @@ -5,6 +5,14 @@ import { ReadStream } from 'fs'; import { v4 as uuidv4 } from 'uuid'; import { LiteralClient } from '.'; +import { + Dataset, + DatasetExperiment, + DatasetExperimentItem, + DatasetItem, + DatasetType +} from './evaluation/dataset'; +import { Score, ScoreConstructor } from './evaluation/score'; import { GenerationsFilter, GenerationsOrderBy, @@ -16,32 +24,23 @@ import { ThreadsFilter, ThreadsOrderBy } from './filter'; +import { Attachment } from './observability/attachment'; import { Generation, IGenerationMessage, PersistedGeneration -} from './generation'; +} from './observability/generation'; +import { Step, StepType } from './observability/step'; +import { CleanThreadFields, Thread } from './observability/thread'; +import { Prompt } from './prompt-engineering/prompt'; import { - Attachment, - CleanThreadFields, - Dataset, - DatasetExperiment, - DatasetExperimentItem, - DatasetItem, - DatasetType, Environment, Maybe, OmitUtils, PaginatedResponse, - Prompt, - Score, - ScoreConstructor, - Step, - StepType, - Thread, User, Utils -} from './types'; +} from './utils'; // eslint-disable-next-line @typescript-eslint/no-var-requires const packageJson = require('../package.json'); @@ -892,6 +891,7 @@ export class API { metadata?: Maybe>; participantId?: Maybe; tags?: Maybe; + environment?: Maybe; }): Promise; /** @@ -910,7 +910,8 @@ export class API { name?: Maybe, metadata?: Maybe>, participantId?: Maybe, - tags?: Maybe + tags?: Maybe, + environment?: Maybe ): Promise; async upsertThread( @@ -918,7 +919,8 @@ export class API { name?: Maybe, metadata?: Maybe>, participantId?: Maybe, - tags?: Maybe + tags?: Maybe, + environment?: Maybe ): Promise { let threadId = threadIdOrOptions; if (typeof threadIdOrOptions === 'object') { @@ -927,6 +929,13 @@ export class API { metadata = threadIdOrOptions.metadata; participantId = threadIdOrOptions.participantId; tags = threadIdOrOptions.tags; + environment = threadIdOrOptions.environment; + } + + const originalEnvironment = this.environment; + + if (environment && environment !== originalEnvironment) { + this.environment = environment; } const query = ` @@ -958,6 +967,11 @@ export class API { }; const response = await this.makeGqlCall(query, variables); + + if (environment && environment !== originalEnvironment) { + this.environment = originalEnvironment; + } + return new Thread(this.client, response.data.upsertThread); } diff --git a/src/evaluation/dataset.ts b/src/evaluation/dataset.ts new file mode 100644 index 0000000..12f52c7 --- /dev/null +++ b/src/evaluation/dataset.ts @@ -0,0 +1,237 @@ +import { API } from '../api'; +import { Maybe, OmitUtils, Utils } from '../utils'; +import { ScoreConstructor } from './score'; + +export type DatasetType = 'key_value' | 'generation'; + +class DatasetFields extends Utils { + id!: string; + createdAt!: string; + name?: Maybe; + description?: Maybe; + metadata!: Record; + items!: Array>; + type?: DatasetType; +} + +export type DatasetConstructor = OmitUtils; + +export class Dataset extends DatasetFields { + api: API; + + /** + * Constructs a new Dataset instance. + * @param api - The API instance to interact with backend services. + * @param data - The initial data for the dataset. + */ + constructor(api: API, data: DatasetConstructor) { + super(); + this.api = api; + Object.assign(this, data); + if (!this.items) { + this.items = []; + } + if (!this.type) { + this.type = 'key_value'; + } + } + + /** + * Updates the dataset with new data. + * @param dataset - The dataset data to update. + * @returns The updated dataset instance. + */ + async update(dataset: { + name?: Maybe; + description?: Maybe; + metadata?: Maybe>; + }) { + const update_res = await this.api.updateDataset(this.id, dataset); + this.name = update_res.name; + this.description = update_res.description; + this.metadata = update_res.metadata; + } + + /** + * Deletes the dataset. + * @returns A promise that resolves when the dataset is deleted. + */ + async delete() { + return this.api.deleteDataset(this.id); + } + + /** + * Creates a new item in the dataset. + * @param datasetItem - The new item to be added to the dataset. + * @returns The newly created dataset item. + */ + async createItem(datasetItem: { + input: Record; + expectedOutput?: Maybe>; + metadata?: Maybe>; + }) { + const item = await this.api.createDatasetItem(this.id, datasetItem); + + this.items.push(item); + return item; + } + + /** + * Deletes an item from the dataset. + * @param id - The ID of the item to delete. + * @returns The deleted dataset item. + */ + async deleteItem(id: string) { + const deletedItem = await this.api.deleteDatasetItem(id); + if (this.items) { + this.items = this.items.filter((item) => item.id !== id); + } + return deletedItem; + } + + /** + * Creates a new experiment associated with the dataset. + * @param experiment - The experiment details including name, optional prompt ID, and parameters. + * @returns A new instance of DatasetExperiment containing the created experiment. + */ + async createExperiment(experiment: { + name: string; + promptId?: string; + params?: Record | Array>; + }) { + const datasetExperiment = await this.api.createExperiment({ + name: experiment.name, + datasetId: this.id, + promptId: experiment.promptId, + params: experiment.params + }); + return new DatasetExperiment(this.api, datasetExperiment); + } + + /** + * Adds a step to the dataset. + * @param stepId - The ID of the step to add. + * @param metadata - Optional metadata for the step. + * @returns The added dataset item. + * @throws Error if the dataset type is 'generation'. + */ + public async addStep( + stepId: string, + metadata?: Maybe> + ) { + if (this.type === 'generation') { + throw new Error('Cannot add steps to a generation dataset'); + } + const item = await this.api.addStepToDataset(this.id, stepId, metadata); + this.items.push(item); + return item; + } + + /** + * Adds a generation to the dataset. + * @param generationId - The ID of the generation to add. + * @param metadata - Optional metadata for the generation. + * @returns The added dataset item. + */ + public async addGeneration( + generationId: string, + metadata?: Maybe> + ) { + const item = await this.api.addGenerationToDataset( + this.id, + generationId, + metadata + ); + this.items.push(item); + return item; + } + + public async addGenerations(generationIds?: string[]) { + if (generationIds == undefined || generationIds?.length === 0) { + return []; + } + + const items = await this.api.addGenerationsToDataset( + this.id, + generationIds + ); + this.items = this.items.concat(items); + return items; + } +} + +export class DatasetItem extends Utils { + id!: string; + createdAt!: string; + datasetId!: string; + metadata!: Record; + input!: Record; + expectedOutput?: Maybe>; + intermediarySteps!: Array>; + + constructor(data: OmitUtils) { + super(); + Object.assign(this, data); + } +} + +class DatasetExperimentItemFields extends Utils { + id?: string; + datasetExperimentId!: string; + datasetItemId?: string; + experimentRunId?: string; + scores!: ScoreConstructor[]; + input?: Record; + output?: Record; +} + +export class DatasetExperiment extends Utils { + id!: string; + createdAt!: string; + name!: string; + datasetId?: string; + promptId?: string; + api: API; + params!: Record | Array>; + items!: DatasetExperimentItem[]; + + constructor(api: API, data: OmitUtils) { + super(); + this.api = api; + Object.assign(this, data); + if (!this.items) { + this.items = []; + } + } + + async log( + itemFields: Omit< + OmitUtils, + 'id' | 'datasetExperimentId' + > + ) { + const currentStore = this.api.client.store.getStore(); + const experimentRunId = currentStore?.currentExperimentRunId; + + const datasetExperimentItem = new DatasetExperimentItem({ + ...itemFields, + datasetExperimentId: this.id, + ...(experimentRunId && { experimentRunId }) + }); + + const item = await this.api.createExperimentItem(datasetExperimentItem); + + this.items.push(item); + return item; + } +} + +export type DatasetExperimentItemConstructor = + OmitUtils; + +export class DatasetExperimentItem extends DatasetExperimentItemFields { + constructor(data: DatasetExperimentItemConstructor) { + super(); + Object.assign(this, data); + } +} diff --git a/src/evaluation/experiment-run.ts b/src/evaluation/experiment-run.ts new file mode 100644 index 0000000..de92aa4 --- /dev/null +++ b/src/evaluation/experiment-run.ts @@ -0,0 +1,61 @@ +import { LiteralClient } from '..'; +import { API } from '../api'; +import { Step, StepConstructor } from '../observability/step'; + +/** + * Represents a step in a process or workflow, extending the fields and methods from StepFields. + */ +export class ExperimentRun extends Step { + api: API; + client: LiteralClient; + + /** + * Constructs a new ExperimentRun instance. + * @param api The API instance to be used for sending and managing steps. + * @param data The initial data for the step, excluding utility properties. + */ + constructor( + client: LiteralClient, + data: StepConstructor, + ignoreContext?: true + ) { + super(client, data, ignoreContext); + this.client = client; + this.api = client.api; + } + + async wrap( + cb: (step: Step) => Output | Promise, + updateStep?: + | Partial + | ((output: Output) => Partial) + | ((output: Output) => Promise>) + ) { + const originalEnvironment = this.api.environment; + this.api.environment = 'experiment'; + + const currentStore = this.client.store.getStore(); + const output: Output = await this.client.store.run( + { + currentThread: currentStore?.currentThread ?? null, + currentStep: this, + currentExperimentRunId: this.id ?? null + }, + async () => { + try { + const output = await super.wrap(cb, updateStep); + return output; + } finally { + // Clear the currentExperimentRunId after execution + const updatedStore = this.client.store.getStore(); + if (updatedStore) { + updatedStore.currentExperimentRunId = null; + } + } + } + ); + + this.api.environment = originalEnvironment; + return output; + } +} diff --git a/src/evaluation/score.ts b/src/evaluation/score.ts new file mode 100644 index 0000000..c510161 --- /dev/null +++ b/src/evaluation/score.ts @@ -0,0 +1,29 @@ +import { Maybe, OmitUtils, Utils } from '../utils'; + +export type ScoreType = 'HUMAN' | 'AI'; + +class ScoreFields extends Utils { + id?: Maybe; + stepId?: Maybe; + generationId?: Maybe; + datasetExperimentItemId?: Maybe; + name: string = 'user-feedback'; + value: number = 0; + type: ScoreType = 'AI'; + scorer?: Maybe; + comment?: Maybe; + tags?: Maybe; +} + +export type ScoreConstructor = OmitUtils; + +/** + * Represents a score entity with properties to track various aspects of scoring. + * It extends the `Utils` class for serialization capabilities. + */ +export class Score extends ScoreFields { + constructor(data: ScoreConstructor) { + super(); + Object.assign(this, data); + } +} diff --git a/src/index.ts b/src/index.ts index da2bf51..7b55e8c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,19 +1,15 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import { API } from './api'; +import { ExperimentRun } from './evaluation/experiment-run'; import instrumentation from './instrumentation'; +import { Step, StepConstructor } from './observability/step'; +import { Thread, ThreadConstructor } from './observability/thread'; import openai from './openai'; -import { - Environment, - ExperimentRun, - Step, - StepConstructor, - Thread, - ThreadConstructor -} from './types'; - -export * from './types'; -export * from './generation'; +import { Environment } from './utils'; + +export * from './utils'; +export * from './observability/generation'; export type * from './instrumentation'; @@ -31,6 +27,15 @@ export class LiteralClient { instrumentation: ReturnType; store: AsyncLocalStorage = storage; + /** + * Initialize a new Literal AI Client. + * @param options + * @param options.apiKey The API key to use for the Literal AI API. Defaults to the LITERAL_API_KEY environment variable. + * @param options.apiUrl The URL of the Literal AI API. Defaults to the LITERAL_API_URL environment variable. + * @param options.environment The environment to use for the Literal AI API. + * @param options.disabled If set to true, no call will be made to the Literal AI API. + * @returns A new LiteralClient instance. + */ constructor({ apiKey, apiUrl, @@ -55,18 +60,38 @@ export class LiteralClient { this.instrumentation = instrumentation(this); } + /** + * Creates a new thread without sending it to the Literal AI API. + * @param data Optional initial data for the thread. + * @returns A new thread instance. + */ thread(data?: ThreadConstructor) { return new Thread(this, data); } + /** + * Creates a new step without sending it to the Literal AI API. + * @param data Optional initial data for the step. + * @returns A new step instance. + */ step(data: StepConstructor) { return new Step(this, data); } + /** + * Creates a new step with the type set to 'run'. + * @param data Optional initial data for the step. + * @returns A new step instance. + */ run(data: Omit) { return this.step({ ...data, type: 'run' }); } + /** + * Creates a new Experiment Run. + * @param data Optional initial data for the step. + * @returns A new step instance. + */ experimentRun(data?: Omit) { return new ExperimentRun(this, { ...(data || {}), @@ -75,12 +100,20 @@ export class LiteralClient { }); } + /** + * Returns the current thread from the context or null if none. + * @returns The current thread, if any. + */ _currentThread(): Thread | null { const store = storage.getStore(); return store?.currentThread || null; } + /** + * Returns the current step from the context or null if none. + * @returns The current step, if any. + */ _currentStep(): Step | null { const store = storage.getStore(); diff --git a/src/instrumentation/langchain.ts b/src/instrumentation/langchain.ts index b5e95d2..0e3f398 100644 --- a/src/instrumentation/langchain.ts +++ b/src/instrumentation/langchain.ts @@ -30,7 +30,7 @@ import { ITool, LiteralClient } from '..'; -import { Step } from '../types'; +import { Step } from '../observability/step'; // @ts-expect-error Generics export class CustomChatPromptTemplate extends ChatPromptTemplate { diff --git a/src/instrumentation/llamaindex.ts b/src/instrumentation/llamaindex.ts index d2beb03..5c6e0d6 100644 --- a/src/instrumentation/llamaindex.ts +++ b/src/instrumentation/llamaindex.ts @@ -7,9 +7,9 @@ import { type IGenerationMessage, IImageUrlContent, ITextContent, - LiteralClient, - Thread + LiteralClient } from '..'; +import { Thread } from '../observability/thread'; const convertMessage = (message: ChatMessage): IGenerationMessage => { return { diff --git a/src/instrumentation/openai.ts b/src/instrumentation/openai.ts index f5edba6..0541fae 100644 --- a/src/instrumentation/openai.ts +++ b/src/instrumentation/openai.ts @@ -14,9 +14,9 @@ import { IGenerationMessage, LiteralClient, Maybe, - OpenAIGlobalOptions, - StepConstructor + OpenAIGlobalOptions } from '..'; +import { StepConstructor } from '../observability/step'; type OriginalFunction = ( body: any, diff --git a/src/instrumentation/vercel-sdk.ts b/src/instrumentation/vercel-sdk.ts index 58d53bb..c122abb 100644 --- a/src/instrumentation/vercel-sdk.ts +++ b/src/instrumentation/vercel-sdk.ts @@ -16,10 +16,10 @@ import { IGenerationMessage, ILLMSettings, ITool, - LiteralClient, - Step, - Thread + LiteralClient } from '..'; +import { Step } from '../observability/step'; +import { Thread } from '../observability/thread'; export type VercelLanguageModel = LanguageModel; diff --git a/src/observability/attachment.ts b/src/observability/attachment.ts new file mode 100644 index 0000000..9f6492e --- /dev/null +++ b/src/observability/attachment.ts @@ -0,0 +1,24 @@ +import { v4 as uuidv4 } from 'uuid'; + +import { Maybe, OmitUtils, Utils } from '../utils'; + +/** + * Represents an attachment with optional metadata, MIME type, and other properties. + * It extends the `Utils` class for serialization capabilities. + */ +export class Attachment extends Utils { + id?: Maybe; + metadata?: Maybe>; + mime?: Maybe; + name: Maybe; + objectKey?: Maybe; + url?: Maybe; + + constructor(data: OmitUtils) { + super(); + Object.assign(this, data); + if (!this.id) { + this.id = uuidv4(); + } + } +} diff --git a/src/generation.ts b/src/observability/generation.ts similarity index 97% rename from src/generation.ts rename to src/observability/generation.ts index 967527c..07f6a09 100644 --- a/src/generation.ts +++ b/src/observability/generation.ts @@ -1,4 +1,4 @@ -import { Maybe, OmitUtils, Utils } from './types'; +import { Maybe, OmitUtils, Utils } from '../utils'; export type GenerationMessageRole = | 'system' diff --git a/src/observability/step.ts b/src/observability/step.ts new file mode 100644 index 0000000..08fb1b2 --- /dev/null +++ b/src/observability/step.ts @@ -0,0 +1,211 @@ +import { v4 as uuidv4 } from 'uuid'; + +import { LiteralClient } from '..'; +import { API } from '../api'; +import { Score } from '../evaluation/score'; +import { Environment, Maybe, OmitUtils, Utils, isPlainObject } from '../utils'; +import { Attachment } from './attachment'; +import { Generation } from './generation'; + +export type StepType = + | 'assistant_message' + | 'embedding' + | 'llm' + | 'rerank' + | 'retrieval' + | 'run' + | 'system_message' + | 'tool' + | 'undefined' + | 'user_message'; + +class StepFields extends Utils { + name!: string; + type!: StepType; + threadId?: string; + createdAt?: Maybe; + startTime?: Maybe; + id?: Maybe; + environment?: Maybe; + error?: Maybe>; + input?: Maybe>; + output?: Maybe>; + metadata?: Maybe>; + tags?: Maybe; + parentId?: Maybe; + endTime?: Maybe; + generation?: Maybe; + scores?: Maybe; + attachments?: Maybe; +} + +export type StepConstructor = OmitUtils; + +/** + * Represents a step in a process or workflow, extending the fields and methods from StepFields. + */ +export class Step extends StepFields { + api: API; + client: LiteralClient; + + /** + * Constructs a new Step instance. + * @param api The API instance to be used for sending and managing steps. + * @param data The initial data for the step, excluding utility properties. + */ + constructor( + client: LiteralClient, + data: StepConstructor, + ignoreContext?: true + ) { + super(); + this.api = client.api; + this.client = client; + + Object.assign(this, data); + + // Automatically generate an ID if not provided. + if (!this.id) { + this.id = uuidv4(); + } + + if (ignoreContext) { + return; + } + + // Automatically assign parent thread & step if there are any in the store. + const store = this.client.store.getStore(); + + if (store?.currentThread) { + this.threadId = store.currentThread.id; + } + if (store?.currentStep) { + this.parentId = store.currentStep.id; + } + + // Set the creation and start time to the current time if not provided. + if (!this.createdAt) { + this.createdAt = new Date().toISOString(); + } + if (!this.startTime) { + this.startTime = new Date().toISOString(); + } + + // If the step is a message, set the end time to the start time. + if (this.isMessage()) { + this.endTime = this.startTime; + } + } + + /** + * Serializes the step instance, converting complex objects to strings as necessary. + * @returns A serialized representation of the step. + */ + serialize() { + const serialized = super.serialize(); + + // Convert error objects to JSON string if present. + if (typeof serialized.error === 'object' && serialized.error !== null) { + serialized.error = JSON.stringify(serialized.error); + } + return serialized; + } + + /** + * Determines if the step is a type of message. + * @returns True if the step is a user, assistant, or system message. + */ + isMessage() { + return ( + this.type === 'user_message' || + this.type === 'assistant_message' || + this.type === 'system_message' + ); + } + + /** + * Creates a new step instance linked to the current step as a parent. + * @param data The data for the new step, excluding the threadId which is inherited. + * @returns A new Step instance. + */ + step(data: Omit) { + return new Step(this.client, { + ...data, + threadId: this.threadId, + parentId: this.id + }); + } + + /** + * Sends the step to the API, handling disabled state and setting the end time if not already set. + * @returns The current Step instance after potentially sending to the API. + */ + async send() { + if (this.api.disabled) { + return this; + } + if (!this.endTime) { + this.endTime = new Date().toISOString(); + } + await this.api.sendSteps([this]); + return this; + } + + /** + * Sends the step to the API, handling disabled state and setting the end time if not already set. + * @param cb The callback function to run within the context of the step. + * @param updateStep Optional update function to modify the step after the callback. + * @returns The output of the wrapped callback function. + */ + async wrap( + cb: (step: Step) => Output | Promise, + updateStep?: + | Partial + | ((output: Output) => Partial) + | ((output: Output) => Promise>) + ) { + const startTime = new Date(); + this.startTime = startTime.toISOString(); + const currentStore = this.client.store.getStore(); + + const output = await this.client.store.run( + { + currentThread: currentStore?.currentThread ?? null, + currentExperimentRunId: currentStore?.currentExperimentRunId ?? null, + currentStep: this + }, + () => cb(this) + ); + + this.output = isPlainObject(output) ? output : { output }; + this.endTime = new Date().toISOString(); + + if (updateStep) { + const updatedStep = + typeof updateStep === 'function' + ? await updateStep(output) + : updateStep; + + this.name = updatedStep.name ?? this.name; + this.type = updatedStep.type ?? this.type; + this.threadId = updatedStep.threadId ?? this.threadId; + this.createdAt = updatedStep.createdAt ?? this.createdAt; + this.startTime = updatedStep.startTime ?? this.startTime; + this.error = updatedStep.error ?? this.error; + this.input = updatedStep.input ?? this.input; + this.output = updatedStep.output ?? this.output; + this.metadata = updatedStep.metadata ?? this.metadata; + this.tags = updatedStep.tags ?? this.tags; + this.parentId = updatedStep.parentId ?? this.parentId; + this.endTime = updatedStep.endTime ?? this.endTime; + this.generation = updatedStep.generation ?? this.generation; + this.scores = updatedStep.scores ?? this.scores; + this.attachments = updatedStep.attachments ?? this.attachments; + this.environment = updatedStep.environment ?? this.environment; + } + + this.send().catch(console.error); + + return output; + } +} diff --git a/src/observability/thread.ts b/src/observability/thread.ts new file mode 100644 index 0000000..68b9ce5 --- /dev/null +++ b/src/observability/thread.ts @@ -0,0 +1,134 @@ +import { v4 as uuidv4 } from 'uuid'; + +import { LiteralClient } from '..'; +import { API } from '../api'; +import { Environment, Maybe, OmitUtils, Utils } from '../utils'; +import { Step, StepConstructor } from './step'; + +/** + * Defines the structure for thread fields, inheriting utilities for serialization. + * This class encapsulates common properties used to describe a thread in various environments. + */ +class ThreadFields extends Utils { + id!: string; + participantId?: Maybe; + environment?: Maybe; + name?: Maybe; + metadata?: Maybe>; + steps?: Maybe; + tags?: Maybe; +} + +export type CleanThreadFields = OmitUtils; +export type ThreadConstructor = Omit & + Partial>; + +/** + * Represents a thread in the system, extending the properties and methods from `ThreadFields`. + * This class manages thread-specific operations such as creation and updates via the API. + */ +export class Thread extends ThreadFields { + api: API; + client: LiteralClient; + + /** + * Constructs a new Thread instance, with a generated ID if none is provided. + * @param api - The API instance to interact with backend services. + * @param data - Optional initial data for the thread. + */ + constructor(client: LiteralClient, data?: ThreadConstructor) { + super(); + this.api = client.api; + this.client = client; + + if (!data) { + data = { id: uuidv4() }; + } else if (!data.id) { + data.id = uuidv4(); + } + + Object.assign(this, data); + } + + /** + * Creates a new step associated with this thread. + * @param data - The data for the new step, excluding the thread ID. + * @returns A new Step instance linked to this thread. + */ + step(data: Omit) { + return new Step(this.client, { + ...data, + threadId: this.id + }); + } + + /** + * Creates a new Run step associated with this thread. + * @param data - The data for the new step, excluding the thread ID and the type + * @returns A new Step instance linked to this thread. + */ + run(data: Omit) { + return this.step({ ...data, type: 'run' }); + } + + /** + * Upserts the thread data to the backend, creating or updating as necessary. + * @returns The updated Thread instance. + */ + async upsert() { + if (this.api.disabled) { + return this; + } + await this.api.upsertThread({ + threadId: this.id, + name: this.name, + environment: this.environment, + metadata: this.metadata, + participantId: this.participantId, + tags: this.tags + }); + return this; + } + + /** + * Sends the thread to the API, handling disabled state and setting the end time if not already set. + * @param cb The callback function to run within the context of the thread. + * @param updateThread Optional update function to modify the thread after the callback. + * @returns The output of the wrapped callback function. + */ + async wrap( + cb: (thread: Thread) => Output | Promise, + updateThread?: + | ThreadConstructor + | ((output: Output) => ThreadConstructor) + | ((output: Output) => Promise) + ) { + const currentStore = this.client.store.getStore(); + + const output = await this.client.store.run( + { + currentThread: this, + currentExperimentRunId: currentStore?.currentExperimentRunId ?? null, + currentStep: null + }, + () => cb(this) + ); + + if (updateThread) { + const updatedThread = + typeof updateThread === 'function' + ? await updateThread(output) + : updateThread; + + this.participantId = updatedThread.participantId ?? this.participantId; + this.environment = updatedThread.environment ?? this.environment; + this.name = updatedThread.name ?? this.name; + this.metadata = updatedThread.metadata ?? this.metadata; + this.tags = updatedThread.tags ?? this.tags; + } + + this.upsert().catch(console.error); + + return output; + } +} diff --git a/src/openai.ts b/src/openai.ts index 48c5bec..fd7b2b3 100644 --- a/src/openai.ts +++ b/src/openai.ts @@ -9,8 +9,9 @@ import type { import { v5 as uuidv5 } from 'uuid'; import { LiteralClient } from '.'; -import { ChatGeneration } from './generation'; -import { Attachment, User } from './types'; +import { Attachment } from './observability/attachment'; +import { ChatGeneration } from './observability/generation'; +import { User } from './utils'; class OpenAIAssistantSyncer { private NAMESPACE_UUID = '1b671a64-40d5-491e-99b0-da01ff1f3341'; diff --git a/src/prompt-engineering/prompt.ts b/src/prompt-engineering/prompt.ts new file mode 100644 index 0000000..e786e41 --- /dev/null +++ b/src/prompt-engineering/prompt.ts @@ -0,0 +1,141 @@ +import mustache from 'mustache'; +import { + ChatCompletionMessageParam, + ChatCompletionTool +} from 'openai/resources'; + +import { API } from '../api'; +import { DatasetItem } from '../evaluation/dataset'; +import { CustomChatPromptTemplate } from '../instrumentation/langchain'; +import { + GenerationType, + IGenerationMessage +} from '../observability/generation'; +import { Maybe, OmitUtils, Utils } from '../utils'; + +export interface IPromptVariableDefinition { + name: string; + language: 'json' | 'plaintext'; +} + +export interface IProviderSettings { + provider: string; + model: string; + frequency_penalty: number; + max_tokens: number; + presence_penalty: number; + stop?: string[]; + temperature: number; + top_p: number; +} + +class PromptFields extends Utils { + id!: string; + type!: GenerationType; + createdAt!: string; + name!: string; + version!: number; + versionDesc?: Maybe; + metadata!: Record; + items!: Array>; + variablesDefaultValues?: Maybe>; + templateMessages!: IGenerationMessage[]; + tools?: ChatCompletionTool[]; + provider!: string; + settings!: IProviderSettings; + variables!: IPromptVariableDefinition[]; +} + +export type PromptConstructor = OmitUtils; + +export class Prompt extends PromptFields { + api: API; + + /** + * Constructs a new Prompt instance. + * @param api - The API instance to interact with backend services. + * @param data - The initial data for the prompt. + */ + constructor(api: API, data: PromptConstructor) { + super(); + this.api = api; + Object.assign(this, data); + if (this.tools?.length === 0) { + this.tools = undefined; + } + } + + /** + * Formats the prompt's template messages with the given variables. + * @param variables - Optional variables to resolve in the template messages. + * @returns An array of formatted chat completion messages. + */ + formatMessages( + variables?: Record + ): ChatCompletionMessageParam[] { + const variablesWithDefault = { + ...(this.variablesDefaultValues || {}), + ...variables + }; + + const promptId = this.id; + + return this.templateMessages.map( + ({ uuid, templated, ...templateMessage }) => { + const formattedMessage = { + ...templateMessage + } as ChatCompletionMessageParam; + // @ts-expect-error Hacky way to add metadata to the formatted message + formattedMessage.literalMetadata = () => { + return { + uuid: uuid, + promptId, + variables: variablesWithDefault + }; + }; + if (Array.isArray(formattedMessage.content)) { + formattedMessage.content = formattedMessage.content.map((content) => { + if (content.type === 'text') { + return { + ...content, + text: mustache.render(content.text, variablesWithDefault) + }; + } + return content; + }); + } else if (typeof formattedMessage.content === 'string') { + formattedMessage.content = mustache.render( + formattedMessage.content, + variablesWithDefault + ); + } + + return formattedMessage; + } + ); + } + + /** + * @deprecated Please use `formatMessages` instead. + */ + format(variables?: Record): ChatCompletionMessageParam[] { + return this.formatMessages(variables); + } + + /** + * Converts the prompt's template messages into a Langchain chat prompt template. + * @returns A custom chat prompt template configured with the prompt's data. + */ + toLangchainChatPromptTemplate() { + const lcMessages: [string, string][] = this.templateMessages.map((m) => [ + m.role, + m.content as string + ]); + const chatTemplate = CustomChatPromptTemplate.fromMessages(lcMessages); + chatTemplate.variablesDefaultValues = this.variablesDefaultValues; + chatTemplate.literalTemplateMessages = this.templateMessages; + chatTemplate.promptId = this.id; + + return chatTemplate; + } +} diff --git a/src/types.ts b/src/types.ts deleted file mode 100644 index 6065566..0000000 --- a/src/types.ts +++ /dev/null @@ -1,889 +0,0 @@ -import mustache from 'mustache'; -import { - ChatCompletionMessageParam, - ChatCompletionTool -} from 'openai/resources'; -import { v4 as uuidv4 } from 'uuid'; - -import { LiteralClient } from '.'; -import { API } from './api'; -import { Generation, GenerationType, IGenerationMessage } from './generation'; -import { CustomChatPromptTemplate } from './instrumentation/langchain'; - -export type Maybe = T | null | undefined; - -export type OmitUtils = Omit; - -export type Environment = 'dev' | 'staging' | 'prod' | 'experiment'; - -export type PageInfo = { - hasNextPage: boolean; - startCursor: string; - endCursor: string; -}; - -export type PaginatedResponse = { - data: T[]; - pageInfo: PageInfo; -}; - -function isPlainObject(value: unknown): value is Record { - if (typeof value !== 'object' || value === null) { - return false; - } - - const prototype = Object.getPrototypeOf(value); - return prototype === null || prototype === Object.prototype; -} - -/** - * Represents a utility class with serialization capabilities. - */ -export class Utils { - /** - * Serializes the properties of the current instance into a dictionary, excluding the 'api' property. - * It handles nested objects that also implement a serialize method. - * - * @returns A dictionary representing the serialized properties of the object. - */ - serialize(): any { - const dict: any = {}; - Object.keys(this as any).forEach((key) => { - if (['api', 'client'].includes(key)) { - return; - } - if ((this as any)[key] !== undefined) { - if (Array.isArray((this as any)[key])) { - dict[key] = (this as any)[key].map((item: any) => { - if ( - item instanceof Object && - typeof item.serialize === 'function' - ) { - return item.serialize(); - } else { - return item; - } - }); - } else if ( - (this as any)[key] instanceof Object && - typeof (this as any)[key].serialize === 'function' - ) { - dict[key] = (this as any)[key].serialize(); - } else { - dict[key] = (this as any)[key]; - } - } - }); - return dict; - } -} - -export type ScoreType = 'HUMAN' | 'AI'; - -class ScoreFields extends Utils { - id?: Maybe; - stepId?: Maybe; - generationId?: Maybe; - datasetExperimentItemId?: Maybe; - name: string = 'user-feedback'; - value: number = 0; - type: ScoreType = 'AI'; - scorer?: Maybe; - comment?: Maybe; - tags?: Maybe; -} - -export type ScoreConstructor = OmitUtils; - -/** - * Represents a score entity with properties to track various aspects of scoring. - * It extends the `Utils` class for serialization capabilities. - */ -export class Score extends ScoreFields { - constructor(data: ScoreConstructor) { - super(); - Object.assign(this, data); - } -} - -/** - * Represents an attachment with optional metadata, MIME type, and other properties. - * It extends the `Utils` class for serialization capabilities. - */ -export class Attachment extends Utils { - id?: Maybe; - metadata?: Maybe>; - mime?: Maybe; - name: Maybe; - objectKey?: Maybe; - url?: Maybe; - - constructor(data: OmitUtils) { - super(); - Object.assign(this, data); - if (!this.id) { - this.id = uuidv4(); - } - } -} - -/** - * Defines the structure for thread fields, inheriting utilities for serialization. - * This class encapsulates common properties used to describe a thread in various environments. - */ -class ThreadFields extends Utils { - id!: string; - participantId?: Maybe; - environment?: Maybe; - name?: Maybe; - metadata?: Maybe>; - steps?: Maybe; - tags?: Maybe; -} - -export type CleanThreadFields = OmitUtils; -export type ThreadConstructor = Omit & - Partial>; - -/** - * Represents a thread in the system, extending the properties and methods from `ThreadFields`. - * This class manages thread-specific operations such as creation and updates via the API. - */ -export class Thread extends ThreadFields { - api: API; - client: LiteralClient; - - /** - * Constructs a new Thread instance. - * @param api - The API instance to interact with backend services. - * @param data - Optional initial data for the thread, with an auto-generated ID if not provided. - */ - constructor(client: LiteralClient, data?: ThreadConstructor) { - super(); - this.api = client.api; - this.client = client; - - if (!data) { - data = { id: uuidv4() }; - } else if (!data.id) { - data.id = uuidv4(); - } - - Object.assign(this, data); - } - - /** - * Creates a new step associated with this thread. - * @param data - The data for the new step, excluding the thread ID. - * @returns A new Step instance linked to this thread. - */ - step(data: Omit) { - return new Step(this.client, { - ...data, - threadId: this.id - }); - } - - /** - * Creates a new Run step associated with this thread. - * @param data - The data for the new step, excluding the thread ID and the type - * @returns A new Step instance linked to this thread. - */ - run(data: Omit) { - return this.step({ ...data, type: 'run' }); - } - - /** - * Upserts the thread data to the backend, creating or updating as necessary. - * @returns The updated Thread instance. - */ - async upsert() { - if (this.api.disabled) { - return this; - } - await this.api.upsertThread({ - threadId: this.id, - name: this.name, - metadata: this.metadata, - participantId: this.participantId, - tags: this.tags - }); - return this; - } - - /** - * Sends the thread to the API, handling disabled state and setting the end time if not already set. - * @param cb The callback function to run within the context of the thread. - * @param updateThread Optional update function to modify the thread after the callback. - * @returns The output of the wrapped callback function. - */ - async wrap( - cb: (thread: Thread) => Output | Promise, - updateThread?: - | ThreadConstructor - | ((output: Output) => ThreadConstructor) - | ((output: Output) => Promise) - ) { - const currentStore = this.client.store.getStore(); - - const output = await this.client.store.run( - { - currentThread: this, - currentExperimentRunId: currentStore?.currentExperimentRunId ?? null, - currentStep: null - }, - () => cb(this) - ); - - if (updateThread) { - const updatedThread = - typeof updateThread === 'function' - ? await updateThread(output) - : updateThread; - - this.participantId = updatedThread.participantId ?? this.participantId; - this.environment = updatedThread.environment ?? this.environment; - this.name = updatedThread.name ?? this.name; - this.metadata = updatedThread.metadata ?? this.metadata; - this.tags = updatedThread.tags ?? this.tags; - } - - this.upsert().catch(console.error); - - return output; - } -} - -export type StepType = - | 'assistant_message' - | 'embedding' - | 'llm' - | 'rerank' - | 'retrieval' - | 'run' - | 'system_message' - | 'tool' - | 'undefined' - | 'user_message'; - -class StepFields extends Utils { - name!: string; - type!: StepType; - threadId?: string; - createdAt?: Maybe; - startTime?: Maybe; - id?: Maybe; - environment?: Maybe; - error?: Maybe>; - input?: Maybe>; - output?: Maybe>; - metadata?: Maybe>; - tags?: Maybe; - parentId?: Maybe; - endTime?: Maybe; - generation?: Maybe; - scores?: Maybe; - attachments?: Maybe; -} - -export type StepConstructor = OmitUtils; - -/** - * Represents a step in a process or workflow, extending the fields and methods from StepFields. - */ -export class Step extends StepFields { - api: API; - client: LiteralClient; - - /** - * Constructs a new Step instance. - * @param api The API instance to be used for sending and managing steps. - * @param data The initial data for the step, excluding utility properties. - */ - constructor( - client: LiteralClient, - data: StepConstructor, - ignoreContext?: true - ) { - super(); - this.api = client.api; - this.client = client; - - Object.assign(this, data); - - // Automatically generate an ID if not provided. - if (!this.id) { - this.id = uuidv4(); - } - - if (ignoreContext) { - return; - } - - // Automatically assign parent thread & step if there are any in the store. - const store = this.client.store.getStore(); - - if (store?.currentThread) { - this.threadId = store.currentThread.id; - } - if (store?.currentStep) { - this.parentId = store.currentStep.id; - } - - // Set the creation and start time to the current time if not provided. - if (!this.createdAt) { - this.createdAt = new Date().toISOString(); - } - if (!this.startTime) { - this.startTime = new Date().toISOString(); - } - - // If the step is a message, set the end time to the start time. - if (this.isMessage()) { - this.endTime = this.startTime; - } - } - - /** - * Serializes the step instance, converting complex objects to strings as necessary. - * @returns A serialized representation of the step. - */ - serialize() { - const serialized = super.serialize(); - - // Convert error objects to JSON string if present. - if (typeof serialized.error === 'object' && serialized.error !== null) { - serialized.error = JSON.stringify(serialized.error); - } - return serialized; - } - - /** - * Determines if the step is a type of message. - * @returns True if the step is a user, assistant, or system message. - */ - isMessage() { - return ( - this.type === 'user_message' || - this.type === 'assistant_message' || - this.type === 'system_message' - ); - } - - /** - * Creates a new step instance linked to the current step as a parent. - * @param data The data for the new step, excluding the threadId which is inherited. - * @returns A new Step instance. - */ - step(data: Omit) { - return new Step(this.client, { - ...data, - threadId: this.threadId, - parentId: this.id - }); - } - - /** - * Sends the step to the API, handling disabled state and setting the end time if not already set. - * @returns The current Step instance after potentially sending to the API. - */ - async send() { - if (this.api.disabled) { - return this; - } - if (!this.endTime) { - this.endTime = new Date().toISOString(); - } - await this.api.sendSteps([this]); - return this; - } - - /** - * Sends the step to the API, handling disabled state and setting the end time if not already set. - * @param cb The callback function to run within the context of the step. - * @param updateStep Optional update function to modify the step after the callback. - * @returns The output of the wrapped callback function. - */ - async wrap( - cb: (step: Step) => Output | Promise, - updateStep?: - | Partial - | ((output: Output) => Partial) - | ((output: Output) => Promise>) - ) { - const startTime = new Date(); - this.startTime = startTime.toISOString(); - const currentStore = this.client.store.getStore(); - - const output = await this.client.store.run( - { - currentThread: currentStore?.currentThread ?? null, - currentExperimentRunId: currentStore?.currentExperimentRunId ?? null, - currentStep: this - }, - () => cb(this) - ); - - this.output = isPlainObject(output) ? output : { output }; - this.endTime = new Date().toISOString(); - - if (updateStep) { - const updatedStep = - typeof updateStep === 'function' - ? await updateStep(output) - : updateStep; - - this.name = updatedStep.name ?? this.name; - this.type = updatedStep.type ?? this.type; - this.threadId = updatedStep.threadId ?? this.threadId; - this.createdAt = updatedStep.createdAt ?? this.createdAt; - this.startTime = updatedStep.startTime ?? this.startTime; - this.error = updatedStep.error ?? this.error; - this.input = updatedStep.input ?? this.input; - this.output = updatedStep.output ?? this.output; - this.metadata = updatedStep.metadata ?? this.metadata; - this.tags = updatedStep.tags ?? this.tags; - this.parentId = updatedStep.parentId ?? this.parentId; - this.endTime = updatedStep.endTime ?? this.endTime; - this.generation = updatedStep.generation ?? this.generation; - this.scores = updatedStep.scores ?? this.scores; - this.attachments = updatedStep.attachments ?? this.attachments; - } - - this.send().catch(console.error); - - return output; - } -} - -/** - * Represents a step in a process or workflow, extending the fields and methods from StepFields. - */ -export class ExperimentRun extends Step { - api: API; - client: LiteralClient; - - /** - * Constructs a new ExperimentRun instance. - * @param api The API instance to be used for sending and managing steps. - * @param data The initial data for the step, excluding utility properties. - */ - constructor( - client: LiteralClient, - data: StepConstructor, - ignoreContext?: true - ) { - super(client, data, ignoreContext); - this.client = client; - this.api = client.api; - } - - async wrap( - cb: (step: Step) => Output | Promise, - updateStep?: - | Partial - | ((output: Output) => Partial) - | ((output: Output) => Promise>) - ) { - const originalEnvironment = this.api.environment; - this.api.environment = 'experiment'; - - const currentStore = this.client.store.getStore(); - const output: Output = await this.client.store.run( - { - currentThread: currentStore?.currentThread ?? null, - currentStep: this, - currentExperimentRunId: this.id ?? null - }, - async () => { - try { - const output = await super.wrap(cb, updateStep); - return output; - } finally { - // Clear the currentExperimentRunId after execution - const updatedStore = this.client.store.getStore(); - if (updatedStore) { - updatedStore.currentExperimentRunId = null; - } - } - } - ); - - this.api.environment = originalEnvironment; - return output; - } -} - -/** - * Represents a user with optional metadata and identifier. - */ -export class User extends Utils { - id?: Maybe; - identifier!: string; - metadata?: Maybe>; - - constructor(data: OmitUtils) { - super(); - Object.assign(this, data); - } -} - -export type DatasetType = 'key_value' | 'generation'; - -class DatasetFields extends Utils { - id!: string; - createdAt!: string; - name?: Maybe; - description?: Maybe; - metadata!: Record; - items!: Array>; - type?: DatasetType; -} - -export type DatasetConstructor = OmitUtils; - -export class Dataset extends DatasetFields { - api: API; - - /** - * Constructs a new Dataset instance. - * @param api - The API instance to interact with backend services. - * @param data - The initial data for the dataset. - */ - constructor(api: API, data: DatasetConstructor) { - super(); - this.api = api; - Object.assign(this, data); - if (!this.items) { - this.items = []; - } - if (!this.type) { - this.type = 'key_value'; - } - } - - /** - * Updates the dataset with new data. - * @param dataset - The dataset data to update. - * @returns The updated dataset instance. - */ - async update(dataset: { - name?: Maybe; - description?: Maybe; - metadata?: Maybe>; - }) { - const update_res = await this.api.updateDataset(this.id, dataset); - this.name = update_res.name; - this.description = update_res.description; - this.metadata = update_res.metadata; - } - - /** - * Deletes the dataset. - * @returns A promise that resolves when the dataset is deleted. - */ - async delete() { - return this.api.deleteDataset(this.id); - } - - /** - * Creates a new item in the dataset. - * @param datasetItem - The new item to be added to the dataset. - * @returns The newly created dataset item. - */ - async createItem(datasetItem: { - input: Record; - expectedOutput?: Maybe>; - metadata?: Maybe>; - }) { - const item = await this.api.createDatasetItem(this.id, datasetItem); - - this.items.push(item); - return item; - } - - /** - * Deletes an item from the dataset. - * @param id - The ID of the item to delete. - * @returns The deleted dataset item. - */ - async deleteItem(id: string) { - const deletedItem = await this.api.deleteDatasetItem(id); - if (this.items) { - this.items = this.items.filter((item) => item.id !== id); - } - return deletedItem; - } - - /** - * Creates a new experiment associated with the dataset. - * @param experiment - The experiment details including name, optional prompt ID, and parameters. - * @returns A new instance of DatasetExperiment containing the created experiment. - */ - async createExperiment(experiment: { - name: string; - promptId?: string; - params?: Record | Array>; - }) { - const datasetExperiment = await this.api.createExperiment({ - name: experiment.name, - datasetId: this.id, - promptId: experiment.promptId, - params: experiment.params - }); - return new DatasetExperiment(this.api, datasetExperiment); - } - - /** - * Adds a step to the dataset. - * @param stepId - The ID of the step to add. - * @param metadata - Optional metadata for the step. - * @returns The added dataset item. - * @throws Error if the dataset type is 'generation'. - */ - public async addStep( - stepId: string, - metadata?: Maybe> - ) { - if (this.type === 'generation') { - throw new Error('Cannot add steps to a generation dataset'); - } - const item = await this.api.addStepToDataset(this.id, stepId, metadata); - this.items.push(item); - return item; - } - - /** - * Adds a generation to the dataset. - * @param generationId - The ID of the generation to add. - * @param metadata - Optional metadata for the generation. - * @returns The added dataset item. - */ - public async addGeneration( - generationId: string, - metadata?: Maybe> - ) { - const item = await this.api.addGenerationToDataset( - this.id, - generationId, - metadata - ); - this.items.push(item); - return item; - } - - public async addGenerations(generationIds?: string[]) { - if (generationIds == undefined || generationIds?.length === 0) { - return []; - } - - const items = await this.api.addGenerationsToDataset( - this.id, - generationIds - ); - this.items = this.items.concat(items); - return items; - } -} - -export class DatasetItem extends Utils { - id!: string; - createdAt!: string; - datasetId!: string; - metadata!: Record; - input!: Record; - expectedOutput?: Maybe>; - intermediarySteps!: Array>; - - constructor(data: OmitUtils) { - super(); - Object.assign(this, data); - } -} - -class DatasetExperimentItemFields extends Utils { - id?: string; - datasetExperimentId!: string; - datasetItemId?: string; - experimentRunId?: string; - scores!: ScoreConstructor[]; - input?: Record; - output?: Record; -} - -export class DatasetExperiment extends Utils { - id!: string; - createdAt!: string; - name!: string; - datasetId?: string; - promptId?: string; - api: API; - params!: Record | Array>; - items!: DatasetExperimentItem[]; - - constructor(api: API, data: OmitUtils) { - super(); - this.api = api; - Object.assign(this, data); - if (!this.items) { - this.items = []; - } - } - - async log( - itemFields: Omit< - OmitUtils, - 'id' | 'datasetExperimentId' - > - ) { - const currentStore = this.api.client.store.getStore(); - const experimentRunId = currentStore?.currentExperimentRunId; - - const datasetExperimentItem = new DatasetExperimentItem({ - ...itemFields, - datasetExperimentId: this.id, - ...(experimentRunId && { experimentRunId }) - }); - - const item = await this.api.createExperimentItem(datasetExperimentItem); - - this.items.push(item); - return item; - } -} - -export type DatasetExperimentItemConstructor = - OmitUtils; - -export class DatasetExperimentItem extends DatasetExperimentItemFields { - constructor(data: DatasetExperimentItemConstructor) { - super(); - Object.assign(this, data); - } -} -export interface IPromptVariableDefinition { - name: string; - language: 'json' | 'plaintext'; -} - -export interface IProviderSettings { - provider: string; - model: string; - frequency_penalty: number; - max_tokens: number; - presence_penalty: number; - stop?: string[]; - temperature: number; - top_p: number; -} - -class PromptFields extends Utils { - id!: string; - type!: GenerationType; - createdAt!: string; - name!: string; - version!: number; - versionDesc?: Maybe; - metadata!: Record; - items!: Array>; - variablesDefaultValues?: Maybe>; - templateMessages!: IGenerationMessage[]; - tools?: ChatCompletionTool[]; - provider!: string; - settings!: IProviderSettings; - variables!: IPromptVariableDefinition[]; -} - -export type PromptConstructor = OmitUtils; - -export class Prompt extends PromptFields { - api: API; - - /** - * Constructs a new Prompt instance. - * @param api - The API instance to interact with backend services. - * @param data - The initial data for the prompt. - */ - constructor(api: API, data: PromptConstructor) { - super(); - this.api = api; - Object.assign(this, data); - if (this.tools?.length === 0) { - this.tools = undefined; - } - } - - /** - * Formats the prompt's template messages with the given variables. - * @param variables - Optional variables to resolve in the template messages. - * @returns An array of formatted chat completion messages. - */ - formatMessages( - variables?: Record - ): ChatCompletionMessageParam[] { - const variablesWithDefault = { - ...(this.variablesDefaultValues || {}), - ...variables - }; - - const promptId = this.id; - - return this.templateMessages.map( - ({ uuid, templated, ...templateMessage }) => { - const formattedMessage = { - ...templateMessage - } as ChatCompletionMessageParam; - // @ts-expect-error Hacky way to add metadata to the formatted message - formattedMessage.literalMetadata = () => { - return { - uuid: uuid, - promptId, - variables: variablesWithDefault - }; - }; - if (Array.isArray(formattedMessage.content)) { - formattedMessage.content = formattedMessage.content.map((content) => { - if (content.type === 'text') { - return { - ...content, - text: mustache.render(content.text, variablesWithDefault) - }; - } - return content; - }); - } else if (typeof formattedMessage.content === 'string') { - formattedMessage.content = mustache.render( - formattedMessage.content, - variablesWithDefault - ); - } - - return formattedMessage; - } - ); - } - - /** - * @deprecated Please use `formatMessages` instead. - */ - format(variables?: Record): ChatCompletionMessageParam[] { - return this.formatMessages(variables); - } - - /** - * Converts the prompt's template messages into a Langchain chat prompt template. - * @returns A custom chat prompt template configured with the prompt's data. - */ - toLangchainChatPromptTemplate() { - const lcMessages: [string, string][] = this.templateMessages.map((m) => [ - m.role, - m.content as string - ]); - const chatTemplate = CustomChatPromptTemplate.fromMessages(lcMessages); - chatTemplate.variablesDefaultValues = this.variablesDefaultValues; - chatTemplate.literalTemplateMessages = this.templateMessages; - chatTemplate.promptId = this.id; - - return chatTemplate; - } -} diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 0000000..d04e5ff --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,81 @@ +export type Maybe = T | null | undefined; + +export type OmitUtils = Omit; + +export type Environment = 'dev' | 'staging' | 'prod' | 'experiment'; + +export type PageInfo = { + hasNextPage: boolean; + startCursor: string; + endCursor: string; +}; + +export type PaginatedResponse = { + data: T[]; + pageInfo: PageInfo; +}; + +export function isPlainObject(value: unknown): value is Record { + if (typeof value !== 'object' || value === null) { + return false; + } + + const prototype = Object.getPrototypeOf(value); + return prototype === null || prototype === Object.prototype; +} + +/** + * Represents a utility class with serialization capabilities. + */ +export class Utils { + /** + * Serializes the properties of the current instance into a dictionary, excluding the 'api' property. + * It handles nested objects that also implement a serialize method. + * + * @returns A dictionary representing the serialized properties of the object. + */ + serialize(): any { + const dict: any = {}; + Object.keys(this as any).forEach((key) => { + if (['api', 'client'].includes(key)) { + return; + } + if ((this as any)[key] !== undefined) { + if (Array.isArray((this as any)[key])) { + dict[key] = (this as any)[key].map((item: any) => { + if ( + item instanceof Object && + typeof item.serialize === 'function' + ) { + return item.serialize(); + } else { + return item; + } + }); + } else if ( + (this as any)[key] instanceof Object && + typeof (this as any)[key].serialize === 'function' + ) { + dict[key] = (this as any)[key].serialize(); + } else { + dict[key] = (this as any)[key]; + } + } + }); + return dict; + } +} + +/** + * Represents a user with optional metadata and identifier. + */ +export class User extends Utils { + id?: Maybe; + identifier!: string; + metadata?: Maybe>; + + constructor(data: OmitUtils) { + super(); + Object.assign(this, data); + } +} diff --git a/tests/api.test.ts b/tests/api.test.ts index 34a09cb..84cdefe 100644 --- a/tests/api.test.ts +++ b/tests/api.test.ts @@ -1,7 +1,9 @@ import 'dotenv/config'; import { v4 as uuidv4 } from 'uuid'; -import { ChatGeneration, Dataset, LiteralClient, Score } from '../src'; +import { ChatGeneration, LiteralClient } from '../src'; +import { Dataset } from '../src/evaluation/dataset'; +import { Score } from '../src/evaluation/score'; describe('End to end tests for the SDK', function () { let client: LiteralClient; diff --git a/tests/attachments.test.ts b/tests/attachments.test.ts index 514f4e4..ccb277d 100644 --- a/tests/attachments.test.ts +++ b/tests/attachments.test.ts @@ -1,7 +1,8 @@ import 'dotenv/config'; import { createReadStream, readFileSync } from 'fs'; -import { Attachment, LiteralClient, Maybe } from '../src'; +import { LiteralClient, Maybe } from '../src'; +import { Attachment } from '../src/observability/attachment'; const apiUrl = process.env.LITERAL_API_URL; const apiKey = process.env.LITERAL_API_KEY; diff --git a/tests/integration/openai.test.ts b/tests/integration/openai.test.ts index d68999d..0cd8238 100644 --- a/tests/integration/openai.test.ts +++ b/tests/integration/openai.test.ts @@ -3,13 +3,8 @@ import OpenAI from 'openai'; import { PassThrough } from 'stream'; import { v4 as uuidv4 } from 'uuid'; -import { - ChatGeneration, - LiteralClient, - Maybe, - OmitUtils, - Step -} from '../../src'; +import { ChatGeneration, LiteralClient, Maybe, OmitUtils } from '../../src'; +import { Step } from '../../src/observability/step'; const apiUrl = process.env.LITERAL_API_URL; const apiKey = process.env.LITERAL_API_KEY; diff --git a/tests/wrappers.test.ts b/tests/wrappers.test.ts index 01deeb7..53dff04 100644 --- a/tests/wrappers.test.ts +++ b/tests/wrappers.test.ts @@ -1,6 +1,8 @@ import 'dotenv/config'; -import { DatasetExperimentItem, LiteralClient, Maybe, Step } from '../src'; +import { LiteralClient, Maybe } from '../src'; +import { DatasetExperimentItem } from '../src/evaluation/dataset'; +import { Step } from '../src/observability/step'; const url = process.env.LITERAL_API_URL; const apiKey = process.env.LITERAL_API_KEY; From 5f7556d467542e7ae7ddfab605ec4c2ccf9313cf Mon Sep 17 00:00:00 2001 From: Damien BUTY Date: Tue, 30 Jul 2024 11:45:34 +0200 Subject: [PATCH 2/9] chore: add missing JSDoc comments --- src/api.ts | 89 ++++++++++--------- src/index.ts | 2 +- src/instrumentation/index.ts | 29 +++++- .../openai-syncer.ts} | 14 ++- 4 files changed, 82 insertions(+), 52 deletions(-) rename src/{openai.ts => instrumentation/openai-syncer.ts} (93%) diff --git a/src/api.ts b/src/api.ts index c1e1baa..8a63037 100644 --- a/src/api.ts +++ b/src/api.ts @@ -117,13 +117,6 @@ steps { ${stepFields} }`; -/** - * Serializes the step object with a suffix ID to each key. - * - * @param object - The step object to serialize. - * @param id - The numeric identifier to append to each key in the serialized object. - * @returns A new object with serialized key-value pairs where each key is suffixed with the provided id. - */ function serialize(object: Utils, id: number) { const result: any = {}; @@ -134,12 +127,6 @@ function serialize(object: Utils, id: number) { return result; } -/** - * Constructs a variables object for GraphQL queries by serializing each step with a unique suffix. - * - * @param objects - An array of `Step` objects to be serialized and added to the variables object. - * @returns An object containing serialized steps with keys suffixed by their index in the input array. - */ function variablesBuilder(objects: Utils[]) { let variables: any = {}; for (let i = 0; i < objects.length; i++) { @@ -160,13 +147,6 @@ function generationsVariablesBuilder( return variables; } -/** - * Builds a string for GraphQL field definitions for ingesting multiple steps. - * Each step's fields are suffixed with its index to create unique variable names. - * - * @param steps - An array of `Step` objects. Each `Step` object represents a step to be ingested. - * @returns A string containing GraphQL field definitions for all provided steps. - */ function ingestStepsFieldsBuilder(steps: Step[]) { let generated = ''; for (let id = 0; id < steps.length; id++) { @@ -190,14 +170,6 @@ function ingestStepsFieldsBuilder(steps: Step[]) { return generated; } -/** - * Constructs the arguments for a GraphQL mutation to ingest multiple steps. - * Each step is transformed into a call to the `ingestStep` mutation with parameters - * suffixed by the step's index to ensure uniqueness. - * - * @param steps - An array of `Step` objects. Each `Step` object represents a step to be ingested. - * @returns A string containing the GraphQL mutation arguments for all provided steps. - */ function ingestStepsArgsBuilder(steps: Step[]) { let generated = ''; for (let id = 0; id < steps.length; id++) { @@ -227,12 +199,6 @@ function ingestStepsArgsBuilder(steps: Step[]) { return generated; } -/** - * Constructs a complete GraphQL mutation query for adding multiple steps. - * - * @param steps - An array of `Step` objects to be ingested. This parameter is required. - * @returns A string representing the complete GraphQL mutation for adding steps. - */ function ingestStepsQueryBuilder(steps: Step[]) { return ` mutation AddStep(${ingestStepsFieldsBuilder(steps)}) { @@ -615,7 +581,6 @@ export class API { return result.data.deleteStep.id; } - // Upload /** * Uploads a file to a specified thread. This method supports uploading either through direct content or via a file path. * It first signs the upload through a pre-configured endpoint and then proceeds to upload the file using the signed URL. @@ -629,7 +594,6 @@ export class API { * @returns An object containing the `objectKey` of the uploaded file and the signed `url`, or `null` values if the upload fails. * @throws {Error} Throws an error if neither `content` nor `path` is provided, or if the server response is invalid. */ - async uploadFile(params: UploadFileParamsWithContent): Promise<{ objectKey: Maybe; url: Maybe; @@ -717,6 +681,21 @@ export class API { } } + /** + * Uploads a file to a specified thread and creates an attachment object. + * If called inside a context, the attachment will be added to the current step and thread. + * + * @param params - The parameters for uploading a file, including: + * @param params.name - The name of the file. + * @param params.metadata - Additional metadata for the file as a key-value pair object. + * @param params.content - The content of the file to upload. Optional if `path` is provided. + * @param params.path - The path to the file to upload. Optional if `content` is provided. + * @param params.id - The unique identifier for the file. If not provided, a UUID will be generated. + * @param params.threadId - The unique identifier of the thread to which the file is being uploaded. + * @param params.mime - The MIME type of the file. Defaults to 'application/octet-stream' if not provided. + * @returns An object containing the `objectKey` of the uploaded file and the signed `url`, or `null` values if the upload fails. + * @throws {Error} Throws an error if neither `content` nor `path` is provided, or if the server response is invalid. + */ async createAttachment( params: UploadFileParamsWithContent & CreateAttachmentParams ): Promise; @@ -763,7 +742,6 @@ export class API { return attachment; } - // Generation /** * Retrieves a paginated list of Generations based on the provided filters and sorting order. * @@ -873,7 +851,6 @@ export class API { return response.data.createGeneration as PersistedGeneration; } - // Thread /** * Upserts a Thread with new information. * @@ -1091,7 +1068,6 @@ export class API { return response.data.deleteThread.id; } - // User /** * Retrieves a list of users with optional filters. * @@ -1296,7 +1272,6 @@ export class API { return result.data.deleteParticipant.id; } - // Score /** * Get all scores connected to the platform. * @@ -1516,8 +1491,6 @@ export class API { return result.data.deleteScore; } - // Dataset - /** * List all datasets in the platform. * @@ -1819,6 +1792,13 @@ export class API { return new DatasetItem(result.data.addGenerationToDataset); } + /** + * Adds multiple generation items to a dataset. + * + * @param datasetId - The unique identifier of the dataset. This parameter is required. + * @param generationIds - An array of unique identifiers for the generations to be added. This parameter is required. + * @returns An array of `DatasetItem` instances populated with the data of the newly added generations + */ public async addGenerationsToDataset( datasetId: string, generationIds: string[] @@ -1830,6 +1810,15 @@ export class API { return Object.values(result.data).map((x: any) => new DatasetItem(x)); } + /** + * Creates a new dataset experiment. + * @param datasetExperiment + * @param datasetExperiment.name The name of the dataset experiment. + * @param datasetExperiment.datasetId The dataset ID to associate with the experiment. + * @param datasetExperiment.promptId The prompt ID to associate with the experiment. + * @param datasetExperiment.params The parameters for the experiment as a key-value pair object or an array of the same. + * @returns The newly created dataset experiment object. + */ public async createExperiment(datasetExperiment: { name: string; datasetId?: string; @@ -1854,6 +1843,18 @@ export class API { return new DatasetExperiment(this, result.data.createDatasetExperiment); } + /** + * Creates a new dataset experiment item. + * + * @param parameters + * @param parameters.datasetExperimentId The dataset experiment ID to associate with the item (required) + * @param parameters.scores An array of scores to associate with the item (required) + * @param parameters.datasetItemId The ID of the dataset item (optional) + * @param parameters.experimentRunId The ID of the experiment run (optional) + * @param parameters.input The input data for the item (optional) + * @param parameters.output The output data for the item (optional) + * @returns The dataset experiment object. + */ public async createExperimentItem({ datasetExperimentId, datasetItemId, @@ -1895,7 +1896,6 @@ export class API { }); } - // Prompt /** * Create a new prompt lineage. * @@ -1949,6 +1949,7 @@ export class API { * @param name The name of the prompt to retrieve or create. * @param templateMessages A list of template messages for the prompt. * @param settings Optional settings for the prompt. + * @param tools Optional tools for the prompt. * @returns The prompt that was retrieved or created. */ public async getOrCreatePrompt( @@ -2064,7 +2065,7 @@ export class API { return await this.getPromptWithQuery(query, { name, version }); } - public async getPromptWithQuery( + private async getPromptWithQuery( query: string, variables: Record ) { diff --git a/src/index.ts b/src/index.ts index 7b55e8c..216d52c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,9 +3,9 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import { API } from './api'; import { ExperimentRun } from './evaluation/experiment-run'; import instrumentation from './instrumentation'; +import openai from './instrumentation/openai-syncer'; import { Step, StepConstructor } from './observability/step'; import { Thread, ThreadConstructor } from './observability/thread'; -import openai from './openai'; import { Environment } from './utils'; export * from './utils'; diff --git a/src/instrumentation/index.ts b/src/instrumentation/index.ts index d382c0b..c616b0d 100644 --- a/src/instrumentation/index.ts +++ b/src/instrumentation/index.ts @@ -10,25 +10,48 @@ export type OpenAIGlobalOptions = { }; export default (client: LiteralClient) => ({ + /** + * Instrument the OpenAI client to log all generations to the Literal AI API. + * Compatible with OpenAI's `chat.completions.create`, `completions.create` and `images.generate` functions. + * If you want to add tags or metadata at the call level, you should use the augmented client returned by this function. + * Its functions will be augmented with `literalaiTags` and `literalaiMetadata` options. + * @param options + * @param options.tags Tags to attach to all generations. + * @param options.metadata Metadata to attach to all generations. + * @returns + */ openai: (options?: OpenAIGlobalOptions) => instrumentOpenAI(client, options), + langchain: { literalCallback: (threadId?: string) => { try { return new LiteralCallbackHandler(client, threadId); - - // Proceed with using `handler` as intended } catch (error) { throw new Error( - 'Failed to load the langchain. Please ensure langchain is installed.' + 'Failed to load langchain. Please ensure langchain is installed.' ); } } }, + vercel: { + /** + * Instrument a Vercel SDK function to log all invocations to the Literal AI API. + * It will be augmented with a `literalAiParent` option that allows you to specify a parent step or thread. + * @param fn The function to instrument. This can be Vercel SDK's `generateText` or `streamText` functions. + * @returns A new function that logs all invocations to the Literal AI API. + */ instrument: makeInstrumentVercelSDK(client) }, + llamaIndex: { + /** + * Instrument the LlamaIndex client to log all generations to the Literal AI API. + */ instrument: () => instrumentLlamaIndex(client), + /** + * Run a callback with a thread context. + */ withThread } }); diff --git a/src/openai.ts b/src/instrumentation/openai-syncer.ts similarity index 93% rename from src/openai.ts rename to src/instrumentation/openai-syncer.ts index fd7b2b3..fa0032c 100644 --- a/src/openai.ts +++ b/src/instrumentation/openai-syncer.ts @@ -8,10 +8,10 @@ import type { } from 'openai/resources/beta/threads/runs/steps'; import { v5 as uuidv5 } from 'uuid'; -import { LiteralClient } from '.'; -import { Attachment } from './observability/attachment'; -import { ChatGeneration } from './observability/generation'; -import { User } from './utils'; +import { LiteralClient } from '..'; +import { Attachment } from '../observability/attachment'; +import { ChatGeneration } from '../observability/generation'; +import { User } from '../utils'; class OpenAIAssistantSyncer { private NAMESPACE_UUID = '1b671a64-40d5-491e-99b0-da01ff1f3341'; @@ -192,6 +192,12 @@ class OpenAIAssistantSyncer { await Promise.all(toolPromises); } + /** + * Logs a thread and all its messages and runs to the Literal AI API. + * @param threadId The ID of the thread to log to + * @param user A User instance containing the information of the user that participated in the thread + * @param threadMetadata Additional metadata to attach to the thread, in key-value pairs + */ async syncThread( threadId: string, user?: User, From d80b621d5edeac94d561fa9959ad72183f56eaab Mon Sep 17 00:00:00 2001 From: Damien BUTY Date: Tue, 30 Jul 2024 11:48:20 +0200 Subject: [PATCH 3/9] docs: generate Typedoc --- docs/README.md | 2690 +++++++++--------------------------------------- 1 file changed, 460 insertions(+), 2230 deletions(-) diff --git a/docs/README.md b/docs/README.md index e9e924b..4ec6a49 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,2149 +6,433 @@ ## Classes -### BaseGeneration - -Represents a utility class with serialization capabilities. - -#### Extends - -- [`Utils`](README.md#utils) - -#### Constructors - -##### new BaseGeneration() - -> **new BaseGeneration**(): [`BaseGeneration`](README.md#basegeneration) - -###### Returns - -[`BaseGeneration`](README.md#basegeneration) - -###### Inherited from - -[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) - -#### Properties - -##### promptId - -> **promptId**?: `string` - -##### provider - -> **provider**?: [`Maybe`](README.md#maybet)\<`string`\> - -##### model - -> **model**?: [`Maybe`](README.md#maybet)\<`string`\> - -##### id - -> **id**?: [`Maybe`](README.md#maybet)\<`string`\> - -##### tags - -> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> - -##### error - -> **error**?: [`Maybe`](README.md#maybet)\<`string`\> - -##### variables - -> **variables**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> - -##### settings - -> **settings**?: [`Maybe`](README.md#maybet)\<[`ILLMSettings`](README.md#illmsettings)\> - -##### tools - -> **tools**?: [`Maybe`](README.md#maybet)\<[`ITool`](README.md#itool)[]\> - -##### tokenCount - -> **tokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> - -##### inputTokenCount - -> **inputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> - -##### outputTokenCount - -> **outputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> - -##### ttFirstToken - -> **ttFirstToken**?: [`Maybe`](README.md#maybet)\<`number`\> - -##### tokenThroughputInSeconds - -> **tokenThroughputInSeconds**?: [`Maybe`](README.md#maybet)\<`number`\> - -##### duration - -> **duration**?: [`Maybe`](README.md#maybet)\<`number`\> - -#### Methods - -##### serialize() - -> **serialize**(): `any` - -Serializes the properties of the current instance into a dictionary, excluding the 'api' property. -It handles nested objects that also implement a serialize method. - -###### Returns - -`any` - -A dictionary representing the serialized properties of the object. - -###### Inherited from - -[`Utils`](README.md#utils).[`serialize`](README.md#serialize-3) - -*** - -### CompletionGeneration - -Represents a utility class with serialization capabilities. - -#### Extends - -- [`BaseGeneration`](README.md#basegeneration) - -#### Constructors - -##### new CompletionGeneration(data) - -> **new CompletionGeneration**(`data`): [`CompletionGeneration`](README.md#completiongeneration) - -###### Parameters - -▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`CompletionGeneration`](README.md#completiongeneration)\> - -###### Returns - -[`CompletionGeneration`](README.md#completiongeneration) - -###### Overrides - -[`BaseGeneration`](README.md#basegeneration).[`constructor`](README.md#constructors) - -#### Properties - -##### promptId - -> **promptId**?: `string` - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`promptId`](README.md#promptid) - -##### provider - -> **provider**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`provider`](README.md#provider) - -##### model - -> **model**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`model`](README.md#model) - -##### id - -> **id**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`id`](README.md#id) - -##### tags - -> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`tags`](README.md#tags) - -##### error - -> **error**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`error`](README.md#error) - -##### variables - -> **variables**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`variables`](README.md#variables) - -##### settings - -> **settings**?: [`Maybe`](README.md#maybet)\<[`ILLMSettings`](README.md#illmsettings)\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`settings`](README.md#settings) - -##### tools - -> **tools**?: [`Maybe`](README.md#maybet)\<[`ITool`](README.md#itool)[]\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`tools`](README.md#tools) - -##### tokenCount - -> **tokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`tokenCount`](README.md#tokencount) - -##### inputTokenCount - -> **inputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`inputTokenCount`](README.md#inputtokencount) - -##### outputTokenCount - -> **outputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`outputTokenCount`](README.md#outputtokencount) - -##### ttFirstToken - -> **ttFirstToken**?: [`Maybe`](README.md#maybet)\<`number`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`ttFirstToken`](README.md#ttfirsttoken) - -##### tokenThroughputInSeconds - -> **tokenThroughputInSeconds**?: [`Maybe`](README.md#maybet)\<`number`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`tokenThroughputInSeconds`](README.md#tokenthroughputinseconds) - -##### duration - -> **duration**?: [`Maybe`](README.md#maybet)\<`number`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`duration`](README.md#duration) - -##### type - -> **type**?: [`GenerationType`](README.md#generationtype) = `'COMPLETION'` - -##### prompt - -> **prompt**?: [`Maybe`](README.md#maybet)\<`string`\> - -##### completion - -> **completion**?: [`Maybe`](README.md#maybet)\<`string`\> - -#### Methods - -##### serialize() - -> **serialize**(): `any` - -Serializes the properties of the current instance into a dictionary, excluding the 'api' property. -It handles nested objects that also implement a serialize method. - -###### Returns - -`any` - -A dictionary representing the serialized properties of the object. - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`serialize`](README.md#serialize) - -*** - -### ChatGeneration - -Represents a utility class with serialization capabilities. - -#### Extends - -- [`BaseGeneration`](README.md#basegeneration) - -#### Constructors - -##### new ChatGeneration(data) - -> **new ChatGeneration**(`data`): [`ChatGeneration`](README.md#chatgeneration) - -###### Parameters - -▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`ChatGeneration`](README.md#chatgeneration)\> - -###### Returns - -[`ChatGeneration`](README.md#chatgeneration) - -###### Overrides - -[`BaseGeneration`](README.md#basegeneration).[`constructor`](README.md#constructors) - -#### Properties - -##### promptId - -> **promptId**?: `string` - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`promptId`](README.md#promptid) - -##### provider - -> **provider**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`provider`](README.md#provider) - -##### model - -> **model**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`model`](README.md#model) - -##### id - -> **id**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`id`](README.md#id) - -##### tags - -> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`tags`](README.md#tags) - -##### error - -> **error**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`error`](README.md#error) - -##### variables - -> **variables**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`variables`](README.md#variables) - -##### settings - -> **settings**?: [`Maybe`](README.md#maybet)\<[`ILLMSettings`](README.md#illmsettings)\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`settings`](README.md#settings) - -##### tools - -> **tools**?: [`Maybe`](README.md#maybet)\<[`ITool`](README.md#itool)[]\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`tools`](README.md#tools) - -##### tokenCount - -> **tokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`tokenCount`](README.md#tokencount) - -##### inputTokenCount - -> **inputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`inputTokenCount`](README.md#inputtokencount) - -##### outputTokenCount - -> **outputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`outputTokenCount`](README.md#outputtokencount) - -##### ttFirstToken - -> **ttFirstToken**?: [`Maybe`](README.md#maybet)\<`number`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`ttFirstToken`](README.md#ttfirsttoken) - -##### tokenThroughputInSeconds - -> **tokenThroughputInSeconds**?: [`Maybe`](README.md#maybet)\<`number`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`tokenThroughputInSeconds`](README.md#tokenthroughputinseconds) - -##### duration - -> **duration**?: [`Maybe`](README.md#maybet)\<`number`\> - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`duration`](README.md#duration) - -##### type - -> **type**?: [`GenerationType`](README.md#generationtype) = `'CHAT'` - -##### messages - -> **messages**?: [`Maybe`](README.md#maybet)\<[`IGenerationMessage`](README.md#igenerationmessage)[]\> = `[]` - -##### messageCompletion - -> **messageCompletion**?: [`Maybe`](README.md#maybet)\<[`IGenerationMessage`](README.md#igenerationmessage)\> - -#### Methods - -##### serialize() - -> **serialize**(): `any` - -Serializes the properties of the current instance into a dictionary, excluding the 'api' property. -It handles nested objects that also implement a serialize method. - -###### Returns - -`any` - -A dictionary representing the serialized properties of the object. - -###### Inherited from - -[`BaseGeneration`](README.md#basegeneration).[`serialize`](README.md#serialize) - -*** - -### LiteralClient - -#### Constructors - -##### new LiteralClient(__namedParameters) - -> **new LiteralClient**(`__namedParameters`): [`LiteralClient`](README.md#literalclient) - -###### Parameters - -▪ **\_\_namedParameters**: `object`= `{}` - -▪ **\_\_namedParameters.apiKey?**: `string` - -▪ **\_\_namedParameters.apiUrl?**: `string` - -▪ **\_\_namedParameters.environment?**: [`Environment`](README.md#environment-3) - -▪ **\_\_namedParameters.disabled?**: `boolean` - -###### Returns - -[`LiteralClient`](README.md#literalclient) - -#### Properties - -##### api - -> **api**: `API` - -##### openai - -> **openai**: (`openai`) => `object` - -###### Parameters - -▪ **openai**: `OpenAI` - -###### Returns - -`object` - -> ###### assistant -> -> > **assistant**: `object` -> -> ###### assistant.syncer -> -> > **assistant.syncer**: `OpenAIAssistantSyncer` -> - -##### instrumentation - -> **instrumentation**: `object` - -###### Type declaration - -###### openai - -> **openai**: (`options`?) => `object` - -###### Parameters - -▪ **options?**: [`OpenAIGlobalOptions`](README.md#openaiglobaloptions) - -###### Returns - -`object` - -> ###### chat -> -> > **chat**: `object` -> -> ###### chat.completions -> -> > **chat.completions**: `object` -> -> ###### chat.completions.create -> -> > **chat.completions.create**: (`this`, `body`, `callOptions`?) => `Promise`\<`Stream`\<`ChatCompletionChunk`\> \| `ChatCompletion`\> = `wrappedChatCompletionsCreate` -> -> ###### Parameters -> -> ▪ **this**: `any` -> -> ▪ **body**: `any` -> -> ▪ **callOptions?**: `RequestOptions` & `OpenAICallOptions` -> -> ###### Returns -> -> `Promise`\<`Stream`\<`ChatCompletionChunk`\> \| `ChatCompletion`\> -> -> ###### completions -> -> > **completions**: `object` -> -> ###### completions.create -> -> > **completions.create**: (`this`, `body`, `callOptions`?) => `Promise`\<`Completion` \| `Stream`\<`Completion`\>\> = `wrappedCompletionsCreate` -> -> ###### Parameters -> -> ▪ **this**: `any` -> -> ▪ **body**: `any` -> -> ▪ **callOptions?**: `RequestOptions` & `OpenAICallOptions` -> -> ###### Returns -> -> `Promise`\<`Completion` \| `Stream`\<`Completion`\>\> -> -> ###### images -> -> > **images**: `object` -> -> ###### images.generate -> -> > **images.generate**: (`this`, `body`, `callOptions`?) => `Promise`\<`ImagesResponse`\> = `wrappedImagesGenerate` -> -> ###### Parameters -> -> ▪ **this**: `any` -> -> ▪ **body**: `any` -> -> ▪ **callOptions?**: `RequestOptions` & `OpenAICallOptions` -> -> ###### Returns -> -> `Promise`\<`ImagesResponse`\> -> - -###### langchain - -> **langchain**: `object` - -###### langchain.literalCallback - -> **langchain.literalCallback**: (`threadId`?) => `LiteralCallbackHandler` - -###### Parameters - -▪ **threadId?**: `string` - -###### Returns - -`LiteralCallbackHandler` - -###### vercel - -> **vercel**: `object` - -###### vercel.instrument - -> **vercel.instrument**: `InstrumentationVercelMethod` - -###### llamaIndex - -> **llamaIndex**: `object` - -###### llamaIndex.instrument - -> **llamaIndex.instrument**: () => `void` - -###### Returns - -`void` - -###### llamaIndex.withThread - -> **llamaIndex.withThread**: \<`R`\>(`thread`, `callback`) => `R` - -###### Type parameters - -▪ **R** - -###### Parameters - -▪ **thread**: [`Thread`](README.md#thread-1) - -▪ **callback**: () => `R` - -###### Returns - -`R` - -##### store - -> **store**: `AsyncLocalStorage`\<`StoredContext`\> = `storage` - -#### Methods - -##### thread() - -> **thread**(`data`?): [`Thread`](README.md#thread-1) - -###### Parameters - -▪ **data?**: [`ThreadConstructor`](README.md#threadconstructor) - -###### Returns - -[`Thread`](README.md#thread-1) - -##### step() - -> **step**(`data`): [`Step`](README.md#step-2) - -###### Parameters - -▪ **data**: [`StepConstructor`](README.md#stepconstructor) - -###### Returns - -[`Step`](README.md#step-2) - -##### run() - -> **run**(`data`): [`Step`](README.md#step-2) - -###### Parameters - -▪ **data**: `Omit`\<[`StepConstructor`](README.md#stepconstructor), `"type"`\> - -###### Returns - -[`Step`](README.md#step-2) - -##### experimentRun() - -> **experimentRun**(`data`?): [`ExperimentRun`](README.md#experimentrun-1) - -###### Parameters - -▪ **data?**: `Omit`\<[`StepConstructor`](README.md#stepconstructor), `"name"` \| `"type"`\> - -###### Returns - -[`ExperimentRun`](README.md#experimentrun-1) - -##### \_currentThread() - -> **\_currentThread**(): `null` \| [`Thread`](README.md#thread-1) - -###### Returns - -`null` \| [`Thread`](README.md#thread-1) - -##### \_currentStep() - -> **\_currentStep**(): `null` \| [`Step`](README.md#step-2) - -###### Returns - -`null` \| [`Step`](README.md#step-2) - -##### getCurrentThread() - -> **getCurrentThread**(): [`Thread`](README.md#thread-1) - -Gets the current thread from the context. -WARNING : this will throw if run outside of a thread context. - -###### Returns - -[`Thread`](README.md#thread-1) - -The current thread, if any. - -##### getCurrentStep() - -> **getCurrentStep**(): [`Step`](README.md#step-2) - -Gets the current step from the context. -WARNING : this will throw if run outside of a step context. - -###### Returns - -[`Step`](README.md#step-2) - -The current step, if any. - -*** - -### Utils - -Represents a utility class with serialization capabilities. - -#### Extended By - -- [`Attachment`](README.md#attachment) -- [`User`](README.md#user) -- [`DatasetItem`](README.md#datasetitem) -- [`DatasetExperiment`](README.md#datasetexperiment) -- [`BaseGeneration`](README.md#basegeneration) - -#### Constructors - -##### new Utils() - -> **new Utils**(): [`Utils`](README.md#utils) - -###### Returns - -[`Utils`](README.md#utils) - -#### Methods - -##### serialize() - -> **serialize**(): `any` - -Serializes the properties of the current instance into a dictionary, excluding the 'api' property. -It handles nested objects that also implement a serialize method. - -###### Returns - -`any` - -A dictionary representing the serialized properties of the object. - -*** - -### Score - -Represents a score entity with properties to track various aspects of scoring. -It extends the `Utils` class for serialization capabilities. - -#### Extends - -- `ScoreFields` - -#### Constructors - -##### new Score(data) - -> **new Score**(`data`): [`Score`](README.md#score) - -###### Parameters - -▪ **data**: [`ScoreConstructor`](README.md#scoreconstructor) - -###### Returns - -[`Score`](README.md#score) - -###### Overrides - -ScoreFields.constructor - -#### Properties - -##### id - -> **id**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -ScoreFields.id - -##### stepId - -> **stepId**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -ScoreFields.stepId - -##### generationId - -> **generationId**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -ScoreFields.generationId - -##### datasetExperimentItemId - -> **datasetExperimentItemId**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -ScoreFields.datasetExperimentItemId - -##### name - -> **name**: `string` = `'user-feedback'` - -###### Inherited from - -ScoreFields.name - -##### value - -> **value**: `number` = `0` - -###### Inherited from - -ScoreFields.value - -##### type - -> **type**: [`ScoreType`](README.md#scoretype) = `'AI'` - -###### Inherited from - -ScoreFields.type - -##### scorer - -> **scorer**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -ScoreFields.scorer - -##### comment - -> **comment**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -ScoreFields.comment - -##### tags - -> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> - -###### Inherited from - -ScoreFields.tags - -#### Methods - -##### serialize() - -> **serialize**(): `any` - -Serializes the properties of the current instance into a dictionary, excluding the 'api' property. -It handles nested objects that also implement a serialize method. - -###### Returns - -`any` - -A dictionary representing the serialized properties of the object. - -###### Inherited from - -ScoreFields.serialize - -*** - -### Attachment - -Represents an attachment with optional metadata, MIME type, and other properties. -It extends the `Utils` class for serialization capabilities. - -#### Extends - -- [`Utils`](README.md#utils) - -#### Constructors - -##### new Attachment(data) - -> **new Attachment**(`data`): [`Attachment`](README.md#attachment) - -###### Parameters - -▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`Attachment`](README.md#attachment)\> - -###### Returns - -[`Attachment`](README.md#attachment) - -###### Overrides - -[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) - -#### Properties - -##### id - -> **id**?: [`Maybe`](README.md#maybet)\<`string`\> - -##### metadata - -> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> - -##### mime - -> **mime**?: [`Maybe`](README.md#maybet)\<`string`\> - -##### name - -> **name**: [`Maybe`](README.md#maybet)\<`string`\> - -##### objectKey - -> **objectKey**?: [`Maybe`](README.md#maybet)\<`string`\> - -##### url - -> **url**?: [`Maybe`](README.md#maybet)\<`string`\> - -#### Methods - -##### serialize() - -> **serialize**(): `any` - -Serializes the properties of the current instance into a dictionary, excluding the 'api' property. -It handles nested objects that also implement a serialize method. - -###### Returns - -`any` - -A dictionary representing the serialized properties of the object. - -###### Inherited from - -[`Utils`](README.md#utils).[`serialize`](README.md#serialize-3) - -*** - -### Thread - -Represents a thread in the system, extending the properties and methods from `ThreadFields`. -This class manages thread-specific operations such as creation and updates via the API. - -#### Extends - -- `ThreadFields` - -#### Constructors - -##### new Thread(client, data) - -> **new Thread**(`client`, `data`?): [`Thread`](README.md#thread-1) - -Constructs a new Thread instance. - -###### Parameters - -▪ **client**: [`LiteralClient`](README.md#literalclient) - -▪ **data?**: [`ThreadConstructor`](README.md#threadconstructor) - -Optional initial data for the thread, with an auto-generated ID if not provided. - -###### Returns - -[`Thread`](README.md#thread-1) - -###### Overrides - -ThreadFields.constructor - -#### Properties - -##### id - -> **id**: `string` - -###### Inherited from - -ThreadFields.id - -##### participantId - -> **participantId**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -ThreadFields.participantId - -##### environment - -> **environment**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -ThreadFields.environment - -##### name - -> **name**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -ThreadFields.name - -##### metadata - -> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> - -###### Inherited from - -ThreadFields.metadata - -##### steps - -> **steps**?: [`Maybe`](README.md#maybet)\<[`Step`](README.md#step-2)[]\> - -###### Inherited from - -ThreadFields.steps - -##### tags - -> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> - -###### Inherited from - -ThreadFields.tags - -##### api - -> **api**: `API` - -##### client - -> **client**: [`LiteralClient`](README.md#literalclient) - -#### Methods - -##### serialize() - -> **serialize**(): `any` - -Serializes the properties of the current instance into a dictionary, excluding the 'api' property. -It handles nested objects that also implement a serialize method. - -###### Returns - -`any` - -A dictionary representing the serialized properties of the object. - -###### Inherited from - -ThreadFields.serialize - -##### step() - -> **step**(`data`): [`Step`](README.md#step-2) - -Creates a new step associated with this thread. - -###### Parameters - -▪ **data**: `Omit`\<[`StepConstructor`](README.md#stepconstructor), `"threadId"`\> - -The data for the new step, excluding the thread ID. - -###### Returns - -[`Step`](README.md#step-2) - -A new Step instance linked to this thread. - -##### run() - -> **run**(`data`): [`Step`](README.md#step-2) - -Creates a new Run step associated with this thread. - -###### Parameters - -▪ **data**: `Omit`\<[`StepConstructor`](README.md#stepconstructor), `"type"` \| `"threadId"`\> - -The data for the new step, excluding the thread ID and the type - -###### Returns - -[`Step`](README.md#step-2) - -A new Step instance linked to this thread. - -##### upsert() - -> **upsert**(): `Promise`\<[`Thread`](README.md#thread-1)\> - -Upserts the thread data to the backend, creating or updating as necessary. - -###### Returns - -`Promise`\<[`Thread`](README.md#thread-1)\> - -The updated Thread instance. - -##### wrap() - -> **wrap**\<`Output`\>(`cb`, `updateThread`?): `Promise`\<`Output`\> - -Sends the thread to the API, handling disabled state and setting the end time if not already set. - -###### Type parameters - -▪ **Output** - -###### Parameters - -▪ **cb**: (`thread`) => `Output` \| `Promise`\<`Output`\> - -The callback function to run within the context of the thread. - -▪ **updateThread?**: [`ThreadConstructor`](README.md#threadconstructor) \| (`output`) => [`ThreadConstructor`](README.md#threadconstructor) \| (`output`) => `Promise`\<[`ThreadConstructor`](README.md#threadconstructor)\> - -Optional update function to modify the thread after the callback. - -###### Returns - -`Promise`\<`Output`\> - -The output of the wrapped callback function. - -*** - -### Step - -Represents a step in a process or workflow, extending the fields and methods from StepFields. - -#### Extends - -- `StepFields` - -#### Constructors - -##### new Step(client, data, ignoreContext) - -> **new Step**(`client`, `data`, `ignoreContext`?): [`Step`](README.md#step-2) - -Constructs a new Step instance. - -###### Parameters - -▪ **client**: [`LiteralClient`](README.md#literalclient) - -▪ **data**: [`StepConstructor`](README.md#stepconstructor) - -The initial data for the step, excluding utility properties. - -▪ **ignoreContext?**: `true` - -###### Returns - -[`Step`](README.md#step-2) - -###### Overrides - -StepFields.constructor - -#### Properties - -##### name - -> **name**: `string` - -###### Inherited from - -StepFields.name - -##### type - -> **type**: [`StepType`](README.md#steptype) - -###### Inherited from - -StepFields.type - -##### threadId - -> **threadId**?: `string` - -###### Inherited from - -StepFields.threadId - -##### createdAt - -> **createdAt**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -StepFields.createdAt - -##### startTime - -> **startTime**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -StepFields.startTime - -##### id - -> **id**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -StepFields.id - -##### environment - -> **environment**?: [`Maybe`](README.md#maybet)\<[`Environment`](README.md#environment-3)\> - -###### Inherited from - -StepFields.environment - -##### error - -> **error**?: [`Maybe`](README.md#maybet)\<`string` \| `Record`\<`string`, `any`\>\> - -###### Inherited from - -StepFields.error - -##### input - -> **input**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> - -###### Inherited from - -StepFields.input - -##### output - -> **output**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> - -###### Inherited from - -StepFields.output - -##### metadata - -> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> - -###### Inherited from - -StepFields.metadata - -##### tags - -> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> - -###### Inherited from - -StepFields.tags - -##### parentId - -> **parentId**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -StepFields.parentId - -##### endTime - -> **endTime**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -StepFields.endTime - -##### generation - -> **generation**?: [`Maybe`](README.md#maybet)\<[`Generation`](README.md#generation-2)\> - -###### Inherited from - -StepFields.generation - -##### scores - -> **scores**?: [`Maybe`](README.md#maybet)\<[`Score`](README.md#score)[]\> - -###### Inherited from - -StepFields.scores - -##### attachments - -> **attachments**?: [`Maybe`](README.md#maybet)\<[`Attachment`](README.md#attachment)[]\> - -###### Inherited from - -StepFields.attachments - -##### api - -> **api**: `API` - -##### client - -> **client**: [`LiteralClient`](README.md#literalclient) - -#### Methods - -##### serialize() - -> **serialize**(): `any` - -Serializes the step instance, converting complex objects to strings as necessary. - -###### Returns - -`any` - -A serialized representation of the step. - -###### Overrides - -StepFields.serialize - -##### isMessage() - -> **isMessage**(): `boolean` - -Determines if the step is a type of message. - -###### Returns - -`boolean` - -True if the step is a user, assistant, or system message. - -##### step() - -> **step**(`data`): [`Step`](README.md#step-2) - -Creates a new step instance linked to the current step as a parent. - -###### Parameters - -▪ **data**: `Omit`\<[`StepConstructor`](README.md#stepconstructor), `"threadId"`\> - -The data for the new step, excluding the threadId which is inherited. - -###### Returns - -[`Step`](README.md#step-2) - -A new Step instance. - -##### send() - -> **send**(): `Promise`\<[`Step`](README.md#step-2)\> - -Sends the step to the API, handling disabled state and setting the end time if not already set. - -###### Returns - -`Promise`\<[`Step`](README.md#step-2)\> - -The current Step instance after potentially sending to the API. - -##### wrap() - -> **wrap**\<`Output`\>(`cb`, `updateStep`?): `Promise`\<`Output`\> - -Sends the step to the API, handling disabled state and setting the end time if not already set. - -###### Type parameters - -▪ **Output** - -###### Parameters - -▪ **cb**: (`step`) => `Output` \| `Promise`\<`Output`\> - -The callback function to run within the context of the step. - -▪ **updateStep?**: `Partial`\<[`StepConstructor`](README.md#stepconstructor)\> \| (`output`) => `Partial`\<[`StepConstructor`](README.md#stepconstructor)\> \| (`output`) => `Promise`\<`Partial`\<[`StepConstructor`](README.md#stepconstructor)\>\> - -Optional update function to modify the step after the callback. - -###### Returns - -`Promise`\<`Output`\> - -The output of the wrapped callback function. - -*** - -### ExperimentRun - -Represents a step in a process or workflow, extending the fields and methods from StepFields. - -#### Extends - -- [`Step`](README.md#step-2) - -#### Constructors - -##### new ExperimentRun(client, data, ignoreContext) - -> **new ExperimentRun**(`client`, `data`, `ignoreContext`?): [`ExperimentRun`](README.md#experimentrun-1) - -Constructs a new ExperimentRun instance. - -###### Parameters - -▪ **client**: [`LiteralClient`](README.md#literalclient) - -▪ **data**: [`StepConstructor`](README.md#stepconstructor) - -The initial data for the step, excluding utility properties. - -▪ **ignoreContext?**: `true` - -###### Returns - -[`ExperimentRun`](README.md#experimentrun-1) - -###### Overrides - -[`Step`](README.md#step-2).[`constructor`](README.md#constructors-8) - -#### Properties - -##### name - -> **name**: `string` - -###### Inherited from - -[`Step`](README.md#step-2).[`name`](README.md#name-3) - -##### type - -> **type**: [`StepType`](README.md#steptype) - -###### Inherited from - -[`Step`](README.md#step-2).[`type`](README.md#type-3) - -##### threadId - -> **threadId**?: `string` - -###### Inherited from - -[`Step`](README.md#step-2).[`threadId`](README.md#threadid) - -##### createdAt - -> **createdAt**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -[`Step`](README.md#step-2).[`createdAt`](README.md#createdat) - -##### startTime - -> **startTime**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -[`Step`](README.md#step-2).[`startTime`](README.md#starttime) - -##### id - -> **id**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -[`Step`](README.md#step-2).[`id`](README.md#id-6) - -##### environment - -> **environment**?: [`Maybe`](README.md#maybet)\<[`Environment`](README.md#environment-3)\> - -###### Inherited from - -[`Step`](README.md#step-2).[`environment`](README.md#environment-1) - -##### error - -> **error**?: [`Maybe`](README.md#maybet)\<`string` \| `Record`\<`string`, `any`\>\> - -###### Inherited from - -[`Step`](README.md#step-2).[`error`](README.md#error-3) - -##### input - -> **input**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> - -###### Inherited from - -[`Step`](README.md#step-2).[`input`](README.md#input) - -##### output - -> **output**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> - -###### Inherited from - -[`Step`](README.md#step-2).[`output`](README.md#output) - -##### metadata - -> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> - -###### Inherited from - -[`Step`](README.md#step-2).[`metadata`](README.md#metadata-2) - -##### tags - -> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> - -###### Inherited from - -[`Step`](README.md#step-2).[`tags`](README.md#tags-5) - -##### parentId - -> **parentId**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -[`Step`](README.md#step-2).[`parentId`](README.md#parentid) - -##### endTime - -> **endTime**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -[`Step`](README.md#step-2).[`endTime`](README.md#endtime) - -##### generation - -> **generation**?: [`Maybe`](README.md#maybet)\<[`Generation`](README.md#generation-2)\> - -###### Inherited from - -[`Step`](README.md#step-2).[`generation`](README.md#generation) - -##### scores - -> **scores**?: [`Maybe`](README.md#maybet)\<[`Score`](README.md#score)[]\> - -###### Inherited from - -[`Step`](README.md#step-2).[`scores`](README.md#scores) - -##### attachments - -> **attachments**?: [`Maybe`](README.md#maybet)\<[`Attachment`](README.md#attachment)[]\> - -###### Inherited from - -[`Step`](README.md#step-2).[`attachments`](README.md#attachments) - -##### api - -> **api**: `API` - -###### Overrides - -[`Step`](README.md#step-2).[`api`](README.md#api-2) - -##### client - -> **client**: [`LiteralClient`](README.md#literalclient) - -###### Overrides - -[`Step`](README.md#step-2).[`client`](README.md#client-1) - -#### Methods - -##### serialize() - -> **serialize**(): `any` - -Serializes the step instance, converting complex objects to strings as necessary. - -###### Returns - -`any` - -A serialized representation of the step. - -###### Inherited from - -[`Step`](README.md#step-2).[`serialize`](README.md#serialize-7) - -##### isMessage() - -> **isMessage**(): `boolean` - -Determines if the step is a type of message. - -###### Returns - -`boolean` - -True if the step is a user, assistant, or system message. - -###### Inherited from +### LiteralClient -[`Step`](README.md#step-2).[`isMessage`](README.md#ismessage) +#### Constructors -##### step() +##### new LiteralClient(options) -> **step**(`data`): [`Step`](README.md#step-2) +> **new LiteralClient**(`options`): [`LiteralClient`](README.md#literalclient) -Creates a new step instance linked to the current step as a parent. +Initialize a new Literal AI Client. ###### Parameters -▪ **data**: `Omit`\<[`StepConstructor`](README.md#stepconstructor), `"threadId"`\> +▪ **options**: `object`= `{}` -The data for the new step, excluding the threadId which is inherited. - -###### Returns +▪ **options.apiKey?**: `string` -[`Step`](README.md#step-2) +The API key to use for the Literal AI API. Defaults to the LITERAL_API_KEY environment variable. -A new Step instance. +▪ **options.apiUrl?**: `string` -###### Inherited from +The URL of the Literal AI API. Defaults to the LITERAL_API_URL environment variable. -[`Step`](README.md#step-2).[`step`](README.md#step-3) +▪ **options.environment?**: [`Environment`](README.md#environment) -##### send() +The environment to use for the Literal AI API. -> **send**(): `Promise`\<[`ExperimentRun`](README.md#experimentrun-1)\> +▪ **options.disabled?**: `boolean` -Sends the step to the API, handling disabled state and setting the end time if not already set. +If set to true, no call will be made to the Literal AI API. ###### Returns -`Promise`\<[`ExperimentRun`](README.md#experimentrun-1)\> - -The current Step instance after potentially sending to the API. - -###### Inherited from +[`LiteralClient`](README.md#literalclient) -[`Step`](README.md#step-2).[`send`](README.md#send) +A new LiteralClient instance. -##### wrap() +#### Properties -> **wrap**\<`Output`\>(`cb`, `updateStep`?): `Promise`\<`Output`\> +##### api -Sends the step to the API, handling disabled state and setting the end time if not already set. +> **api**: `API` -###### Type parameters +##### openai -▪ **Output** +> **openai**: (`openai`) => `object` ###### Parameters -▪ **cb**: (`step`) => `Output` \| `Promise`\<`Output`\> - -The callback function to run within the context of the step. - -▪ **updateStep?**: `Partial`\<[`StepConstructor`](README.md#stepconstructor)\> \| (`output`) => `Partial`\<[`StepConstructor`](README.md#stepconstructor)\> \| (`output`) => `Promise`\<`Partial`\<[`StepConstructor`](README.md#stepconstructor)\>\> - -Optional update function to modify the step after the callback. +▪ **openai**: `OpenAI` ###### Returns -`Promise`\<`Output`\> - -The output of the wrapped callback function. - -###### Overrides - -[`Step`](README.md#step-2).[`wrap`](README.md#wrap-1) - -*** +`object` -### User +> ###### assistant +> +> > **assistant**: `object` +> +> ###### assistant.syncer +> +> > **assistant.syncer**: `OpenAIAssistantSyncer` +> -Represents a user with optional metadata and identifier. +##### instrumentation -#### Extends +> **instrumentation**: `object` -- [`Utils`](README.md#utils) +###### Type declaration -#### Constructors +###### openai -##### new User(data) +> **openai**: (`options`?) => `object` -> **new User**(`data`): [`User`](README.md#user) +Instrument the OpenAI client to log all generations to the Literal AI API. +Compatible with OpenAI's `chat.completions.create`, `completions.create` and `images.generate` functions. +If you want to add tags or metadata at the call level, you should use the augmented client returned by this function. +Its functions will be augmented with `literalaiTags` and `literalaiMetadata` options. ###### Parameters -▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`User`](README.md#user)\> - -###### Returns - -[`User`](README.md#user) - -###### Overrides - -[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) - -#### Properties - -##### id - -> **id**?: [`Maybe`](README.md#maybet)\<`string`\> - -##### identifier - -> **identifier**: `string` - -##### metadata - -> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> - -#### Methods - -##### serialize() - -> **serialize**(): `any` - -Serializes the properties of the current instance into a dictionary, excluding the 'api' property. -It handles nested objects that also implement a serialize method. +▪ **options?**: [`OpenAIGlobalOptions`](README.md#openaiglobaloptions) ###### Returns -`any` - -A dictionary representing the serialized properties of the object. - -###### Inherited from - -[`Utils`](README.md#utils).[`serialize`](README.md#serialize-3) - -*** - -### Dataset - -#### Extends +`object` -- `DatasetFields` +> ###### chat +> +> > **chat**: `object` +> +> ###### chat.completions +> +> > **chat.completions**: `object` +> +> ###### chat.completions.create +> +> > **chat.completions.create**: (`this`, `body`, `callOptions`?) => `Promise`\<`Stream`\<`ChatCompletionChunk`\> \| `ChatCompletion`\> = `wrappedChatCompletionsCreate` +> +> ###### Parameters +> +> ▪ **this**: `any` +> +> ▪ **body**: `any` +> +> ▪ **callOptions?**: `RequestOptions` & `OpenAICallOptions` +> +> ###### Returns +> +> `Promise`\<`Stream`\<`ChatCompletionChunk`\> \| `ChatCompletion`\> +> +> ###### completions +> +> > **completions**: `object` +> +> ###### completions.create +> +> > **completions.create**: (`this`, `body`, `callOptions`?) => `Promise`\<`Completion` \| `Stream`\<`Completion`\>\> = `wrappedCompletionsCreate` +> +> ###### Parameters +> +> ▪ **this**: `any` +> +> ▪ **body**: `any` +> +> ▪ **callOptions?**: `RequestOptions` & `OpenAICallOptions` +> +> ###### Returns +> +> `Promise`\<`Completion` \| `Stream`\<`Completion`\>\> +> +> ###### images +> +> > **images**: `object` +> +> ###### images.generate +> +> > **images.generate**: (`this`, `body`, `callOptions`?) => `Promise`\<`ImagesResponse`\> = `wrappedImagesGenerate` +> +> ###### Parameters +> +> ▪ **this**: `any` +> +> ▪ **body**: `any` +> +> ▪ **callOptions?**: `RequestOptions` & `OpenAICallOptions` +> +> ###### Returns +> +> `Promise`\<`ImagesResponse`\> +> -#### Constructors +###### langchain -##### new Dataset(api, data) +> **langchain**: `object` -> **new Dataset**(`api`, `data`): [`Dataset`](README.md#dataset) +###### langchain.literalCallback -Constructs a new Dataset instance. +> **langchain.literalCallback**: (`threadId`?) => `LiteralCallbackHandler` ###### Parameters -▪ **api**: `API` - -The API instance to interact with backend services. - -▪ **data**: [`DatasetConstructor`](README.md#datasetconstructor) - -The initial data for the dataset. +▪ **threadId?**: `string` ###### Returns -[`Dataset`](README.md#dataset) - -###### Overrides - -DatasetFields.constructor - -#### Properties - -##### id - -> **id**: `string` - -###### Inherited from - -DatasetFields.id - -##### createdAt - -> **createdAt**: `string` - -###### Inherited from - -DatasetFields.createdAt - -##### name - -> **name**?: [`Maybe`](README.md#maybet)\<`string`\> - -###### Inherited from - -DatasetFields.name - -##### description +`LiteralCallbackHandler` -> **description**?: [`Maybe`](README.md#maybet)\<`string`\> +###### vercel -###### Inherited from +> **vercel**: `object` -DatasetFields.description +###### vercel.instrument -##### metadata +> **vercel.instrument**: `InstrumentationVercelMethod` -> **metadata**: `Record`\<`string`, `any`\> +Instrument a Vercel SDK function to log all invocations to the Literal AI API. +It will be augmented with a `literalAiParent` option that allows you to specify a parent step or thread. -###### Inherited from +###### Param -DatasetFields.metadata +The function to instrument. This can be Vercel SDK's `generateText` or `streamText` functions. -##### items +###### llamaIndex -> **items**: [`OmitUtils`](README.md#omitutilst)\<[`DatasetItem`](README.md#datasetitem)\>[] +> **llamaIndex**: `object` -###### Inherited from +###### llamaIndex.instrument -DatasetFields.items +> **llamaIndex.instrument**: () => `void` -##### type +Instrument the LlamaIndex client to log all generations to the Literal AI API. -> **type**?: [`DatasetType`](README.md#datasettype) +###### Returns -###### Inherited from +`void` -DatasetFields.type +###### llamaIndex.withThread -##### api +> **llamaIndex.withThread**: \<`R`\>(`thread`, `callback`) => `R` -> **api**: `API` +###### Type parameters -#### Methods +▪ **R** -##### serialize() +###### Parameters -> **serialize**(): `any` +▪ **thread**: `Thread` -Serializes the properties of the current instance into a dictionary, excluding the 'api' property. -It handles nested objects that also implement a serialize method. +▪ **callback**: () => `R` ###### Returns -`any` +`R` -A dictionary representing the serialized properties of the object. +##### store -###### Inherited from +> **store**: `AsyncLocalStorage`\<`StoredContext`\> = `storage` -DatasetFields.serialize +#### Methods -##### update() +##### thread() -> **update**(`dataset`): `Promise`\<`void`\> +> **thread**(`data`?): `Thread` -Updates the dataset with new data. +Creates a new thread without sending it to the Literal AI API. ###### Parameters -▪ **dataset**: `object` +▪ **data?**: `ThreadConstructor` -The dataset data to update. +Optional initial data for the thread. -▪ **dataset.name?**: [`Maybe`](README.md#maybet)\<`string`\> +###### Returns -▪ **dataset.description?**: [`Maybe`](README.md#maybet)\<`string`\> +`Thread` -▪ **dataset.metadata?**: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> +A new thread instance. -###### Returns +##### step() -`Promise`\<`void`\> +> **step**(`data`): `Step` -The updated dataset instance. +Creates a new step without sending it to the Literal AI API. -##### delete() +###### Parameters -> **delete**(): `Promise`\<[`Dataset`](README.md#dataset)\> +▪ **data**: `StepConstructor` -Deletes the dataset. +Optional initial data for the step. ###### Returns -`Promise`\<[`Dataset`](README.md#dataset)\> +`Step` -A promise that resolves when the dataset is deleted. +A new step instance. -##### createItem() +##### run() -> **createItem**(`datasetItem`): `Promise`\<[`DatasetItem`](README.md#datasetitem)\> +> **run**(`data`): `Step` -Creates a new item in the dataset. +Creates a new step with the type set to 'run'. ###### Parameters -▪ **datasetItem**: `object` - -The new item to be added to the dataset. - -▪ **datasetItem.input**: `Record`\<`string`, `any`\> - -▪ **datasetItem.expectedOutput?**: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> +▪ **data**: `Omit`\<`StepConstructor`, `"type"`\> -▪ **datasetItem.metadata?**: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> +Optional initial data for the step. ###### Returns -`Promise`\<[`DatasetItem`](README.md#datasetitem)\> +`Step` -The newly created dataset item. +A new step instance. -##### deleteItem() +##### experimentRun() -> **deleteItem**(`id`): `Promise`\<[`DatasetItem`](README.md#datasetitem)\> +> **experimentRun**(`data`?): `ExperimentRun` -Deletes an item from the dataset. +Creates a new Experiment Run. ###### Parameters -▪ **id**: `string` +▪ **data?**: `Omit`\<`StepConstructor`, `"name"` \| `"type"`\> -The ID of the item to delete. +Optional initial data for the step. ###### Returns -`Promise`\<[`DatasetItem`](README.md#datasetitem)\> +`ExperimentRun` -The deleted dataset item. +A new step instance. -##### createExperiment() +##### \_currentThread() -> **createExperiment**(`experiment`): `Promise`\<[`DatasetExperiment`](README.md#datasetexperiment)\> +> **\_currentThread**(): `null` \| `Thread` -Creates a new experiment associated with the dataset. +Returns the current thread from the context or null if none. -###### Parameters +###### Returns -▪ **experiment**: `object` +`null` \| `Thread` -The experiment details including name, optional prompt ID, and parameters. +The current thread, if any. -▪ **experiment.name**: `string` +##### \_currentStep() -▪ **experiment.promptId?**: `string` +> **\_currentStep**(): `null` \| `Step` -▪ **experiment.params?**: `Record`\<`string`, `any`\> \| `Record`\<`string`, `any`\>[] +Returns the current step from the context or null if none. ###### Returns -`Promise`\<[`DatasetExperiment`](README.md#datasetexperiment)\> +`null` \| `Step` -A new instance of DatasetExperiment containing the created experiment. - -##### addStep() +The current step, if any. -> **addStep**(`stepId`, `metadata`?): `Promise`\<[`DatasetItem`](README.md#datasetitem)\> +##### getCurrentThread() -Adds a step to the dataset. +> **getCurrentThread**(): `Thread` -###### Parameters +Gets the current thread from the context. +WARNING : this will throw if run outside of a thread context. -▪ **stepId**: `string` +###### Returns -The ID of the step to add. +`Thread` -▪ **metadata?**: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `unknown`\>\> +The current thread, if any. -Optional metadata for the step. +##### getCurrentStep() -###### Returns +> **getCurrentStep**(): `Step` -`Promise`\<[`DatasetItem`](README.md#datasetitem)\> +Gets the current step from the context. +WARNING : this will throw if run outside of a step context. -The added dataset item. +###### Returns -###### Throws +`Step` -Error if the dataset type is 'generation'. +The current step, if any. -##### addGeneration() +*** -> **addGeneration**(`generationId`, `metadata`?): `Promise`\<[`DatasetItem`](README.md#datasetitem)\> +### BaseGeneration -Adds a generation to the dataset. +Represents a utility class with serialization capabilities. -###### Parameters +#### Extends -▪ **generationId**: `string` +- [`Utils`](README.md#utils) -The ID of the generation to add. +#### Constructors -▪ **metadata?**: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `unknown`\>\> +##### new BaseGeneration() -Optional metadata for the generation. +> **new BaseGeneration**(): [`BaseGeneration`](README.md#basegeneration) ###### Returns -`Promise`\<[`DatasetItem`](README.md#datasetitem)\> - -The added dataset item. - -##### addGenerations() - -> **addGenerations**(`generationIds`?): `Promise`\<[`DatasetItem`](README.md#datasetitem)[]\> +[`BaseGeneration`](README.md#basegeneration) -###### Parameters +###### Inherited from -▪ **generationIds?**: `string`[] +[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) -###### Returns +#### Properties -`Promise`\<[`DatasetItem`](README.md#datasetitem)[]\> +##### promptId -*** +> **promptId**?: `string` -### DatasetItem +##### provider -Represents a utility class with serialization capabilities. +> **provider**?: [`Maybe`](README.md#maybet)\<`string`\> -#### Extends +##### model -- [`Utils`](README.md#utils) +> **model**?: [`Maybe`](README.md#maybet)\<`string`\> -#### Constructors +##### id -##### new DatasetItem(data) +> **id**?: [`Maybe`](README.md#maybet)\<`string`\> -> **new DatasetItem**(`data`): [`DatasetItem`](README.md#datasetitem) +##### tags -###### Parameters +> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> -▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`DatasetItem`](README.md#datasetitem)\> +##### error -###### Returns +> **error**?: [`Maybe`](README.md#maybet)\<`string`\> -[`DatasetItem`](README.md#datasetitem) +##### variables -###### Overrides +> **variables**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> -[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) +##### settings -#### Properties +> **settings**?: [`Maybe`](README.md#maybet)\<[`ILLMSettings`](README.md#illmsettings)\> -##### id +##### tools -> **id**: `string` +> **tools**?: [`Maybe`](README.md#maybet)\<[`ITool`](README.md#itool)[]\> -##### createdAt +##### tokenCount -> **createdAt**: `string` +> **tokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> -##### datasetId +##### inputTokenCount -> **datasetId**: `string` +> **inputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> -##### metadata +##### outputTokenCount -> **metadata**: `Record`\<`string`, `any`\> +> **outputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> -##### input +##### ttFirstToken -> **input**: `Record`\<`string`, `any`\> +> **ttFirstToken**?: [`Maybe`](README.md#maybet)\<`number`\> -##### expectedOutput +##### tokenThroughputInSeconds -> **expectedOutput**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> +> **tokenThroughputInSeconds**?: [`Maybe`](README.md#maybet)\<`number`\> -##### intermediarySteps +##### duration -> **intermediarySteps**: `Record`\<`string`, `any`\>[] +> **duration**?: [`Maybe`](README.md#maybet)\<`number`\> #### Methods @@ -2171,182 +455,165 @@ A dictionary representing the serialized properties of the object. *** -### DatasetExperiment +### CompletionGeneration Represents a utility class with serialization capabilities. #### Extends -- [`Utils`](README.md#utils) +- [`BaseGeneration`](README.md#basegeneration) #### Constructors -##### new DatasetExperiment(api, data) +##### new CompletionGeneration(data) -> **new DatasetExperiment**(`api`, `data`): [`DatasetExperiment`](README.md#datasetexperiment) +> **new CompletionGeneration**(`data`): [`CompletionGeneration`](README.md#completiongeneration) ###### Parameters -▪ **api**: `API` - -▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`DatasetExperiment`](README.md#datasetexperiment)\> +▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`CompletionGeneration`](README.md#completiongeneration)\> ###### Returns -[`DatasetExperiment`](README.md#datasetexperiment) +[`CompletionGeneration`](README.md#completiongeneration) ###### Overrides -[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) +[`BaseGeneration`](README.md#basegeneration).[`constructor`](README.md#constructors-1) #### Properties -##### id - -> **id**: `string` - -##### createdAt - -> **createdAt**: `string` - -##### name - -> **name**: `string` +##### promptId -##### datasetId +> **promptId**?: `string` -> **datasetId**?: `string` +###### Inherited from -##### promptId +[`BaseGeneration`](README.md#basegeneration).[`promptId`](README.md#promptid) -> **promptId**?: `string` +##### provider -##### api +> **provider**?: [`Maybe`](README.md#maybet)\<`string`\> -> **api**: `API` +###### Inherited from -##### params +[`BaseGeneration`](README.md#basegeneration).[`provider`](README.md#provider) -> **params**: `Record`\<`string`, `any`\> \| `Record`\<`string`, `any`\>[] +##### model -##### items +> **model**?: [`Maybe`](README.md#maybet)\<`string`\> -> **items**: [`DatasetExperimentItem`](README.md#datasetexperimentitem)[] +###### Inherited from -#### Methods +[`BaseGeneration`](README.md#basegeneration).[`model`](README.md#model) -##### serialize() +##### id -> **serialize**(): `any` +> **id**?: [`Maybe`](README.md#maybet)\<`string`\> -Serializes the properties of the current instance into a dictionary, excluding the 'api' property. -It handles nested objects that also implement a serialize method. +###### Inherited from -###### Returns +[`BaseGeneration`](README.md#basegeneration).[`id`](README.md#id) -`any` +##### tags -A dictionary representing the serialized properties of the object. +> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> ###### Inherited from -[`Utils`](README.md#utils).[`serialize`](README.md#serialize-3) +[`BaseGeneration`](README.md#basegeneration).[`tags`](README.md#tags) -##### log() +##### error -> **log**(`itemFields`): `Promise`\<[`DatasetExperimentItem`](README.md#datasetexperimentitem)\> +> **error**?: [`Maybe`](README.md#maybet)\<`string`\> -###### Parameters +###### Inherited from -▪ **itemFields**: `Omit`\<[`OmitUtils`](README.md#omitutilst)\<`DatasetExperimentItemFields`\>, `"id"` \| `"datasetExperimentId"`\> +[`BaseGeneration`](README.md#basegeneration).[`error`](README.md#error) -###### Returns +##### variables -`Promise`\<[`DatasetExperimentItem`](README.md#datasetexperimentitem)\> +> **variables**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> -*** +###### Inherited from -### DatasetExperimentItem +[`BaseGeneration`](README.md#basegeneration).[`variables`](README.md#variables) -#### Extends +##### settings -- `DatasetExperimentItemFields` +> **settings**?: [`Maybe`](README.md#maybet)\<[`ILLMSettings`](README.md#illmsettings)\> -#### Constructors +###### Inherited from -##### new DatasetExperimentItem(data) +[`BaseGeneration`](README.md#basegeneration).[`settings`](README.md#settings) -> **new DatasetExperimentItem**(`data`): [`DatasetExperimentItem`](README.md#datasetexperimentitem) +##### tools -###### Parameters +> **tools**?: [`Maybe`](README.md#maybet)\<[`ITool`](README.md#itool)[]\> -▪ **data**: [`DatasetExperimentItemConstructor`](README.md#datasetexperimentitemconstructor) +###### Inherited from -###### Returns +[`BaseGeneration`](README.md#basegeneration).[`tools`](README.md#tools) -[`DatasetExperimentItem`](README.md#datasetexperimentitem) +##### tokenCount -###### Overrides +> **tokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> -DatasetExperimentItemFields.constructor +###### Inherited from -#### Properties +[`BaseGeneration`](README.md#basegeneration).[`tokenCount`](README.md#tokencount) -##### id +##### inputTokenCount -> **id**?: `string` +> **inputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> ###### Inherited from -DatasetExperimentItemFields.id +[`BaseGeneration`](README.md#basegeneration).[`inputTokenCount`](README.md#inputtokencount) -##### datasetExperimentId +##### outputTokenCount -> **datasetExperimentId**: `string` +> **outputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> ###### Inherited from -DatasetExperimentItemFields.datasetExperimentId +[`BaseGeneration`](README.md#basegeneration).[`outputTokenCount`](README.md#outputtokencount) -##### datasetItemId +##### ttFirstToken -> **datasetItemId**?: `string` +> **ttFirstToken**?: [`Maybe`](README.md#maybet)\<`number`\> ###### Inherited from -DatasetExperimentItemFields.datasetItemId +[`BaseGeneration`](README.md#basegeneration).[`ttFirstToken`](README.md#ttfirsttoken) -##### experimentRunId +##### tokenThroughputInSeconds -> **experimentRunId**?: `string` +> **tokenThroughputInSeconds**?: [`Maybe`](README.md#maybet)\<`number`\> ###### Inherited from -DatasetExperimentItemFields.experimentRunId +[`BaseGeneration`](README.md#basegeneration).[`tokenThroughputInSeconds`](README.md#tokenthroughputinseconds) -##### scores +##### duration -> **scores**: [`ScoreConstructor`](README.md#scoreconstructor)[] +> **duration**?: [`Maybe`](README.md#maybet)\<`number`\> ###### Inherited from -DatasetExperimentItemFields.scores - -##### input - -> **input**?: `Record`\<`string`, `any`\> +[`BaseGeneration`](README.md#basegeneration).[`duration`](README.md#duration) -###### Inherited from +##### type -DatasetExperimentItemFields.input +> **type**?: [`GenerationType`](README.md#generationtype) = `'COMPLETION'` -##### output +##### prompt -> **output**?: `Record`\<`string`, `any`\> +> **prompt**?: [`Maybe`](README.md#maybet)\<`string`\> -###### Inherited from +##### completion -DatasetExperimentItemFields.output +> **completion**?: [`Maybe`](README.md#maybet)\<`string`\> #### Methods @@ -2365,159 +632,169 @@ A dictionary representing the serialized properties of the object. ###### Inherited from -DatasetExperimentItemFields.serialize +[`BaseGeneration`](README.md#basegeneration).[`serialize`](README.md#serialize) *** -### Prompt +### ChatGeneration + +Represents a utility class with serialization capabilities. #### Extends -- `PromptFields` +- [`BaseGeneration`](README.md#basegeneration) #### Constructors -##### new Prompt(api, data) - -> **new Prompt**(`api`, `data`): [`Prompt`](README.md#prompt-1) +##### new ChatGeneration(data) -Constructs a new Prompt instance. +> **new ChatGeneration**(`data`): [`ChatGeneration`](README.md#chatgeneration) ###### Parameters -▪ **api**: `API` - -The API instance to interact with backend services. - -▪ **data**: [`PromptConstructor`](README.md#promptconstructor) - -The initial data for the prompt. +▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`ChatGeneration`](README.md#chatgeneration)\> ###### Returns -[`Prompt`](README.md#prompt-1) +[`ChatGeneration`](README.md#chatgeneration) ###### Overrides -PromptFields.constructor +[`BaseGeneration`](README.md#basegeneration).[`constructor`](README.md#constructors-1) #### Properties -##### id +##### promptId -> **id**: `string` +> **promptId**?: `string` ###### Inherited from -PromptFields.id +[`BaseGeneration`](README.md#basegeneration).[`promptId`](README.md#promptid) -##### type +##### provider -> **type**: [`GenerationType`](README.md#generationtype) +> **provider**?: [`Maybe`](README.md#maybet)\<`string`\> ###### Inherited from -PromptFields.type +[`BaseGeneration`](README.md#basegeneration).[`provider`](README.md#provider) -##### createdAt +##### model -> **createdAt**: `string` +> **model**?: [`Maybe`](README.md#maybet)\<`string`\> ###### Inherited from -PromptFields.createdAt +[`BaseGeneration`](README.md#basegeneration).[`model`](README.md#model) -##### name +##### id -> **name**: `string` +> **id**?: [`Maybe`](README.md#maybet)\<`string`\> ###### Inherited from -PromptFields.name +[`BaseGeneration`](README.md#basegeneration).[`id`](README.md#id) -##### version +##### tags -> **version**: `number` +> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> ###### Inherited from -PromptFields.version +[`BaseGeneration`](README.md#basegeneration).[`tags`](README.md#tags) -##### versionDesc +##### error -> **versionDesc**?: [`Maybe`](README.md#maybet)\<`string`\> +> **error**?: [`Maybe`](README.md#maybet)\<`string`\> ###### Inherited from -PromptFields.versionDesc +[`BaseGeneration`](README.md#basegeneration).[`error`](README.md#error) -##### metadata +##### variables -> **metadata**: `Record`\<`string`, `any`\> +> **variables**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> ###### Inherited from -PromptFields.metadata +[`BaseGeneration`](README.md#basegeneration).[`variables`](README.md#variables) -##### items +##### settings -> **items**: [`OmitUtils`](README.md#omitutilst)\<[`DatasetItem`](README.md#datasetitem)\>[] +> **settings**?: [`Maybe`](README.md#maybet)\<[`ILLMSettings`](README.md#illmsettings)\> ###### Inherited from -PromptFields.items +[`BaseGeneration`](README.md#basegeneration).[`settings`](README.md#settings) -##### variablesDefaultValues +##### tools -> **variablesDefaultValues**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> +> **tools**?: [`Maybe`](README.md#maybet)\<[`ITool`](README.md#itool)[]\> ###### Inherited from -PromptFields.variablesDefaultValues +[`BaseGeneration`](README.md#basegeneration).[`tools`](README.md#tools) -##### templateMessages +##### tokenCount -> **templateMessages**: [`IGenerationMessage`](README.md#igenerationmessage)[] +> **tokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> ###### Inherited from -PromptFields.templateMessages +[`BaseGeneration`](README.md#basegeneration).[`tokenCount`](README.md#tokencount) -##### tools +##### inputTokenCount -> **tools**?: `ChatCompletionTool`[] +> **inputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> ###### Inherited from -PromptFields.tools +[`BaseGeneration`](README.md#basegeneration).[`inputTokenCount`](README.md#inputtokencount) -##### provider +##### outputTokenCount -> **provider**: `string` +> **outputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> ###### Inherited from -PromptFields.provider +[`BaseGeneration`](README.md#basegeneration).[`outputTokenCount`](README.md#outputtokencount) -##### settings +##### ttFirstToken -> **settings**: [`IProviderSettings`](README.md#iprovidersettings) +> **ttFirstToken**?: [`Maybe`](README.md#maybet)\<`number`\> ###### Inherited from -PromptFields.settings +[`BaseGeneration`](README.md#basegeneration).[`ttFirstToken`](README.md#ttfirsttoken) -##### variables +##### tokenThroughputInSeconds -> **variables**: [`IPromptVariableDefinition`](README.md#ipromptvariabledefinition)[] +> **tokenThroughputInSeconds**?: [`Maybe`](README.md#maybet)\<`number`\> ###### Inherited from -PromptFields.variables +[`BaseGeneration`](README.md#basegeneration).[`tokenThroughputInSeconds`](README.md#tokenthroughputinseconds) -##### api +##### duration -> **api**: `API` +> **duration**?: [`Maybe`](README.md#maybet)\<`number`\> + +###### Inherited from + +[`BaseGeneration`](README.md#basegeneration).[`duration`](README.md#duration) + +##### type + +> **type**?: [`GenerationType`](README.md#generationtype) = `'CHAT'` + +##### messages + +> **messages**?: [`Maybe`](README.md#maybet)\<[`IGenerationMessage`](README.md#igenerationmessage)[]\> = `[]` + +##### messageCompletion + +> **messageCompletion**?: [`Maybe`](README.md#maybet)\<[`IGenerationMessage`](README.md#igenerationmessage)\> #### Methods @@ -2536,53 +813,104 @@ A dictionary representing the serialized properties of the object. ###### Inherited from -PromptFields.serialize +[`BaseGeneration`](README.md#basegeneration).[`serialize`](README.md#serialize) -##### formatMessages() +*** -> **formatMessages**(`variables`?): `ChatCompletionMessageParam`[] +### Utils -Formats the prompt's template messages with the given variables. +Represents a utility class with serialization capabilities. -###### Parameters +#### Extended By + +- [`User`](README.md#user) +- [`BaseGeneration`](README.md#basegeneration) + +#### Constructors + +##### new Utils() -▪ **variables?**: `Record`\<`string`, `any`\> +> **new Utils**(): [`Utils`](README.md#utils) + +###### Returns + +[`Utils`](README.md#utils) + +#### Methods + +##### serialize() + +> **serialize**(): `any` -Optional variables to resolve in the template messages. +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. ###### Returns -`ChatCompletionMessageParam`[] +`any` + +A dictionary representing the serialized properties of the object. + +*** + +### User + +Represents a user with optional metadata and identifier. + +#### Extends + +- [`Utils`](README.md#utils) -An array of formatted chat completion messages. +#### Constructors -##### format() +##### new User(data) -> **format**(`variables`?): `ChatCompletionMessageParam`[] +> **new User**(`data`): [`User`](README.md#user) ###### Parameters -▪ **variables?**: `Record`\<`string`, `any`\> +▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`User`](README.md#user)\> ###### Returns -`ChatCompletionMessageParam`[] +[`User`](README.md#user) + +###### Overrides + +[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) + +#### Properties + +##### id + +> **id**?: [`Maybe`](README.md#maybet)\<`string`\> + +##### identifier + +> **identifier**: `string` + +##### metadata -###### Deprecated +> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> -Please use `formatMessages` instead. +#### Methods -##### toLangchainChatPromptTemplate() +##### serialize() -> **toLangchainChatPromptTemplate**(): `CustomChatPromptTemplate` +> **serialize**(): `any` -Converts the prompt's template messages into a Langchain chat prompt template. +Serializes the properties of the current instance into a dictionary, excluding the 'api' property. +It handles nested objects that also implement a serialize method. ###### Returns -`CustomChatPromptTemplate` +`any` + +A dictionary representing the serialized properties of the object. + +###### Inherited from -A custom chat prompt template configured with the prompt's data. +[`Utils`](README.md#utils).[`serialize`](README.md#serialize-3) ## Interfaces @@ -2692,59 +1020,23 @@ A custom chat prompt template configured with the prompt's data. > **function**: [`IFunction`](README.md#ifunction) -*** - -### IPromptVariableDefinition - -#### Properties - -##### name - -> **name**: `string` - -##### language - -> **language**: `"json"` \| `"plaintext"` - -*** - -### IProviderSettings - -#### Properties - -##### provider - -> **provider**: `string` - -##### model - -> **model**: `string` - -##### frequency\_penalty - -> **frequency\_penalty**: `number` - -##### max\_tokens - -> **max\_tokens**: `number` - -##### presence\_penalty +## Type Aliases -> **presence\_penalty**: `number` +### OpenAIGlobalOptions -##### stop +> **OpenAIGlobalOptions**: `object` -> **stop**?: `string`[] +#### Type declaration -##### temperature +##### tags -> **temperature**: `number` +> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> -##### top\_p +##### metadata -> **top\_p**: `number` +> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> -## Type Aliases +*** ### GenerationMessageRole @@ -2772,7 +1064,7 @@ A custom chat prompt template configured with the prompt's data. ### PersistedGeneration -> **PersistedGeneration**: [`Generation`](README.md#generation-2) & `object` +> **PersistedGeneration**: [`Generation`](README.md#generation) & `object` #### Type declaration @@ -2782,22 +1074,6 @@ A custom chat prompt template configured with the prompt's data. *** -### OpenAIGlobalOptions - -> **OpenAIGlobalOptions**: `object` - -#### Type declaration - -##### tags - -> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> - -##### metadata - -> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> - -*** - ### Maybe`` > **Maybe**\<`T`\>: `T` \| `null` \| `undefined` @@ -2868,65 +1144,19 @@ A custom chat prompt template configured with the prompt's data. > **pageInfo**: [`PageInfo`](README.md#pageinfo) -*** - -### ScoreType - -> **ScoreType**: `"HUMAN"` \| `"AI"` - -*** - -### ScoreConstructor - -> **ScoreConstructor**: [`OmitUtils`](README.md#omitutilst)\<`ScoreFields`\> - -*** - -### CleanThreadFields - -> **CleanThreadFields**: [`OmitUtils`](README.md#omitutilst)\<`ThreadFields`\> - -*** - -### ThreadConstructor - -> **ThreadConstructor**: `Omit`\<[`CleanThreadFields`](README.md#cleanthreadfields), `"id"`\> & `Partial`\<`Pick`\<[`CleanThreadFields`](README.md#cleanthreadfields), `"id"`\>\> - -*** - -### StepType - -> **StepType**: `"assistant_message"` \| `"embedding"` \| `"llm"` \| `"rerank"` \| `"retrieval"` \| `"run"` \| `"system_message"` \| `"tool"` \| `"undefined"` \| `"user_message"` - -*** - -### StepConstructor - -> **StepConstructor**: [`OmitUtils`](README.md#omitutilst)\<`StepFields`\> - -*** - -### DatasetType - -> **DatasetType**: `"key_value"` \| `"generation"` - -*** - -### DatasetConstructor - -> **DatasetConstructor**: [`OmitUtils`](README.md#omitutilst)\<`DatasetFields`\> +## Functions -*** +### isPlainObject() -### DatasetExperimentItemConstructor +> **isPlainObject**(`value`): `value is Record` -> **DatasetExperimentItemConstructor**: [`OmitUtils`](README.md#omitutilst)\<`DatasetExperimentItemFields`\> +#### Parameters -*** +▪ **value**: `unknown` -### PromptConstructor +#### Returns -> **PromptConstructor**: [`OmitUtils`](README.md#omitutilst)\<`PromptFields`\> +`value is Record` *** From db60e34c7e656fae86e03a738951797cb003df54 Mon Sep 17 00:00:00 2001 From: Damien BUTY Date: Tue, 30 Jul 2024 11:48:59 +0200 Subject: [PATCH 4/9] docs: rename doc to api.md --- docs/{README.md => api.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{README.md => api.md} (100%) diff --git a/docs/README.md b/docs/api.md similarity index 100% rename from docs/README.md rename to docs/api.md From 7f6c0d346c9ad331a177423523f7bbc5a648341a Mon Sep 17 00:00:00 2001 From: Damien BUTY Date: Tue, 30 Jul 2024 12:27:06 +0200 Subject: [PATCH 5/9] chore: moar jsdocs --- src/evaluation/dataset.ts | 14 +++++++++++--- src/evaluation/experiment-run.ts | 7 +++++++ src/index.ts | 29 ++++++++++++++++++++++++++++- src/observability/step.ts | 10 ++-------- 4 files changed, 48 insertions(+), 12 deletions(-) diff --git a/src/evaluation/dataset.ts b/src/evaluation/dataset.ts index 12f52c7..dd4012b 100644 --- a/src/evaluation/dataset.ts +++ b/src/evaluation/dataset.ts @@ -146,6 +146,11 @@ export class Dataset extends DatasetFields { return item; } + /** + * Adds multiple generations to the dataset. + * @param generationIds - The IDs of the steps to add. + * @returns The added dataset items. + */ public async addGenerations(generationIds?: string[]) { if (generationIds == undefined || generationIds?.length === 0) { return []; @@ -194,7 +199,6 @@ export class DatasetExperiment extends Utils { api: API; params!: Record | Array>; items!: DatasetExperimentItem[]; - constructor(api: API, data: OmitUtils) { super(); this.api = api; @@ -204,14 +208,18 @@ export class DatasetExperiment extends Utils { } } + /** + * Logs an item in the dataset experiment. + * @param itemFields the data for this item + * @returns the created item + */ async log( itemFields: Omit< OmitUtils, 'id' | 'datasetExperimentId' > ) { - const currentStore = this.api.client.store.getStore(); - const experimentRunId = currentStore?.currentExperimentRunId; + const experimentRunId = this.api.client._currentExperimentRunId(); const datasetExperimentItem = new DatasetExperimentItem({ ...itemFields, diff --git a/src/evaluation/experiment-run.ts b/src/evaluation/experiment-run.ts index de92aa4..bb8bac4 100644 --- a/src/evaluation/experiment-run.ts +++ b/src/evaluation/experiment-run.ts @@ -24,6 +24,13 @@ export class ExperimentRun extends Step { this.api = client.api; } + /** + * Wraps the provided callback in a new run with the environment set to "experiment". + * Experiment runs are filtered out of the Literal AI UI. + * @param cb The callback to wrap in a new step. + * @param updateStep Optional update to the step data after the callback is executed. + * @returns The output of the callback. + */ async wrap( cb: (step: Step) => Output | Promise, updateStep?: diff --git a/src/index.ts b/src/index.ts index 216d52c..38430a2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,7 +16,7 @@ export type * from './instrumentation'; type StoredContext = { currentThread: Thread | null; currentStep: Step | null; - currentExperimentRunId?: string | null; + currentExperimentRunId: string | null; }; const storage = new AsyncLocalStorage(); @@ -120,6 +120,16 @@ export class LiteralClient { return store?.currentStep || null; } + /** + * Returns the current experiment from the context or null if none. + * @returns The current experiment, if any. + */ + _currentExperimentRunId(): string | null { + const store = storage.getStore(); + + return store?.currentExperimentRunId || null; + } + /** * Gets the current thread from the context. * WARNING : this will throw if run outside of a thread context. @@ -153,4 +163,21 @@ export class LiteralClient { return store.currentStep; } + + /** + * Gets the current experiment run ID from the context. + * WARNING : this will throw if run outside of an experiment context. + * @returns The current experiment, if any. + */ + getCurrentExperimentRunId(): string { + const store = storage.getStore(); + + if (!store?.currentExperimentRunId) { + throw new Error( + 'Literal AI SDK : tried to access current experiment outside of a context.' + ); + } + + return store.currentExperimentRunId; + } } diff --git a/src/observability/step.ts b/src/observability/step.ts index 08fb1b2..40f6dc7 100644 --- a/src/observability/step.ts +++ b/src/observability/step.ts @@ -74,14 +74,8 @@ export class Step extends StepFields { } // Automatically assign parent thread & step if there are any in the store. - const store = this.client.store.getStore(); - - if (store?.currentThread) { - this.threadId = store.currentThread.id; - } - if (store?.currentStep) { - this.parentId = store.currentStep.id; - } + this.threadId = this.client._currentThread()?.id; + this.parentId = this.client._currentStep()?.id; // Set the creation and start time to the current time if not provided. if (!this.createdAt) { From 1136d9cf9323b51434b550997d626f7bd290dfbd Mon Sep 17 00:00:00 2001 From: Damien BUTY Date: Tue, 30 Jul 2024 12:29:07 +0200 Subject: [PATCH 6/9] chore: regenerate typedoc --- docs/api.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/api.md b/docs/api.md index 4ec6a49..f01e9bf 100644 --- a/docs/api.md +++ b/docs/api.md @@ -322,6 +322,18 @@ Returns the current step from the context or null if none. The current step, if any. +##### \_currentExperimentRunId() + +> **\_currentExperimentRunId**(): `null` \| `string` + +Returns the current experiment from the context or null if none. + +###### Returns + +`null` \| `string` + +The current experiment, if any. + ##### getCurrentThread() > **getCurrentThread**(): `Thread` @@ -348,6 +360,19 @@ WARNING : this will throw if run outside of a step context. The current step, if any. +##### getCurrentExperimentRunId() + +> **getCurrentExperimentRunId**(): `string` + +Gets the current experiment run ID from the context. +WARNING : this will throw if run outside of an experiment context. + +###### Returns + +`string` + +The current experiment, if any. + *** ### BaseGeneration From 89471b9e646e94fb5960ceee8fa2ace85d830f44 Mon Sep 17 00:00:00 2001 From: Damien BUTY Date: Wed, 31 Jul 2024 19:55:39 +0200 Subject: [PATCH 7/9] fix: review --- docs/api.md | 242 ++++++++++++++++++++-------------------- src/evaluation/score.ts | 4 +- src/index.ts | 2 +- 3 files changed, 124 insertions(+), 124 deletions(-) diff --git a/docs/api.md b/docs/api.md index f01e9bf..b1a89b1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -12,7 +12,7 @@ ##### new LiteralClient(options) -> **new LiteralClient**(`options`): [`LiteralClient`](README.md#literalclient) +> **new LiteralClient**(`options`): [`LiteralClient`](api.md#literalclient) Initialize a new Literal AI Client. @@ -26,9 +26,9 @@ The API key to use for the Literal AI API. Defaults to the LITERAL_API_KEY envir ▪ **options.apiUrl?**: `string` -The URL of the Literal AI API. Defaults to the LITERAL_API_URL environment variable. +The URL of the Literal AI API. Defaults to the LITERAL_API_URL env var, or https://cloud.getliteral.ai. -▪ **options.environment?**: [`Environment`](README.md#environment) +▪ **options.environment?**: [`Environment`](api.md#environment) The environment to use for the Literal AI API. @@ -38,7 +38,7 @@ If set to true, no call will be made to the Literal AI API. ###### Returns -[`LiteralClient`](README.md#literalclient) +[`LiteralClient`](api.md#literalclient) A new LiteralClient instance. @@ -86,7 +86,7 @@ Its functions will be augmented with `literalaiTags` and `literalaiMetadata` opt ###### Parameters -▪ **options?**: [`OpenAIGlobalOptions`](README.md#openaiglobaloptions) +▪ **options?**: [`OpenAIGlobalOptions`](api.md#openaiglobaloptions) ###### Returns @@ -381,21 +381,21 @@ Represents a utility class with serialization capabilities. #### Extends -- [`Utils`](README.md#utils) +- [`Utils`](api.md#utils) #### Constructors ##### new BaseGeneration() -> **new BaseGeneration**(): [`BaseGeneration`](README.md#basegeneration) +> **new BaseGeneration**(): [`BaseGeneration`](api.md#basegeneration) ###### Returns -[`BaseGeneration`](README.md#basegeneration) +[`BaseGeneration`](api.md#basegeneration) ###### Inherited from -[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) +[`Utils`](api.md#utils).[`constructor`](api.md#constructors-4) #### Properties @@ -405,59 +405,59 @@ Represents a utility class with serialization capabilities. ##### provider -> **provider**?: [`Maybe`](README.md#maybet)\<`string`\> +> **provider**?: [`Maybe`](api.md#maybet)\<`string`\> ##### model -> **model**?: [`Maybe`](README.md#maybet)\<`string`\> +> **model**?: [`Maybe`](api.md#maybet)\<`string`\> ##### id -> **id**?: [`Maybe`](README.md#maybet)\<`string`\> +> **id**?: [`Maybe`](api.md#maybet)\<`string`\> ##### tags -> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> +> **tags**?: [`Maybe`](api.md#maybet)\<`string`[]\> ##### error -> **error**?: [`Maybe`](README.md#maybet)\<`string`\> +> **error**?: [`Maybe`](api.md#maybet)\<`string`\> ##### variables -> **variables**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> +> **variables**?: [`Maybe`](api.md#maybet)\<`Record`\<`string`, `any`\>\> ##### settings -> **settings**?: [`Maybe`](README.md#maybet)\<[`ILLMSettings`](README.md#illmsettings)\> +> **settings**?: [`Maybe`](api.md#maybet)\<[`ILLMSettings`](api.md#illmsettings)\> ##### tools -> **tools**?: [`Maybe`](README.md#maybet)\<[`ITool`](README.md#itool)[]\> +> **tools**?: [`Maybe`](api.md#maybet)\<[`ITool`](api.md#itool)[]\> ##### tokenCount -> **tokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> +> **tokenCount**?: [`Maybe`](api.md#maybet)\<`number`\> ##### inputTokenCount -> **inputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> +> **inputTokenCount**?: [`Maybe`](api.md#maybet)\<`number`\> ##### outputTokenCount -> **outputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> +> **outputTokenCount**?: [`Maybe`](api.md#maybet)\<`number`\> ##### ttFirstToken -> **ttFirstToken**?: [`Maybe`](README.md#maybet)\<`number`\> +> **ttFirstToken**?: [`Maybe`](api.md#maybet)\<`number`\> ##### tokenThroughputInSeconds -> **tokenThroughputInSeconds**?: [`Maybe`](README.md#maybet)\<`number`\> +> **tokenThroughputInSeconds**?: [`Maybe`](api.md#maybet)\<`number`\> ##### duration -> **duration**?: [`Maybe`](README.md#maybet)\<`number`\> +> **duration**?: [`Maybe`](api.md#maybet)\<`number`\> #### Methods @@ -476,7 +476,7 @@ A dictionary representing the serialized properties of the object. ###### Inherited from -[`Utils`](README.md#utils).[`serialize`](README.md#serialize-3) +[`Utils`](api.md#utils).[`serialize`](api.md#serialize-3) *** @@ -486,25 +486,25 @@ Represents a utility class with serialization capabilities. #### Extends -- [`BaseGeneration`](README.md#basegeneration) +- [`BaseGeneration`](api.md#basegeneration) #### Constructors ##### new CompletionGeneration(data) -> **new CompletionGeneration**(`data`): [`CompletionGeneration`](README.md#completiongeneration) +> **new CompletionGeneration**(`data`): [`CompletionGeneration`](api.md#completiongeneration) ###### Parameters -▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`CompletionGeneration`](README.md#completiongeneration)\> +▪ **data**: [`OmitUtils`](api.md#omitutilst)\<[`CompletionGeneration`](api.md#completiongeneration)\> ###### Returns -[`CompletionGeneration`](README.md#completiongeneration) +[`CompletionGeneration`](api.md#completiongeneration) ###### Overrides -[`BaseGeneration`](README.md#basegeneration).[`constructor`](README.md#constructors-1) +[`BaseGeneration`](api.md#basegeneration).[`constructor`](api.md#constructors-1) #### Properties @@ -514,131 +514,131 @@ Represents a utility class with serialization capabilities. ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`promptId`](README.md#promptid) +[`BaseGeneration`](api.md#basegeneration).[`promptId`](api.md#promptid) ##### provider -> **provider**?: [`Maybe`](README.md#maybet)\<`string`\> +> **provider**?: [`Maybe`](api.md#maybet)\<`string`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`provider`](README.md#provider) +[`BaseGeneration`](api.md#basegeneration).[`provider`](api.md#provider) ##### model -> **model**?: [`Maybe`](README.md#maybet)\<`string`\> +> **model**?: [`Maybe`](api.md#maybet)\<`string`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`model`](README.md#model) +[`BaseGeneration`](api.md#basegeneration).[`model`](api.md#model) ##### id -> **id**?: [`Maybe`](README.md#maybet)\<`string`\> +> **id**?: [`Maybe`](api.md#maybet)\<`string`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`id`](README.md#id) +[`BaseGeneration`](api.md#basegeneration).[`id`](api.md#id) ##### tags -> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> +> **tags**?: [`Maybe`](api.md#maybet)\<`string`[]\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`tags`](README.md#tags) +[`BaseGeneration`](api.md#basegeneration).[`tags`](api.md#tags) ##### error -> **error**?: [`Maybe`](README.md#maybet)\<`string`\> +> **error**?: [`Maybe`](api.md#maybet)\<`string`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`error`](README.md#error) +[`BaseGeneration`](api.md#basegeneration).[`error`](api.md#error) ##### variables -> **variables**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> +> **variables**?: [`Maybe`](api.md#maybet)\<`Record`\<`string`, `any`\>\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`variables`](README.md#variables) +[`BaseGeneration`](api.md#basegeneration).[`variables`](api.md#variables) ##### settings -> **settings**?: [`Maybe`](README.md#maybet)\<[`ILLMSettings`](README.md#illmsettings)\> +> **settings**?: [`Maybe`](api.md#maybet)\<[`ILLMSettings`](api.md#illmsettings)\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`settings`](README.md#settings) +[`BaseGeneration`](api.md#basegeneration).[`settings`](api.md#settings) ##### tools -> **tools**?: [`Maybe`](README.md#maybet)\<[`ITool`](README.md#itool)[]\> +> **tools**?: [`Maybe`](api.md#maybet)\<[`ITool`](api.md#itool)[]\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`tools`](README.md#tools) +[`BaseGeneration`](api.md#basegeneration).[`tools`](api.md#tools) ##### tokenCount -> **tokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> +> **tokenCount**?: [`Maybe`](api.md#maybet)\<`number`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`tokenCount`](README.md#tokencount) +[`BaseGeneration`](api.md#basegeneration).[`tokenCount`](api.md#tokencount) ##### inputTokenCount -> **inputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> +> **inputTokenCount**?: [`Maybe`](api.md#maybet)\<`number`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`inputTokenCount`](README.md#inputtokencount) +[`BaseGeneration`](api.md#basegeneration).[`inputTokenCount`](api.md#inputtokencount) ##### outputTokenCount -> **outputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> +> **outputTokenCount**?: [`Maybe`](api.md#maybet)\<`number`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`outputTokenCount`](README.md#outputtokencount) +[`BaseGeneration`](api.md#basegeneration).[`outputTokenCount`](api.md#outputtokencount) ##### ttFirstToken -> **ttFirstToken**?: [`Maybe`](README.md#maybet)\<`number`\> +> **ttFirstToken**?: [`Maybe`](api.md#maybet)\<`number`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`ttFirstToken`](README.md#ttfirsttoken) +[`BaseGeneration`](api.md#basegeneration).[`ttFirstToken`](api.md#ttfirsttoken) ##### tokenThroughputInSeconds -> **tokenThroughputInSeconds**?: [`Maybe`](README.md#maybet)\<`number`\> +> **tokenThroughputInSeconds**?: [`Maybe`](api.md#maybet)\<`number`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`tokenThroughputInSeconds`](README.md#tokenthroughputinseconds) +[`BaseGeneration`](api.md#basegeneration).[`tokenThroughputInSeconds`](api.md#tokenthroughputinseconds) ##### duration -> **duration**?: [`Maybe`](README.md#maybet)\<`number`\> +> **duration**?: [`Maybe`](api.md#maybet)\<`number`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`duration`](README.md#duration) +[`BaseGeneration`](api.md#basegeneration).[`duration`](api.md#duration) ##### type -> **type**?: [`GenerationType`](README.md#generationtype) = `'COMPLETION'` +> **type**?: [`GenerationType`](api.md#generationtype) = `'COMPLETION'` ##### prompt -> **prompt**?: [`Maybe`](README.md#maybet)\<`string`\> +> **prompt**?: [`Maybe`](api.md#maybet)\<`string`\> ##### completion -> **completion**?: [`Maybe`](README.md#maybet)\<`string`\> +> **completion**?: [`Maybe`](api.md#maybet)\<`string`\> #### Methods @@ -657,7 +657,7 @@ A dictionary representing the serialized properties of the object. ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`serialize`](README.md#serialize) +[`BaseGeneration`](api.md#basegeneration).[`serialize`](api.md#serialize) *** @@ -667,25 +667,25 @@ Represents a utility class with serialization capabilities. #### Extends -- [`BaseGeneration`](README.md#basegeneration) +- [`BaseGeneration`](api.md#basegeneration) #### Constructors ##### new ChatGeneration(data) -> **new ChatGeneration**(`data`): [`ChatGeneration`](README.md#chatgeneration) +> **new ChatGeneration**(`data`): [`ChatGeneration`](api.md#chatgeneration) ###### Parameters -▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`ChatGeneration`](README.md#chatgeneration)\> +▪ **data**: [`OmitUtils`](api.md#omitutilst)\<[`ChatGeneration`](api.md#chatgeneration)\> ###### Returns -[`ChatGeneration`](README.md#chatgeneration) +[`ChatGeneration`](api.md#chatgeneration) ###### Overrides -[`BaseGeneration`](README.md#basegeneration).[`constructor`](README.md#constructors-1) +[`BaseGeneration`](api.md#basegeneration).[`constructor`](api.md#constructors-1) #### Properties @@ -695,131 +695,131 @@ Represents a utility class with serialization capabilities. ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`promptId`](README.md#promptid) +[`BaseGeneration`](api.md#basegeneration).[`promptId`](api.md#promptid) ##### provider -> **provider**?: [`Maybe`](README.md#maybet)\<`string`\> +> **provider**?: [`Maybe`](api.md#maybet)\<`string`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`provider`](README.md#provider) +[`BaseGeneration`](api.md#basegeneration).[`provider`](api.md#provider) ##### model -> **model**?: [`Maybe`](README.md#maybet)\<`string`\> +> **model**?: [`Maybe`](api.md#maybet)\<`string`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`model`](README.md#model) +[`BaseGeneration`](api.md#basegeneration).[`model`](api.md#model) ##### id -> **id**?: [`Maybe`](README.md#maybet)\<`string`\> +> **id**?: [`Maybe`](api.md#maybet)\<`string`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`id`](README.md#id) +[`BaseGeneration`](api.md#basegeneration).[`id`](api.md#id) ##### tags -> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> +> **tags**?: [`Maybe`](api.md#maybet)\<`string`[]\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`tags`](README.md#tags) +[`BaseGeneration`](api.md#basegeneration).[`tags`](api.md#tags) ##### error -> **error**?: [`Maybe`](README.md#maybet)\<`string`\> +> **error**?: [`Maybe`](api.md#maybet)\<`string`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`error`](README.md#error) +[`BaseGeneration`](api.md#basegeneration).[`error`](api.md#error) ##### variables -> **variables**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> +> **variables**?: [`Maybe`](api.md#maybet)\<`Record`\<`string`, `any`\>\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`variables`](README.md#variables) +[`BaseGeneration`](api.md#basegeneration).[`variables`](api.md#variables) ##### settings -> **settings**?: [`Maybe`](README.md#maybet)\<[`ILLMSettings`](README.md#illmsettings)\> +> **settings**?: [`Maybe`](api.md#maybet)\<[`ILLMSettings`](api.md#illmsettings)\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`settings`](README.md#settings) +[`BaseGeneration`](api.md#basegeneration).[`settings`](api.md#settings) ##### tools -> **tools**?: [`Maybe`](README.md#maybet)\<[`ITool`](README.md#itool)[]\> +> **tools**?: [`Maybe`](api.md#maybet)\<[`ITool`](api.md#itool)[]\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`tools`](README.md#tools) +[`BaseGeneration`](api.md#basegeneration).[`tools`](api.md#tools) ##### tokenCount -> **tokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> +> **tokenCount**?: [`Maybe`](api.md#maybet)\<`number`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`tokenCount`](README.md#tokencount) +[`BaseGeneration`](api.md#basegeneration).[`tokenCount`](api.md#tokencount) ##### inputTokenCount -> **inputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> +> **inputTokenCount**?: [`Maybe`](api.md#maybet)\<`number`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`inputTokenCount`](README.md#inputtokencount) +[`BaseGeneration`](api.md#basegeneration).[`inputTokenCount`](api.md#inputtokencount) ##### outputTokenCount -> **outputTokenCount**?: [`Maybe`](README.md#maybet)\<`number`\> +> **outputTokenCount**?: [`Maybe`](api.md#maybet)\<`number`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`outputTokenCount`](README.md#outputtokencount) +[`BaseGeneration`](api.md#basegeneration).[`outputTokenCount`](api.md#outputtokencount) ##### ttFirstToken -> **ttFirstToken**?: [`Maybe`](README.md#maybet)\<`number`\> +> **ttFirstToken**?: [`Maybe`](api.md#maybet)\<`number`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`ttFirstToken`](README.md#ttfirsttoken) +[`BaseGeneration`](api.md#basegeneration).[`ttFirstToken`](api.md#ttfirsttoken) ##### tokenThroughputInSeconds -> **tokenThroughputInSeconds**?: [`Maybe`](README.md#maybet)\<`number`\> +> **tokenThroughputInSeconds**?: [`Maybe`](api.md#maybet)\<`number`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`tokenThroughputInSeconds`](README.md#tokenthroughputinseconds) +[`BaseGeneration`](api.md#basegeneration).[`tokenThroughputInSeconds`](api.md#tokenthroughputinseconds) ##### duration -> **duration**?: [`Maybe`](README.md#maybet)\<`number`\> +> **duration**?: [`Maybe`](api.md#maybet)\<`number`\> ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`duration`](README.md#duration) +[`BaseGeneration`](api.md#basegeneration).[`duration`](api.md#duration) ##### type -> **type**?: [`GenerationType`](README.md#generationtype) = `'CHAT'` +> **type**?: [`GenerationType`](api.md#generationtype) = `'CHAT'` ##### messages -> **messages**?: [`Maybe`](README.md#maybet)\<[`IGenerationMessage`](README.md#igenerationmessage)[]\> = `[]` +> **messages**?: [`Maybe`](api.md#maybet)\<[`IGenerationMessage`](api.md#igenerationmessage)[]\> = `[]` ##### messageCompletion -> **messageCompletion**?: [`Maybe`](README.md#maybet)\<[`IGenerationMessage`](README.md#igenerationmessage)\> +> **messageCompletion**?: [`Maybe`](api.md#maybet)\<[`IGenerationMessage`](api.md#igenerationmessage)\> #### Methods @@ -838,7 +838,7 @@ A dictionary representing the serialized properties of the object. ###### Inherited from -[`BaseGeneration`](README.md#basegeneration).[`serialize`](README.md#serialize) +[`BaseGeneration`](api.md#basegeneration).[`serialize`](api.md#serialize) *** @@ -848,18 +848,18 @@ Represents a utility class with serialization capabilities. #### Extended By -- [`User`](README.md#user) -- [`BaseGeneration`](README.md#basegeneration) +- [`User`](api.md#user) +- [`BaseGeneration`](api.md#basegeneration) #### Constructors ##### new Utils() -> **new Utils**(): [`Utils`](README.md#utils) +> **new Utils**(): [`Utils`](api.md#utils) ###### Returns -[`Utils`](README.md#utils) +[`Utils`](api.md#utils) #### Methods @@ -884,31 +884,31 @@ Represents a user with optional metadata and identifier. #### Extends -- [`Utils`](README.md#utils) +- [`Utils`](api.md#utils) #### Constructors ##### new User(data) -> **new User**(`data`): [`User`](README.md#user) +> **new User**(`data`): [`User`](api.md#user) ###### Parameters -▪ **data**: [`OmitUtils`](README.md#omitutilst)\<[`User`](README.md#user)\> +▪ **data**: [`OmitUtils`](api.md#omitutilst)\<[`User`](api.md#user)\> ###### Returns -[`User`](README.md#user) +[`User`](api.md#user) ###### Overrides -[`Utils`](README.md#utils).[`constructor`](README.md#constructors-4) +[`Utils`](api.md#utils).[`constructor`](api.md#constructors-4) #### Properties ##### id -> **id**?: [`Maybe`](README.md#maybet)\<`string`\> +> **id**?: [`Maybe`](api.md#maybet)\<`string`\> ##### identifier @@ -916,7 +916,7 @@ Represents a user with optional metadata and identifier. ##### metadata -> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> +> **metadata**?: [`Maybe`](api.md#maybet)\<`Record`\<`string`, `any`\>\> #### Methods @@ -935,7 +935,7 @@ A dictionary representing the serialized properties of the object. ###### Inherited from -[`Utils`](README.md#utils).[`serialize`](README.md#serialize-3) +[`Utils`](api.md#utils).[`serialize`](api.md#serialize-3) ## Interfaces @@ -981,11 +981,11 @@ A dictionary representing the serialized properties of the object. ##### content -> **content**: `null` \| `string` \| ([`ITextContent`](README.md#itextcontent) \| [`IImageUrlContent`](README.md#iimageurlcontent))[] +> **content**: `null` \| `string` \| ([`ITextContent`](api.md#itextcontent) \| [`IImageUrlContent`](api.md#iimageurlcontent))[] ##### role -> **role**: [`GenerationMessageRole`](README.md#generationmessagerole) +> **role**: [`GenerationMessageRole`](api.md#generationmessagerole) ##### name @@ -1043,7 +1043,7 @@ A dictionary representing the serialized properties of the object. ##### function -> **function**: [`IFunction`](README.md#ifunction) +> **function**: [`IFunction`](api.md#ifunction) ## Type Aliases @@ -1055,11 +1055,11 @@ A dictionary representing the serialized properties of the object. ##### tags -> **tags**?: [`Maybe`](README.md#maybet)\<`string`[]\> +> **tags**?: [`Maybe`](api.md#maybet)\<`string`[]\> ##### metadata -> **metadata**?: [`Maybe`](README.md#maybet)\<`Record`\<`string`, `any`\>\> +> **metadata**?: [`Maybe`](api.md#maybet)\<`Record`\<`string`, `any`\>\> *** @@ -1083,13 +1083,13 @@ A dictionary representing the serialized properties of the object. ### Generation -> **Generation**: [`OmitUtils`](README.md#omitutilst)\<[`CompletionGeneration`](README.md#completiongeneration)\> \| [`OmitUtils`](README.md#omitutilst)\<[`ChatGeneration`](README.md#chatgeneration)\> +> **Generation**: [`OmitUtils`](api.md#omitutilst)\<[`CompletionGeneration`](api.md#completiongeneration)\> \| [`OmitUtils`](api.md#omitutilst)\<[`ChatGeneration`](api.md#chatgeneration)\> *** ### PersistedGeneration -> **PersistedGeneration**: [`Generation`](README.md#generation) & `object` +> **PersistedGeneration**: [`Generation`](api.md#generation) & `object` #### Type declaration @@ -1113,7 +1113,7 @@ A dictionary representing the serialized properties of the object. ### OmitUtils`` -> **OmitUtils**\<`T`\>: `Omit`\<`T`, keyof [`Utils`](README.md#utils)\> +> **OmitUtils**\<`T`\>: `Omit`\<`T`, keyof [`Utils`](api.md#utils)\> #### Type parameters @@ -1167,7 +1167,7 @@ A dictionary representing the serialized properties of the object. ##### pageInfo -> **pageInfo**: [`PageInfo`](README.md#pageinfo) +> **pageInfo**: [`PageInfo`](api.md#pageinfo) ## Functions diff --git a/src/evaluation/score.ts b/src/evaluation/score.ts index c510161..6915aa1 100644 --- a/src/evaluation/score.ts +++ b/src/evaluation/score.ts @@ -8,8 +8,8 @@ class ScoreFields extends Utils { generationId?: Maybe; datasetExperimentItemId?: Maybe; name: string = 'user-feedback'; - value: number = 0; - type: ScoreType = 'AI'; + value: number = 1; + type: ScoreType = 'HUMAN'; scorer?: Maybe; comment?: Maybe; tags?: Maybe; diff --git a/src/index.ts b/src/index.ts index 38430a2..b73c174 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,7 +31,7 @@ export class LiteralClient { * Initialize a new Literal AI Client. * @param options * @param options.apiKey The API key to use for the Literal AI API. Defaults to the LITERAL_API_KEY environment variable. - * @param options.apiUrl The URL of the Literal AI API. Defaults to the LITERAL_API_URL environment variable. + * @param options.apiUrl The URL of the Literal AI API. Defaults to the LITERAL_API_URL env var, or https://cloud.getliteral.ai. * @param options.environment The environment to use for the Literal AI API. * @param options.disabled If set to true, no call will be made to the Literal AI API. * @returns A new LiteralClient instance. From d1aee075575f215d61097b993dd42a98fa366ea7 Mon Sep 17 00:00:00 2001 From: Damien BUTY Date: Thu, 1 Aug 2024 12:41:28 +0200 Subject: [PATCH 8/9] fix: tests --- src/api.ts | 1 + src/observability/step.ts | 4 ++-- tests/api.test.ts | 11 ++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/api.ts b/src/api.ts index 8a63037..a925c71 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1065,6 +1065,7 @@ export class API { const variables = { threadId: id }; const response = await this.makeGqlCall(query, variables); + return response.data.deleteThread.id; } diff --git a/src/observability/step.ts b/src/observability/step.ts index 5a1ecfd..4dc2d95 100644 --- a/src/observability/step.ts +++ b/src/observability/step.ts @@ -74,8 +74,8 @@ export class Step extends StepFields { } // Automatically assign parent thread & step if there are any in the store. - this.threadId = this.client._currentThread()?.id; - this.parentId = this.client._currentStep()?.id; + this.threadId = this.threadId ?? this.client._currentThread()?.id; + this.parentId = this.parentId ?? this.client._currentStep()?.id; // Set the creation and start time to the current time if not provided. if (!this.createdAt) { diff --git a/tests/api.test.ts b/tests/api.test.ts index 84cdefe..ff78430 100644 --- a/tests/api.test.ts +++ b/tests/api.test.ts @@ -226,6 +226,7 @@ describe('End to end tests for the SDK', function () { it('should test steps', async function () { const thread = await client.thread({ id: uuidv4() }); + const step = await thread .step({ name: 'test', @@ -234,9 +235,7 @@ describe('End to end tests for the SDK', function () { }) .send(); - if (!step.id) { - throw new Error('Step id is null'); - } + expect(step.id).not.toBeNull(); await new Promise((resolve) => setTimeout(resolve, 1000)); @@ -245,7 +244,7 @@ describe('End to end tests for the SDK', function () { { field: 'id', operator: 'eq', - value: step.id + value: step.id! }, { field: 'tags', @@ -509,7 +508,7 @@ describe('End to end tests for the SDK', function () { }); it('should add a step to a dataset', async () => { - const thread = await client.thread({ id: uuidv4() }).upsert(); + const thread = await client.thread({ id: uuidv4() }); const step = await thread .step({ name: 'Run', @@ -528,6 +527,8 @@ describe('End to end tests for the SDK', function () { }) .send(); + await new Promise((resolve) => setTimeout(resolve, 1000)); + const datasetItem = await dataset.addStep(step.id!); expect(datasetItem.id).not.toBeNull(); From 8e64371bbbb097f0eccec82c50f627d34bdf633e Mon Sep 17 00:00:00 2001 From: Damien BUTY Date: Thu, 1 Aug 2024 15:22:32 +0200 Subject: [PATCH 9/9] fix: remove env support --- src/api.ts | 18 ++---------------- src/observability/thread.ts | 1 - 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/src/api.ts b/src/api.ts index a925c71..5ca43de 100644 --- a/src/api.ts +++ b/src/api.ts @@ -868,7 +868,6 @@ export class API { metadata?: Maybe>; participantId?: Maybe; tags?: Maybe; - environment?: Maybe; }): Promise; /** @@ -887,8 +886,7 @@ export class API { name?: Maybe, metadata?: Maybe>, participantId?: Maybe, - tags?: Maybe, - environment?: Maybe + tags?: Maybe ): Promise; async upsertThread( @@ -896,8 +894,7 @@ export class API { name?: Maybe, metadata?: Maybe>, participantId?: Maybe, - tags?: Maybe, - environment?: Maybe + tags?: Maybe ): Promise { let threadId = threadIdOrOptions; if (typeof threadIdOrOptions === 'object') { @@ -906,13 +903,6 @@ export class API { metadata = threadIdOrOptions.metadata; participantId = threadIdOrOptions.participantId; tags = threadIdOrOptions.tags; - environment = threadIdOrOptions.environment; - } - - const originalEnvironment = this.environment; - - if (environment && environment !== originalEnvironment) { - this.environment = environment; } const query = ` @@ -945,10 +935,6 @@ export class API { const response = await this.makeGqlCall(query, variables); - if (environment && environment !== originalEnvironment) { - this.environment = originalEnvironment; - } - return new Thread(this.client, response.data.upsertThread); } diff --git a/src/observability/thread.ts b/src/observability/thread.ts index dfebd58..51b37bd 100644 --- a/src/observability/thread.ts +++ b/src/observability/thread.ts @@ -82,7 +82,6 @@ export class Thread extends ThreadFields { await this.api.upsertThread({ threadId: this.id, name: this.name, - environment: this.environment, metadata: this.metadata, participantId: this.participantId, tags: this.tags