From a003cf666aee65b34e831fa3a9f8df69fdc44d5b Mon Sep 17 00:00:00 2001 From: Willy Douhard Date: Mon, 11 Nov 2024 17:29:17 +0100 Subject: [PATCH 1/4] feat: add prompt variant --- package.json | 2 +- src/api.ts | 95 +++++++++++++++++++++++++++++++++++++-- src/evaluation/dataset.ts | 4 +- tests/api.test.ts | 14 ++++++ 4 files changed, 108 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index b30acad..0a7bb90 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@literalai/client", - "version": "0.0.602", + "version": "0.0.603", "description": "", "exports": { ".": { diff --git a/src/api.ts b/src/api.ts index a6d64d3..1e3bfcf 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1828,6 +1828,65 @@ export class API { return Object.values(result.data).map((x: any) => new DatasetItem(x)); } + /** + * Creates a prompt variation for an experiment. + * This variation is not an official version until manually saved. + * + * @param name The name of the prompt to retrieve or create. + * @param templateMessages A list of template messages for the prompt. + * @param settings Optional settings for the prompt. + * @param tools Optional tools for the prompt. + * @returns The prompt that was retrieved or created. + */ + public async createPromptVariant( + name: string, + templateMessages: IGenerationMessage[], + settings?: Maybe>, + tools?: Maybe> + ): Promise { + const mutation = `mutation createPromptExperiment( + $fromLineageId: String + $fromVersion: Int + $scoreTemplateId: String + $templateMessages: Json + $settings: Json + $tools: Json + $variables: Json + ) { + createPromptExperiment( + fromLineageId: $fromLineageId + fromVersion: $fromVersion + scoreTemplateId: $scoreTemplateId + templateMessages: $templateMessages + settings: $settings + tools: $tools + variables: $variables + ) { + id + fromLineageId + fromVersion + scoreTemplateId + projectId + projectUserId + tools + settings + variables + templateMessages + } + } + `; + + const lineage = await this.getPromptLineageByName(name); + const result = await this.makeGqlCall(mutation, { + fromLineageId: lineage?.id, + templateMessages, + settings, + tools + }); + + return result.data.createPromptExperiment?.id; + } + /** * Creates a new dataset experiment. * @param datasetExperiment @@ -1840,12 +1899,12 @@ export class API { public async createExperiment(datasetExperiment: { name: string; datasetId?: string; - promptId?: string; + promptVariantId?: string; params?: Record | Array>; }) { const query = ` - mutation CreateDatasetExperiment($name: String!, $datasetId: String $promptId: String, $params: Json) { - createDatasetExperiment(name: $name, datasetId: $datasetId, promptId: $promptId, params: $params) { + mutation CreateDatasetExperiment($name: String!, $datasetId: String, $promptExperimentId: String, $params: Json) { + createDatasetExperiment(name: $name, datasetId: $datasetId, promptExperimentId: $promptExperimentId, params: $params) { id } } @@ -1853,7 +1912,7 @@ export class API { const datasetExperimentInput = { name: datasetExperiment.name, datasetId: datasetExperiment.datasetId, - promptId: datasetExperiment.promptId, + promptExperimentId: datasetExperiment.promptVariantId, params: datasetExperiment.params }; const result = await this.makeGqlCall(query, datasetExperimentInput); @@ -1947,6 +2006,34 @@ export class API { return result.data.createPromptLineage; } + /** + * Get an existing prompt lineage by name. + * + * @param name - The name of the prompt lineage. This parameter is required. + * @returns The existing prompt lineage object, or null. + */ + public async getPromptLineageByName(name: string) { + const query = `query promptLineage( + $name: String! + ) { + promptLineage( + name: $name + ) { + id + } + }`; + + const result = await this.makeGqlCall(query, { + name + }); + + if (!result.data || !result.data.promptLineage) { + return null; + } + + return result.data.promptLineage; + } + /** * @deprecated Please use getOrCreatePrompt instead. */ diff --git a/src/evaluation/dataset.ts b/src/evaluation/dataset.ts index 3dfcf96..1fa6edc 100644 --- a/src/evaluation/dataset.ts +++ b/src/evaluation/dataset.ts @@ -96,13 +96,13 @@ export class Dataset extends DatasetFields { */ async createExperiment(experiment: { name: string; - promptId?: string; + promptVariantId?: string; params?: Record | Array>; }) { const datasetExperiment = await this.api.createExperiment({ name: experiment.name, datasetId: this.id, - promptId: experiment.promptId, + promptVariantId: experiment.promptVariantId, params: experiment.params }); return new DatasetExperiment(this.api, datasetExperiment); diff --git a/tests/api.test.ts b/tests/api.test.ts index 3017d07..b87904c 100644 --- a/tests/api.test.ts +++ b/tests/api.test.ts @@ -448,6 +448,20 @@ describe('End to end tests for the SDK', function () { expect(experiment.id).not.toBeNull(); dataset.delete(); }); + + it.only('should create a dataset experiment with a prompt variant', async () => { + const promptVariantId = await client.api.createPromptVariant( + 'Default', + [{ role: 'user', content: 'hello' }], + { temperature: 0.5 } + ); + const experiment = await client.api.createExperiment({ + name: `test_${uuidv4()}`, + promptVariantId: promptVariantId + }); + expect(promptVariantId).toBeDefined(); + expect(experiment.id).not.toBeNull(); + }); }); describe('dataset item api', () => { From 9998ce5703582215558056c3fc9a912d404cf042 Mon Sep 17 00:00:00 2001 From: Willy Douhard Date: Mon, 11 Nov 2024 17:29:47 +0100 Subject: [PATCH 2/4] fix: return desc --- src/api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api.ts b/src/api.ts index 1e3bfcf..7d510ab 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1836,7 +1836,7 @@ export class API { * @param templateMessages A list of template messages for the prompt. * @param settings Optional settings for the prompt. * @param tools Optional tools for the prompt. - * @returns The prompt that was retrieved or created. + * @returns The prompt variant id to link with the experiment. */ public async createPromptVariant( name: string, From 05fdb10f7d8a24e5fc9452d4176066e89981d303 Mon Sep 17 00:00:00 2001 From: Willy Douhard Date: Mon, 11 Nov 2024 17:31:19 +0100 Subject: [PATCH 3/4] fix: remove .only --- tests/api.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api.test.ts b/tests/api.test.ts index b87904c..a23ac2d 100644 --- a/tests/api.test.ts +++ b/tests/api.test.ts @@ -449,7 +449,7 @@ describe('End to end tests for the SDK', function () { dataset.delete(); }); - it.only('should create a dataset experiment with a prompt variant', async () => { + it('should create a dataset experiment with a prompt variant', async () => { const promptVariantId = await client.api.createPromptVariant( 'Default', [{ role: 'user', content: 'hello' }], From da2f7981e4011ab8109900de37163ff0fbaaa91a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugues=20de=20Saxc=C3=A9?= Date: Tue, 12 Nov 2024 08:47:40 +0100 Subject: [PATCH 4/4] chore: bump version to 0.1.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0a7bb90..397d2e1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@literalai/client", - "version": "0.0.603", + "version": "0.1.0", "description": "", "exports": { ".": {