Skip to content

feat: support root run #54

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ const version = packageJson.version;
const stepFields = `
id
threadId
rootRunId
parentId
startTime
endTime
Expand Down Expand Up @@ -152,6 +153,7 @@ function ingestStepsFieldsBuilder(steps: Step[]) {
for (let id = 0; id < steps.length; id++) {
generated += `$id_${id}: String!
$threadId_${id}: String
$rootRunId_${id}: String
$type_${id}: StepType
$startTime_${id}: DateTime
$endTime_${id}: DateTime
Expand All @@ -177,6 +179,7 @@ function ingestStepsArgsBuilder(steps: Step[]) {
step${id}: ingestStep(
id: $id_${id}
threadId: $threadId_${id}
rootRunId: $rootRunId_${id}
startTime: $startTime_${id}
endTime: $endTime_${id}
type: $type_${id}
Expand Down Expand Up @@ -1651,6 +1654,7 @@ export class API {
input
expectedOutput
intermediarySteps
stepId
}
}
`;
Expand All @@ -1676,6 +1680,7 @@ export class API {
input
expectedOutput
intermediarySteps
stepId
}
}
`;
Expand All @@ -1701,6 +1706,7 @@ export class API {
input
expectedOutput
intermediarySteps
stepId
}
}
`;
Expand Down Expand Up @@ -1732,6 +1738,7 @@ export class API {
input
expectedOutput
intermediarySteps
stepId
}
}
`;
Expand Down Expand Up @@ -1767,6 +1774,7 @@ export class API {
input
expectedOutput
intermediarySteps
stepId
}
}
`;
Expand Down
7 changes: 6 additions & 1 deletion src/evaluation/experiment-item-run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,12 @@ export class ExperimentItemRun extends Step {
{
currentThread: currentStore?.currentThread ?? null,
currentStep: this,
currentExperimentItemRunId: this.id ?? null
currentExperimentItemRunId: this.id ?? null,
rootRun: currentStore?.rootRun
? currentStore?.rootRun
: this.type === 'run'
? this
: null
},
async () => {
try {
Expand Down
28 changes: 28 additions & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ type StoredContext = {
currentThread: Thread | null;
currentStep: Step | null;
currentExperimentItemRunId?: string | null;
rootRun: Step | null;
};

const storage = new AsyncLocalStorage<StoredContext>();
Expand Down Expand Up @@ -125,6 +126,16 @@ export class LiteralClient {
return store?.currentExperimentItemRunId || null;
}

/**
* Returns the root run from the context or null if none.
* @returns The root run, if any.
*/
_rootRun(): Step | null {
const store = storage.getStore();

return store?.rootRun || null;
}

/**
* Gets the current thread from the context.
* WARNING : this will throw if run outside of a thread context.
Expand Down Expand Up @@ -175,4 +186,21 @@ export class LiteralClient {

return store?.currentExperimentItemRunId;
}

/**
* Gets the root run from the context.
* WARNING : this will throw if run outside of a step context.
* @returns The current step, if any.
*/
getRootRun(): Step {
const store = storage.getStore();

if (!store?.rootRun) {
throw new Error(
'Literal AI SDK : tried to access root run outside of a context.'
);
}

return store.rootRun;
}
}
12 changes: 10 additions & 2 deletions src/observability/step.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ class StepFields extends Utils {
name!: string;
type!: StepType;
threadId?: string;
rootRunId?: Maybe<string>;
createdAt?: Maybe<string>;
startTime?: Maybe<string>;
id?: Maybe<string>;
Expand Down Expand Up @@ -73,9 +74,10 @@ export class Step extends StepFields {
return;
}

// Automatically assign parent thread & step if there are any in the store.
// Automatically assign parent thread & step & rootRun if there are any in the store.
this.threadId = this.threadId ?? this.client._currentThread()?.id;
this.parentId = this.parentId ?? this.client._currentStep()?.id;
this.rootRunId = this.rootRunId ?? this.client._rootRun()?.id;

// Set the creation and start time to the current time if not provided.
if (!this.createdAt) {
Expand Down Expand Up @@ -167,7 +169,12 @@ export class Step extends StepFields {
currentThread: currentStore?.currentThread ?? null,
currentExperimentItemRunId:
currentStore?.currentExperimentItemRunId ?? null,
currentStep: this
currentStep: this,
rootRun: currentStore?.rootRun
? currentStore?.rootRun
: this.type === 'run'
? this
: null
},
() => cb(this)
);
Expand Down Expand Up @@ -197,6 +204,7 @@ export class Step extends StepFields {
this.scores = updatedStep.scores ?? this.scores;
this.attachments = updatedStep.attachments ?? this.attachments;
this.environment = updatedStep.environment ?? this.environment;
this.rootRunId = updatedStep.rootRunId ?? this.rootRunId;
}

this.send().catch(console.error);
Expand Down
3 changes: 2 additions & 1 deletion src/observability/thread.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,8 @@ export class Thread extends ThreadFields {
currentThread: this,
currentExperimentItemRunId:
currentStore?.currentExperimentItemRunId ?? null,
currentStep: null
currentStep: null,
rootRun: null
},
() => cb(this)
);
Expand Down
27 changes: 27 additions & 0 deletions tests/wrappers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,32 @@ describe('Wrapper', () => {
});
});

it('handles nested runs', async () => {
let runId: Maybe<string>;
let stepId: Maybe<string>;

const step = async (_query: string) =>
client.step({ name: 'foo', type: 'undefined' }).wrap(async () => {
stepId = client.getCurrentStep()!.id;
});

await client.thread({ name: 'Test Wrappers Thread' }).wrap(async () => {
return client.run({ name: 'Test Wrappers Run' }).wrap(async () => {
runId = client.getCurrentStep()!.id;

return client.run({ name: 'Test Nested Run' }).wrap(async () => {
await step('foo');
});
});
});

await sleep(1000);
const run = await client.api.getStep(runId!);
const createdStep = await client.api.getStep(stepId!);

expect(createdStep!.rootRunId).toEqual(run!.id);
});

it('handles steps outside of a thread', async () => {
let runId: Maybe<string>;
let stepId: Maybe<string>;
Expand Down Expand Up @@ -172,6 +198,7 @@ describe('Wrapper', () => {
expect(step!.name).toEqual('Test Wrappers Step');
expect(step!.threadId).toBeNull();
expect(step!.parentId).toEqual(run!.id);
expect(step!.rootRunId).toEqual(run!.id);
});

it("doesn't leak the current store when getting entities from the API", async () => {
Expand Down
Loading