From b6f1b192313eeb4a1b4fd3eafdedaf04336c1435 Mon Sep 17 00:00:00 2001 From: Anirudh Kamath Date: Wed, 28 May 2025 16:18:52 -0700 Subject: [PATCH 1/2] remove caching --- lib/StagehandPage.ts | 8 - lib/cache.ts | 98 ------- lib/cache/ActionCache.ts | 158 ----------- lib/cache/BaseCache.ts | 568 ------------------------------------- lib/cache/LLMCache.ts | 48 ---- lib/index.ts | 3 +- lib/llm/AnthropicClient.ts | 92 ------ lib/llm/CerebrasClient.ts | 66 +---- lib/llm/GoogleClient.ts | 60 ---- lib/llm/GroqClient.ts | 65 ----- lib/llm/LLMProvider.ts | 38 +-- lib/llm/OpenAIClient.ts | 92 +----- lib/llm/aisdk.ts | 98 ------- pnpm-lock.yaml | 18 +- 14 files changed, 13 insertions(+), 1399 deletions(-) delete mode 100644 lib/cache.ts delete mode 100644 lib/cache/ActionCache.ts delete mode 100644 lib/cache/BaseCache.ts delete mode 100644 lib/cache/LLMCache.ts diff --git a/lib/StagehandPage.ts b/lib/StagehandPage.ts index 9a2797573..e5ab1c440 100644 --- a/lib/StagehandPage.ts +++ b/lib/StagehandPage.ts @@ -784,10 +784,6 @@ ${scriptContent} \ }, }); - if (this.stagehand.enableCaching) { - this.stagehand.llmProvider.cleanRequestCache(requestId); - } - throw e; }); @@ -899,10 +895,6 @@ ${scriptContent} \ }, }); - if (this.stagehand.enableCaching) { - this.stagehand.llmProvider.cleanRequestCache(requestId); - } - throw e; }); diff --git a/lib/cache.ts b/lib/cache.ts deleted file mode 100644 index a9e2a981d..000000000 --- a/lib/cache.ts +++ /dev/null @@ -1,98 +0,0 @@ -import fs from "fs"; -const observationsPath = "./.cache/observations.json"; -const actionsPath = "./.cache/actions.json"; - -/** - * A file system cache to skip inference when repeating steps - * It also acts as the source of truth for identifying previously seen actions and observations - */ -class Cache { - disabled: boolean; - - constructor({ disabled = false } = {}) { - this.disabled = disabled; - if (!this.disabled) { - this.initCache(); - } - } - - readObservations() { - if (this.disabled) { - return {}; - } - try { - return JSON.parse(fs.readFileSync(observationsPath, "utf8")); - } catch (error) { - console.error("Error reading from observations.json", error); - return {}; - } - } - - readActions() { - if (this.disabled) { - return {}; - } - try { - return JSON.parse(fs.readFileSync(actionsPath, "utf8")); - } catch (error) { - console.error("Error reading from actions.json", error); - return {}; - } - } - - writeObservations({ - key, - value, - }: { - key: string; - value: { id: string; result: string }; - }) { - if (this.disabled) { - return; - } - - const observations = this.readObservations(); - observations[key] = value; - fs.writeFileSync(observationsPath, JSON.stringify(observations, null, 2)); - } - - writeActions({ - key, - value, - }: { - key: string; - value: { id: string; result: string }; - }) { - if (this.disabled) { - return; - } - - const actions = this.readActions(); - actions[key] = value; - fs.writeFileSync(actionsPath, JSON.stringify(actions, null, 2)); - } - - evictCache() { - throw new Error("implement me"); - } - - private initCache() { - if (this.disabled) { - return; - } - const cacheDir = ".cache"; - - if (!fs.existsSync(cacheDir)) { - fs.mkdirSync(cacheDir); - } - if (!fs.existsSync(actionsPath)) { - fs.writeFileSync(actionsPath, JSON.stringify({})); - } - - if (!fs.existsSync(observationsPath)) { - fs.writeFileSync(observationsPath, JSON.stringify({})); - } - } -} - -export default Cache; diff --git a/lib/cache/ActionCache.ts b/lib/cache/ActionCache.ts deleted file mode 100644 index b54801d6c..000000000 --- a/lib/cache/ActionCache.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { LogLine } from "../../types/log"; -import { BaseCache, CacheEntry } from "./BaseCache"; - -export interface PlaywrightCommand { - method: string; - args: string[]; -} - -export interface ActionEntry extends CacheEntry { - data: { - playwrightCommand: PlaywrightCommand; - componentString: string; - xpaths: string[]; - newStepString: string; - completed: boolean; - previousSelectors: string[]; - action: string; - }; -} - -/** - * ActionCache handles logging and retrieving actions along with their Playwright commands. - */ -export class ActionCache extends BaseCache { - constructor( - logger: (message: LogLine) => void, - cacheDir?: string, - cacheFile?: string, - ) { - super(logger, cacheDir, cacheFile || "action_cache.json"); - } - - public async addActionStep({ - url, - action, - previousSelectors, - playwrightCommand, - componentString, - xpaths, - newStepString, - completed, - requestId, - }: { - url: string; - action: string; - previousSelectors: string[]; - playwrightCommand: PlaywrightCommand; - componentString: string; - requestId: string; - xpaths: string[]; - newStepString: string; - completed: boolean; - }): Promise { - this.logger({ - category: "action_cache", - message: "adding action step to cache", - level: 1, - auxiliary: { - action: { - value: action, - type: "string", - }, - requestId: { - value: requestId, - type: "string", - }, - url: { - value: url, - type: "string", - }, - previousSelectors: { - value: JSON.stringify(previousSelectors), - type: "object", - }, - }, - }); - - await this.set( - { url, action, previousSelectors }, - { - playwrightCommand, - componentString, - xpaths, - newStepString, - completed, - previousSelectors, - action, - }, - requestId, - ); - } - - /** - * Retrieves all actions for a specific trajectory. - * @param trajectoryId - Unique identifier for the trajectory. - * @param requestId - The identifier for the current request. - * @returns An array of TrajectoryEntry objects or null if not found. - */ - public async getActionStep({ - url, - action, - previousSelectors, - requestId, - }: { - url: string; - action: string; - previousSelectors: string[]; - requestId: string; - }): Promise { - const data = await super.get({ url, action, previousSelectors }, requestId); - if (!data) { - return null; - } - - return data; - } - - public async removeActionStep(cacheHashObj: { - url: string; - action: string; - previousSelectors: string[]; - requestId: string; - }): Promise { - await super.delete(cacheHashObj); - } - - /** - * Clears all actions for a specific trajectory. - * @param trajectoryId - Unique identifier for the trajectory. - * @param requestId - The identifier for the current request. - */ - public async clearAction(requestId: string): Promise { - await super.deleteCacheForRequestId(requestId); - this.logger({ - category: "action_cache", - message: "cleared action for ID", - level: 1, - auxiliary: { - requestId: { - value: requestId, - type: "string", - }, - }, - }); - } - - /** - * Resets the entire action cache. - */ - public async resetCache(): Promise { - await super.resetCache(); - this.logger({ - category: "action_cache", - message: "Action cache has been reset.", - level: 1, - }); - } -} diff --git a/lib/cache/BaseCache.ts b/lib/cache/BaseCache.ts deleted file mode 100644 index 18ef61690..000000000 --- a/lib/cache/BaseCache.ts +++ /dev/null @@ -1,568 +0,0 @@ -import * as fs from "fs"; -import * as path from "path"; -import * as crypto from "crypto"; -import { LogLine } from "../../types/log"; - -export interface CacheEntry { - timestamp: number; - data: unknown; - requestId: string; -} - -export interface CacheStore { - [key: string]: CacheEntry; -} - -export class BaseCache { - private readonly CACHE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 1 week in milliseconds - private readonly CLEANUP_PROBABILITY = 0.01; // 1% chance - - protected cacheDir: string; - protected cacheFile: string; - protected lockFile: string; - protected logger: (message: LogLine) => void; - - private readonly LOCK_TIMEOUT_MS = 1_000; - protected lockAcquired = false; - protected lockAcquireFailures = 0; - - // Added for request ID tracking - protected requestIdToUsedHashes: { [key: string]: string[] } = {}; - - constructor( - logger: (message: LogLine) => void, - cacheDir: string = path.join(process.cwd(), "tmp", ".cache"), - cacheFile: string = "cache.json", - ) { - this.logger = logger; - this.cacheDir = cacheDir; - this.cacheFile = path.join(cacheDir, cacheFile); - this.lockFile = path.join(cacheDir, "cache.lock"); - this.ensureCacheDirectory(); - this.setupProcessHandlers(); - } - - private setupProcessHandlers(): void { - const releaseLockAndExit = () => { - this.releaseLock(); - process.exit(); - }; - - process.on("exit", releaseLockAndExit); - process.on("SIGINT", releaseLockAndExit); - process.on("SIGTERM", releaseLockAndExit); - process.on("uncaughtException", (err) => { - this.logger({ - category: "base_cache", - message: "uncaught exception", - level: 2, - auxiliary: { - error: { - value: err.message, - type: "string", - }, - trace: { - value: err.stack, - type: "string", - }, - }, - }); - if (this.lockAcquired) { - releaseLockAndExit(); - } - }); - } - - protected ensureCacheDirectory(): void { - if (!fs.existsSync(this.cacheDir)) { - fs.mkdirSync(this.cacheDir, { recursive: true }); - this.logger({ - category: "base_cache", - message: "created cache directory", - level: 1, - auxiliary: { - cacheDir: { - value: this.cacheDir, - type: "string", - }, - }, - }); - } - } - - protected createHash(data: unknown): string { - const hash = crypto.createHash("sha256"); - return hash.update(JSON.stringify(data)).digest("hex"); - } - - protected sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); - } - - public async acquireLock(): Promise { - const startTime = Date.now(); - while (Date.now() - startTime < this.LOCK_TIMEOUT_MS) { - try { - if (fs.existsSync(this.lockFile)) { - const lockAge = Date.now() - fs.statSync(this.lockFile).mtimeMs; - if (lockAge > this.LOCK_TIMEOUT_MS) { - fs.unlinkSync(this.lockFile); - this.logger({ - category: "base_cache", - message: "Stale lock file removed", - level: 1, - }); - } - } - - fs.writeFileSync(this.lockFile, process.pid.toString(), { flag: "wx" }); - this.lockAcquireFailures = 0; - this.lockAcquired = true; - this.logger({ - category: "base_cache", - message: "Lock acquired", - level: 1, - }); - return true; - } catch (e) { - this.logger({ - category: "base_cache", - message: "error acquiring lock", - level: 2, - auxiliary: { - trace: { - value: e.stack, - type: "string", - }, - message: { - value: e.message, - type: "string", - }, - }, - }); - await this.sleep(5); - } - } - this.logger({ - category: "base_cache", - message: "Failed to acquire lock after timeout", - level: 2, - }); - this.lockAcquireFailures++; - if (this.lockAcquireFailures >= 3) { - this.logger({ - category: "base_cache", - message: - "Failed to acquire lock 3 times in a row. Releasing lock manually.", - level: 1, - }); - this.releaseLock(); - } - return false; - } - - public releaseLock(): void { - try { - if (fs.existsSync(this.lockFile)) { - fs.unlinkSync(this.lockFile); - this.logger({ - category: "base_cache", - message: "Lock released", - level: 1, - }); - } - this.lockAcquired = false; - } catch (error) { - this.logger({ - category: "base_cache", - message: "error releasing lock", - level: 2, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - } - } - - /** - * Cleans up stale cache entries that exceed the maximum age. - */ - public async cleanupStaleEntries(): Promise { - if (!(await this.acquireLock())) { - this.logger({ - category: "llm_cache", - message: "failed to acquire lock for cleanup", - level: 2, - }); - return; - } - - try { - const cache = this.readCache(); - const now = Date.now(); - let entriesRemoved = 0; - - for (const [hash, entry] of Object.entries(cache)) { - if (now - entry.timestamp > this.CACHE_MAX_AGE_MS) { - delete cache[hash]; - entriesRemoved++; - } - } - - if (entriesRemoved > 0) { - this.writeCache(cache); - this.logger({ - category: "llm_cache", - message: "cleaned up stale cache entries", - level: 1, - auxiliary: { - entriesRemoved: { - value: entriesRemoved.toString(), - type: "integer", - }, - }, - }); - } - } catch (error) { - this.logger({ - category: "llm_cache", - message: "error during cache cleanup", - level: 2, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - } finally { - this.releaseLock(); - } - } - - protected readCache(): CacheStore { - if (fs.existsSync(this.cacheFile)) { - try { - const data = fs.readFileSync(this.cacheFile, "utf-8"); - return JSON.parse(data) as CacheStore; - } catch (error) { - this.logger({ - category: "base_cache", - message: "error reading cache file. resetting cache.", - level: 1, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - this.resetCache(); - return {}; - } - } - return {}; - } - - protected writeCache(cache: CacheStore): void { - try { - fs.writeFileSync(this.cacheFile, JSON.stringify(cache, null, 2)); - this.logger({ - category: "base_cache", - message: "Cache written to file", - level: 1, - }); - } catch (error) { - this.logger({ - category: "base_cache", - message: "error writing cache file", - level: 2, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - } finally { - this.releaseLock(); - } - } - - /** - * Retrieves data from the cache based on the provided options. - * @param hashObj - The options used to generate the cache key. - * @param requestId - The identifier for the current request. - * @returns The cached data if available, otherwise null. - */ - public async get( - hashObj: Record | string, - requestId: string, - ): Promise { - if (!(await this.acquireLock())) { - this.logger({ - category: "base_cache", - message: "Failed to acquire lock for getting cache", - level: 2, - }); - return null; - } - - try { - const hash = this.createHash(hashObj); - const cache = this.readCache(); - - if (cache[hash]) { - this.trackRequestIdUsage(requestId, hash); - return cache[hash].data; - } - return null; - } catch (error) { - this.logger({ - category: "base_cache", - message: "error getting cache. resetting cache.", - level: 1, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - - this.resetCache(); - return null; - } finally { - this.releaseLock(); - } - } - - /** - * Stores data in the cache based on the provided options and requestId. - * @param hashObj - The options used to generate the cache key. - * @param data - The data to be cached. - * @param requestId - The identifier for the cache entry. - */ - public async set( - hashObj: Record, - data: T["data"], - requestId: string, - ): Promise { - if (!(await this.acquireLock())) { - this.logger({ - category: "base_cache", - message: "Failed to acquire lock for setting cache", - level: 2, - }); - return; - } - - try { - const hash = this.createHash(hashObj); - const cache = this.readCache(); - cache[hash] = { - data, - timestamp: Date.now(), - requestId, - }; - - this.writeCache(cache); - this.trackRequestIdUsage(requestId, hash); - } catch (error) { - this.logger({ - category: "base_cache", - message: "error setting cache. resetting cache.", - level: 1, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - - this.resetCache(); - } finally { - this.releaseLock(); - - if (Math.random() < this.CLEANUP_PROBABILITY) { - this.cleanupStaleEntries(); - } - } - } - - public async delete(hashObj: Record): Promise { - if (!(await this.acquireLock())) { - this.logger({ - category: "base_cache", - message: "Failed to acquire lock for removing cache entry", - level: 2, - }); - return; - } - - try { - const hash = this.createHash(hashObj); - const cache = this.readCache(); - - if (cache[hash]) { - delete cache[hash]; - this.writeCache(cache); - } else { - this.logger({ - category: "base_cache", - message: "Cache entry not found to delete", - level: 1, - }); - } - } catch (error) { - this.logger({ - category: "base_cache", - message: "error removing cache entry", - level: 2, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - } finally { - this.releaseLock(); - } - } - - /** - * Tracks the usage of a hash with a specific requestId. - * @param requestId - The identifier for the current request. - * @param hash - The cache key hash. - */ - protected trackRequestIdUsage(requestId: string, hash: string): void { - this.requestIdToUsedHashes[requestId] ??= []; - this.requestIdToUsedHashes[requestId].push(hash); - } - - /** - * Deletes all cache entries associated with a specific requestId. - * @param requestId - The identifier for the request whose cache entries should be deleted. - */ - public async deleteCacheForRequestId(requestId: string): Promise { - if (!(await this.acquireLock())) { - this.logger({ - category: "base_cache", - message: "Failed to acquire lock for deleting cache", - level: 2, - }); - return; - } - try { - const cache = this.readCache(); - const hashes = this.requestIdToUsedHashes[requestId] ?? []; - let entriesRemoved = 0; - for (const hash of hashes) { - if (cache[hash]) { - delete cache[hash]; - entriesRemoved++; - } - } - if (entriesRemoved > 0) { - this.writeCache(cache); - } else { - this.logger({ - category: "base_cache", - message: "no cache entries found for requestId", - level: 1, - auxiliary: { - requestId: { - value: requestId, - type: "string", - }, - }, - }); - } - // Remove the requestId from the mapping after deletion - delete this.requestIdToUsedHashes[requestId]; - } catch (error) { - this.logger({ - category: "base_cache", - message: "error deleting cache for requestId", - level: 2, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - requestId: { - value: requestId, - type: "string", - }, - }, - }); - } finally { - this.releaseLock(); - } - } - - /** - * Resets the entire cache by clearing the cache file. - */ - public resetCache(): void { - try { - fs.writeFileSync(this.cacheFile, "{}"); - this.requestIdToUsedHashes = {}; // Reset requestId tracking - } catch (error) { - this.logger({ - category: "base_cache", - message: "error resetting cache", - level: 2, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - } finally { - this.releaseLock(); - } - } -} diff --git a/lib/cache/LLMCache.ts b/lib/cache/LLMCache.ts deleted file mode 100644 index 53bd78213..000000000 --- a/lib/cache/LLMCache.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { BaseCache, CacheEntry } from "./BaseCache"; - -export class LLMCache extends BaseCache { - constructor( - logger: (message: { - category?: string; - message: string; - level?: number; - }) => void, - cacheDir?: string, - cacheFile?: string, - ) { - super(logger, cacheDir, cacheFile || "llm_calls.json"); - } - - /** - * Overrides the get method to track used hashes by requestId. - * @param options - The options used to generate the cache key. - * @param requestId - The identifier for the current request. - * @returns The cached data if available, otherwise null. - */ - public async get( - options: Record, - requestId: string, - ): Promise { - const data = await super.get(options, requestId); - return data as T | null; // TODO: remove this cast - } - - /** - * Overrides the set method to include cache cleanup logic. - * @param options - The options used to generate the cache key. - * @param data - The data to be cached. - * @param requestId - The identifier for the current request. - */ - public async set( - options: Record, - data: unknown, - requestId: string, - ): Promise { - await super.set(options, data, requestId); - this.logger({ - category: "llm_cache", - message: "Cache miss - saved new response", - level: 1, - }); - } -} diff --git a/lib/index.ts b/lib/index.ts index bfa62d9c6..fc2463074 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -526,8 +526,7 @@ export class Stagehand { enableCaching ?? (process.env.ENABLE_CACHING && process.env.ENABLE_CACHING === "true"); - this.llmProvider = - llmProvider || new LLMProvider(this.logger, this.enableCaching); + this.llmProvider = llmProvider || new LLMProvider(this.logger); this.apiKey = apiKey ?? process.env.BROWSERBASE_API_KEY; this.projectId = projectId ?? process.env.BROWSERBASE_PROJECT_ID; diff --git a/lib/llm/AnthropicClient.ts b/lib/llm/AnthropicClient.ts index 27d8f4c7c..c103d274d 100644 --- a/lib/llm/AnthropicClient.ts +++ b/lib/llm/AnthropicClient.ts @@ -8,7 +8,6 @@ import { import { zodToJsonSchema } from "zod-to-json-schema"; import { LogLine } from "../../types/log"; import { AnthropicJsonSchemaObject, AvailableModel } from "../../types/model"; -import { LLMCache } from "../cache/LLMCache"; import { CreateChatCompletionOptions, LLMClient, @@ -19,28 +18,20 @@ import { CreateChatCompletionResponseError } from "@/types/stagehandErrors"; export class AnthropicClient extends LLMClient { public type = "anthropic" as const; private client: Anthropic; - private cache: LLMCache | undefined; - private enableCaching: boolean; public clientOptions: ClientOptions; constructor({ - enableCaching = false, - cache, modelName, clientOptions, userProvidedInstructions, }: { logger: (message: LogLine) => void; - enableCaching?: boolean; - cache?: LLMCache; modelName: AvailableModel; clientOptions?: ClientOptions; userProvidedInstructions?: string; }) { super(modelName); this.client = new Anthropic(clientOptions); - this.cache = cache; - this.enableCaching = enableCaching; this.modelName = modelName; this.clientOptions = clientOptions; this.userProvidedInstructions = userProvidedInstructions; @@ -66,62 +57,6 @@ export class AnthropicClient extends LLMClient { }, }); - // Try to get cached response - const cacheOptions = { - model: this.modelName, - messages: options.messages, - temperature: options.temperature, - image: options.image, - response_model: options.response_model, - tools: options.tools, - retries: retries, - }; - - if (this.enableCaching) { - const cachedResponse = await this.cache.get( - cacheOptions, - options.requestId, - ); - if (cachedResponse) { - logger({ - category: "llm_cache", - message: "LLM cache hit - returning cached response", - level: 1, - auxiliary: { - cachedResponse: { - value: JSON.stringify(cachedResponse), - type: "object", - }, - requestId: { - value: options.requestId, - type: "string", - }, - cacheOptions: { - value: JSON.stringify(cacheOptions), - type: "object", - }, - }, - }); - return cachedResponse as T; - } else { - logger({ - category: "llm_cache", - message: "LLM cache miss - no cached response found", - level: 1, - auxiliary: { - cacheOptions: { - value: JSON.stringify(cacheOptions), - type: "object", - }, - requestId: { - value: options.requestId, - type: "string", - }, - }, - }); - } - } - const systemMessage = options.messages.find((msg) => { if (msg.role === "system") { if (typeof msg.content === "string") { @@ -316,10 +251,6 @@ export class AnthropicClient extends LLMClient { usage: usageData, } as unknown as T; - if (this.enableCaching) { - this.cache.set(cacheOptions, finalParsedResponse, options.requestId); - } - return finalParsedResponse; } else { if (!retries || retries < 5) { @@ -346,29 +277,6 @@ export class AnthropicClient extends LLMClient { } } - if (this.enableCaching) { - this.cache.set(cacheOptions, transformedResponse, options.requestId); - logger({ - category: "anthropic", - message: "cached response", - level: 1, - auxiliary: { - requestId: { - value: options.requestId, - type: "string", - }, - transformedResponse: { - value: JSON.stringify(transformedResponse), - type: "object", - }, - cacheOptions: { - value: JSON.stringify(cacheOptions), - type: "object", - }, - }, - }); - } - // if the function was called with a response model, it would have returned earlier // so we can safely cast here to T, which defaults to AnthropicTransformedResponse return transformedResponse as T; diff --git a/lib/llm/CerebrasClient.ts b/lib/llm/CerebrasClient.ts index 4b12c0380..256db20ad 100644 --- a/lib/llm/CerebrasClient.ts +++ b/lib/llm/CerebrasClient.ts @@ -3,7 +3,6 @@ import type { ClientOptions } from "openai"; import { zodToJsonSchema } from "zod-to-json-schema"; import { LogLine } from "../../types/log"; import { AvailableModel } from "../../types/model"; -import { LLMCache } from "../cache/LLMCache"; import { ChatMessage, CreateChatCompletionOptions, @@ -15,21 +14,15 @@ import { CreateChatCompletionResponseError } from "@/types/stagehandErrors"; export class CerebrasClient extends LLMClient { public type = "cerebras" as const; private client: OpenAI; - private cache: LLMCache | undefined; - private enableCaching: boolean; public clientOptions: ClientOptions; public hasVision = false; constructor({ - enableCaching = false, - cache, modelName, clientOptions, userProvidedInstructions, }: { logger: (message: LogLine) => void; - enableCaching?: boolean; - cache?: LLMCache; modelName: AvailableModel; clientOptions?: ClientOptions; userProvidedInstructions?: string; @@ -43,8 +36,6 @@ export class CerebrasClient extends LLMClient { ...clientOptions, }); - this.cache = cache; - this.enableCaching = enableCaching; this.modelName = modelName; this.clientOptions = clientOptions; } @@ -69,45 +60,6 @@ export class CerebrasClient extends LLMClient { }, }); - // Try to get cached response - const cacheOptions = { - model: this.modelName.split("cerebras-")[1], - messages: options.messages, - temperature: options.temperature, - response_model: options.response_model, - tools: options.tools, - retries: retries, - }; - - if (this.enableCaching) { - const cachedResponse = await this.cache.get( - cacheOptions, - options.requestId, - ); - if (cachedResponse) { - logger({ - category: "llm_cache", - message: "LLM cache hit - returning cached response", - level: 1, - auxiliary: { - cachedResponse: { - value: JSON.stringify(cachedResponse), - type: "object", - }, - requestId: { - value: options.requestId, - type: "string", - }, - cacheOptions: { - value: JSON.stringify(cacheOptions), - type: "object", - }, - }, - }); - return cachedResponse as T; - } - } - // Format messages for Cerebras API (using OpenAI format) const formattedMessages = options.messages.map((msg: ChatMessage) => { const baseMessage = { @@ -238,9 +190,6 @@ export class CerebrasClient extends LLMClient { // If we have no response model, just return the entire LLMResponse if (!options.response_model) { - if (this.enableCaching) { - await this.cache.set(cacheOptions, response, options.requestId); - } return response as T; } @@ -253,13 +202,6 @@ export class CerebrasClient extends LLMClient { data: result, usage: response.usage, }; - if (this.enableCaching) { - await this.cache.set( - cacheOptions, - finalResponse, - options.requestId, - ); - } return finalResponse as T; } catch (e) { logger({ @@ -287,13 +229,7 @@ export class CerebrasClient extends LLMClient { data: result, usage: response.usage, }; - if (this.enableCaching) { - await this.cache.set( - cacheOptions, - finalResponse, - options.requestId, - ); - } + return finalResponse as T; } } catch (e) { diff --git a/lib/llm/GoogleClient.ts b/lib/llm/GoogleClient.ts index 5460bea73..b41122cf4 100644 --- a/lib/llm/GoogleClient.ts +++ b/lib/llm/GoogleClient.ts @@ -9,11 +9,9 @@ import { Schema, Type, } from "@google/genai"; -import zodToJsonSchema from "zod-to-json-schema"; import { LogLine } from "../../types/log"; import { AvailableModel, ClientOptions } from "../../types/model"; -import { LLMCache } from "../cache/LLMCache"; import { validateZodSchema, toGeminiSchema, loadApiKeyFromEnv } from "../utils"; import { ChatCompletionOptions, @@ -58,22 +56,16 @@ const safetySettings = [ export class GoogleClient extends LLMClient { public type = "google" as const; private client: GoogleGenAI; - private cache: LLMCache | undefined; - private enableCaching: boolean; public clientOptions: ClientOptions; public hasVision: boolean; private logger: (message: LogLine) => void; constructor({ logger, // Added logger based on other clients - enableCaching = false, - cache, modelName, clientOptions, }: { logger: (message: LogLine) => void; // Added logger type - enableCaching?: boolean; - cache?: LLMCache; modelName: AvailableModel; clientOptions?: ClientOptions; // Expecting { apiKey: string } here }) { @@ -84,8 +76,6 @@ export class GoogleClient extends LLMClient { } this.clientOptions = clientOptions; this.client = new GoogleGenAI({ apiKey: clientOptions.apiKey }); - this.cache = cache; - this.enableCaching = enableCaching; this.modelName = modelName; this.logger = logger; // Determine vision capability based on model name (adjust as needed) @@ -235,48 +225,6 @@ export class GoogleClient extends LLMClient { maxTokens, } = options; - const cacheKeyOptions = { - model: this.modelName, - messages: options.messages, - temperature: temperature, - top_p: top_p, - // frequency_penalty and presence_penalty are not directly supported in Gemini API - image: image - ? { description: image.description, bufferLength: image.buffer.length } - : undefined, // Use buffer length for caching key stability - response_model: response_model - ? { - name: response_model.name, - schema: JSON.stringify(zodToJsonSchema(response_model.schema)), - } - : undefined, - tools: tools, - maxTokens: maxTokens, - }; - - if (this.enableCaching) { - const cachedResponse = await this.cache.get( - cacheKeyOptions, - requestId, - ); - if (cachedResponse) { - logger({ - category: "llm_cache", - message: "LLM cache hit - returning cached response", - level: 1, - auxiliary: { requestId: { value: requestId, type: "string" } }, - }); - return cachedResponse; - } else { - logger({ - category: "llm_cache", - message: "LLM cache miss - proceeding with API call", - level: 1, - auxiliary: { requestId: { value: requestId, type: "string" } }, - }); - } - } - const formattedMessages = this.formatMessages(options.messages, image); const formattedTools = this.formatTools(tools); @@ -459,17 +407,9 @@ export class GoogleClient extends LLMClient { usage: llmResponse.usage, }; - if (this.enableCaching) { - await this.cache.set(cacheKeyOptions, extractionResult, requestId); - } return extractionResult as T; } - // Cache the standard response if not using response_model - if (this.enableCaching) { - await this.cache.set(cacheKeyOptions, llmResponse, requestId); - } - return llmResponse as T; } catch (error) { logger({ diff --git a/lib/llm/GroqClient.ts b/lib/llm/GroqClient.ts index d512ed6f9..5f851f346 100644 --- a/lib/llm/GroqClient.ts +++ b/lib/llm/GroqClient.ts @@ -3,7 +3,6 @@ import OpenAI from "openai"; import { zodToJsonSchema } from "zod-to-json-schema"; import { LogLine } from "../../types/log"; import { AvailableModel } from "../../types/model"; -import { LLMCache } from "../cache/LLMCache"; import { ChatMessage, CreateChatCompletionOptions, @@ -15,21 +14,15 @@ import { CreateChatCompletionResponseError } from "@/types/stagehandErrors"; export class GroqClient extends LLMClient { public type = "groq" as const; private client: OpenAI; - private cache: LLMCache | undefined; - private enableCaching: boolean; public clientOptions: ClientOptions; public hasVision = false; constructor({ - enableCaching = false, - cache, modelName, clientOptions, userProvidedInstructions, }: { logger: (message: LogLine) => void; - enableCaching?: boolean; - cache?: LLMCache; modelName: AvailableModel; clientOptions?: ClientOptions; userProvidedInstructions?: string; @@ -43,8 +36,6 @@ export class GroqClient extends LLMClient { ...clientOptions, }); - this.cache = cache; - this.enableCaching = enableCaching; this.modelName = modelName; this.clientOptions = clientOptions; } @@ -69,45 +60,6 @@ export class GroqClient extends LLMClient { }, }); - // Try to get cached response - const cacheOptions = { - model: this.modelName.split("groq-")[1], - messages: options.messages, - temperature: options.temperature, - response_model: options.response_model, - tools: options.tools, - retries: retries, - }; - - if (this.enableCaching) { - const cachedResponse = await this.cache.get( - cacheOptions, - options.requestId, - ); - if (cachedResponse) { - logger({ - category: "llm_cache", - message: "LLM cache hit - returning cached response", - level: 1, - auxiliary: { - cachedResponse: { - value: JSON.stringify(cachedResponse), - type: "object", - }, - requestId: { - value: options.requestId, - type: "string", - }, - cacheOptions: { - value: JSON.stringify(cacheOptions), - type: "object", - }, - }, - }); - return cachedResponse as T; - } - } - // Format messages for Groq API (using OpenAI format) const formattedMessages = options.messages.map((msg: ChatMessage) => { const baseMessage = { @@ -238,9 +190,6 @@ export class GroqClient extends LLMClient { // If there's no response model, return the entire response object if (!options.response_model) { - if (this.enableCaching) { - await this.cache.set(cacheOptions, response, options.requestId); - } return response as T; } @@ -253,13 +202,6 @@ export class GroqClient extends LLMClient { data: result, usage: response.usage, }; - if (this.enableCaching) { - await this.cache.set( - cacheOptions, - finalResponse, - options.requestId, - ); - } return finalResponse as T; } catch (e) { logger({ @@ -288,13 +230,6 @@ export class GroqClient extends LLMClient { data: result, usage: response.usage, }; - if (this.enableCaching) { - await this.cache.set( - cacheOptions, - finalResponse, - options.requestId, - ); - } return finalResponse as T; } } catch (e) { diff --git a/lib/llm/LLMProvider.ts b/lib/llm/LLMProvider.ts index fc11c5753..12c4df581 100644 --- a/lib/llm/LLMProvider.ts +++ b/lib/llm/LLMProvider.ts @@ -9,7 +9,6 @@ import { ClientOptions, ModelProvider, } from "../../types/model"; -import { LLMCache } from "../cache/LLMCache"; import { AISdkClient } from "./aisdk"; import { AnthropicClient } from "./AnthropicClient"; import { CerebrasClient } from "./CerebrasClient"; @@ -124,32 +123,9 @@ function getAISDKLanguageModel( export class LLMProvider { private logger: (message: LogLine) => void; - private enableCaching: boolean; - private cache: LLMCache | undefined; - constructor(logger: (message: LogLine) => void, enableCaching: boolean) { + constructor(logger: (message: LogLine) => void) { this.logger = logger; - this.enableCaching = enableCaching; - this.cache = enableCaching ? new LLMCache(logger) : undefined; - } - - cleanRequestCache(requestId: string): void { - if (!this.enableCaching) { - return; - } - - this.logger({ - category: "llm_cache", - message: "cleaning up cache", - level: 1, - auxiliary: { - requestId: { - value: requestId, - type: "string", - }, - }, - }); - this.cache.deleteCacheForRequestId(requestId); } getClient( @@ -170,8 +146,6 @@ export class LLMProvider { return new AISdkClient({ model: languageModel, logger: this.logger, - enableCaching: this.enableCaching, - cache: this.cache, }); } @@ -184,40 +158,30 @@ export class LLMProvider { case "openai": return new OpenAIClient({ logger: this.logger, - enableCaching: this.enableCaching, - cache: this.cache, modelName: availableModel, clientOptions, }); case "anthropic": return new AnthropicClient({ logger: this.logger, - enableCaching: this.enableCaching, - cache: this.cache, modelName: availableModel, clientOptions, }); case "cerebras": return new CerebrasClient({ logger: this.logger, - enableCaching: this.enableCaching, - cache: this.cache, modelName: availableModel, clientOptions, }); case "groq": return new GroqClient({ logger: this.logger, - enableCaching: this.enableCaching, - cache: this.cache, modelName: availableModel, clientOptions, }); case "google": return new GoogleClient({ logger: this.logger, - enableCaching: this.enableCaching, - cache: this.cache, modelName: availableModel, clientOptions, }); diff --git a/lib/llm/OpenAIClient.ts b/lib/llm/OpenAIClient.ts index 008215bd2..5e947d34b 100644 --- a/lib/llm/OpenAIClient.ts +++ b/lib/llm/OpenAIClient.ts @@ -12,7 +12,6 @@ import { import zodToJsonSchema from "zod-to-json-schema"; import { LogLine } from "../../types/log"; import { AvailableModel } from "../../types/model"; -import { LLMCache } from "../cache/LLMCache"; import { validateZodSchema } from "../utils"; import { ChatCompletionOptions, @@ -30,27 +29,19 @@ import { export class OpenAIClient extends LLMClient { public type = "openai" as const; private client: OpenAI; - private cache: LLMCache | undefined; - private enableCaching: boolean; public clientOptions: ClientOptions; constructor({ - enableCaching = false, - cache, modelName, clientOptions, }: { logger: (message: LogLine) => void; - enableCaching?: boolean; - cache?: LLMCache; modelName: AvailableModel; clientOptions?: ClientOptions; }) { super(modelName); this.clientOptions = clientOptions; this.client = new OpenAI(clientOptions); - this.cache = cache; - this.enableCaching = enableCaching; this.modelName = modelName; } @@ -121,7 +112,7 @@ export class OpenAIClient extends LLMClient { throw new StagehandError("Temperature is not supported for o1 models"); } - const { image, requestId, ...optionsWithoutImageAndRequestId } = options; + const { requestId, ...optionsWithoutImageAndRequestId } = options; logger({ category: "openai", @@ -142,54 +133,6 @@ export class OpenAIClient extends LLMClient { }, }); - const cacheOptions = { - model: this.modelName, - messages: options.messages, - temperature: options.temperature, - top_p: options.top_p, - frequency_penalty: options.frequency_penalty, - presence_penalty: options.presence_penalty, - image: image, - response_model: options.response_model, - }; - - if (this.enableCaching) { - const cachedResponse = await this.cache.get( - cacheOptions, - options.requestId, - ); - if (cachedResponse) { - logger({ - category: "llm_cache", - message: "LLM cache hit - returning cached response", - level: 1, - auxiliary: { - requestId: { - value: options.requestId, - type: "string", - }, - cachedResponse: { - value: JSON.stringify(cachedResponse), - type: "object", - }, - }, - }); - return cachedResponse; - } else { - logger({ - category: "llm_cache", - message: "LLM cache miss - no cached response found", - level: 1, - auxiliary: { - requestId: { - value: options.requestId, - type: "string", - }, - }, - }); - } - } - if (options.image) { const screenshotMessage: ChatMessage = { role: "user", @@ -447,45 +390,12 @@ export class OpenAIClient extends LLMClient { throw e; } - if (this.enableCaching) { - this.cache.set( - cacheOptions, - { - ...parsedData, - }, - options.requestId, - ); - } - return { data: parsedData, usage: response.usage, } as T; } - if (this.enableCaching) { - logger({ - category: "llm_cache", - message: "caching response", - level: 1, - auxiliary: { - requestId: { - value: options.requestId, - type: "string", - }, - cacheOptions: { - value: JSON.stringify(cacheOptions), - type: "object", - }, - response: { - value: JSON.stringify(response), - type: "object", - }, - }, - }); - this.cache.set(cacheOptions, response, options.requestId); - } - // if the function was called with a response model, it would have returned earlier // so we can safely cast here to T, which defaults to ChatCompletion return response as T; diff --git a/lib/llm/aisdk.ts b/lib/llm/aisdk.ts index 2448f052b..23c5326a7 100644 --- a/lib/llm/aisdk.ts +++ b/lib/llm/aisdk.ts @@ -14,31 +14,22 @@ import { CreateChatCompletionOptions, LLMClient } from "./LLMClient"; import { LogLine } from "../../types/log"; import { AvailableModel } from "../../types/model"; import { ChatCompletion } from "openai/resources"; -import { LLMCache } from "../cache/LLMCache"; export class AISdkClient extends LLMClient { public type = "aisdk" as const; private model: LanguageModel; private logger?: (message: LogLine) => void; - private cache: LLMCache | undefined; - private enableCaching: boolean; constructor({ model, logger, - enableCaching = false, - cache, }: { model: LanguageModel; logger?: (message: LogLine) => void; - enableCaching?: boolean; - cache?: LLMCache; }) { super(model.modelId as AvailableModel); this.model = model; this.logger = logger; - this.cache = cache; - this.enableCaching = enableCaching; } async createChatCompletion({ @@ -60,49 +51,6 @@ export class AISdkClient extends LLMClient { }, }); - const cacheOptions = { - model: this.model.modelId, - messages: options.messages, - response_model: options.response_model, - }; - - if (this.enableCaching && this.cache) { - const cachedResponse = await this.cache.get( - cacheOptions, - options.requestId, - ); - if (cachedResponse) { - this.logger?.({ - category: "llm_cache", - message: "LLM cache hit - returning cached response", - level: 1, - auxiliary: { - requestId: { - value: options.requestId, - type: "string", - }, - cachedResponse: { - value: JSON.stringify(cachedResponse), - type: "object", - }, - }, - }); - return cachedResponse; - } else { - this.logger?.({ - category: "llm_cache", - message: "LLM cache miss - no cached response found", - level: 1, - auxiliary: { - requestId: { - value: options.requestId, - type: "string", - }, - }, - }); - } - } - const formattedMessages: CoreMessage[] = options.messages.map((message) => { if (Array.isArray(message.content)) { if (message.role === "system") { @@ -172,29 +120,6 @@ export class AISdkClient extends LLMClient { }, } as T; - if (this.enableCaching) { - this.logger?.({ - category: "llm_cache", - message: "caching response", - level: 1, - auxiliary: { - requestId: { - value: options.requestId, - type: "string", - }, - cacheOptions: { - value: JSON.stringify(cacheOptions), - type: "object", - }, - response: { - value: JSON.stringify(result), - type: "object", - }, - }, - }); - this.cache.set(cacheOptions, result, options.requestId); - } - this.logger?.({ category: "aisdk", message: "response", @@ -238,29 +163,6 @@ export class AISdkClient extends LLMClient { }, } as T; - if (this.enableCaching) { - this.logger?.({ - category: "llm_cache", - message: "caching response", - level: 1, - auxiliary: { - requestId: { - value: options.requestId, - type: "string", - }, - cacheOptions: { - value: JSON.stringify(cacheOptions), - type: "object", - }, - response: { - value: JSON.stringify(result), - type: "object", - }, - }, - }); - this.cache.set(cacheOptions, result, options.requestId); - } - this.logger?.({ category: "aisdk", message: "response", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d2a576546..ffc629e6f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: deepmerge: specifier: ^4.3.1 version: 4.3.1 + devtools-protocol: + specifier: ^0.0.1464554 + version: 0.0.1464554 dotenv: specifier: ^16.4.5 version: 16.5.0 @@ -135,10 +138,7 @@ importers: version: 1.0.0 chromium-bidi: specifier: ^0.10.0 - version: 0.10.2(devtools-protocol@0.0.1462014) - devtools-protocol: - specifier: ^0.0.1462014 - version: 0.0.1462014 + version: 0.10.2(devtools-protocol@0.0.1464554) esbuild: specifier: ^0.21.4 version: 0.21.5 @@ -2454,8 +2454,8 @@ packages: devtools-protocol@0.0.1312386: resolution: {integrity: sha512-DPnhUXvmvKT2dFA/j7B+riVLUt9Q6RKJlcppojL5CoRywJJKLDYnRlw0gTFKfgDPHP5E04UoB71SxoJlVZy8FA==} - devtools-protocol@0.0.1462014: - resolution: {integrity: sha512-z2rfWuBdh/1p6vVjEU27w7eoS73nZf1lTe/nwvSFjy719dBOCXuADmzNQvdT+coAeF/q5khSUGw4CtIH7Cfo8g==} + devtools-protocol@0.0.1464554: + resolution: {integrity: sha512-CAoP3lYfwAGQTaAXYvA6JZR0fjGUb7qec1qf4mToyoH2TZgUFeIqYcjh6f9jNuhHfuZiEdH+PONHYrLhRQX6aw==} diff-match-patch@1.0.5: resolution: {integrity: sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==} @@ -7484,9 +7484,9 @@ snapshots: chownr@2.0.0: {} - chromium-bidi@0.10.2(devtools-protocol@0.0.1462014): + chromium-bidi@0.10.2(devtools-protocol@0.0.1464554): dependencies: - devtools-protocol: 0.0.1462014 + devtools-protocol: 0.0.1464554 mitt: 3.0.1 zod: 3.23.8 @@ -7738,7 +7738,7 @@ snapshots: devtools-protocol@0.0.1312386: {} - devtools-protocol@0.0.1462014: {} + devtools-protocol@0.0.1464554: {} diff-match-patch@1.0.5: {} From 1683cb923e7993c1155f4226294b95112b4670c4 Mon Sep 17 00:00:00 2001 From: Anirudh Kamath Date: Wed, 28 May 2025 16:20:16 -0700 Subject: [PATCH 2/2] changeset --- .changeset/calm-hotels-yell.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/calm-hotels-yell.md diff --git a/.changeset/calm-hotels-yell.md b/.changeset/calm-hotels-yell.md new file mode 100644 index 000000000..61449a422 --- /dev/null +++ b/.changeset/calm-hotels-yell.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Remove existing LLM inference caching