From 3d7f0e8427006f5e5f8a861beec05c0cd61f726a Mon Sep 17 00:00:00 2001 From: Sameel Date: Wed, 9 Jul 2025 16:36:13 -1000 Subject: [PATCH 1/6] add new models --- lib/agent/AgentClient.ts | 3 +- lib/agent/AgentProvider.ts | 11 +++++++- lib/agent/AnthropicCUAClient.ts | 49 +++++++++++++++++++++++++++++++-- lib/agent/OpenAICUAClient.ts | 22 +++++++-------- lib/llm/LLMProvider.ts | 30 +++++++++++--------- 5 files changed, 85 insertions(+), 30 deletions(-) diff --git a/lib/agent/AgentClient.ts b/lib/agent/AgentClient.ts index ac0aa83af..014ac8c7c 100644 --- a/lib/agent/AgentClient.ts +++ b/lib/agent/AgentClient.ts @@ -4,6 +4,7 @@ import { AgentType, AgentExecutionOptions, } from "@/types/agent"; +import { ClientOptions } from "@anthropic-ai/sdk"; /** * Abstract base class for agent clients @@ -12,7 +13,7 @@ import { export abstract class AgentClient { public type: AgentType; public modelName: string; - public clientOptions: Record; + public clientOptions: ClientOptions; public userProvidedInstructions?: string; constructor( diff --git a/lib/agent/AgentProvider.ts b/lib/agent/AgentProvider.ts index cfb9242f9..5f7624a79 100644 --- a/lib/agent/AgentProvider.ts +++ b/lib/agent/AgentProvider.ts @@ -10,10 +10,19 @@ import { // Map model names to their provider types const modelToAgentProviderMap: Record = { + // OpenAI models "computer-use-preview": "openai", "computer-use-preview-2025-03-11": "openai", - "claude-3-7-sonnet-latest": "anthropic", + // Anthropic models + "claude-sonnet-4-0": "anthropic", "claude-sonnet-4-20250514": "anthropic", + "claude-opus-4-0": "anthropic", + "claude-opus-4-20250514": "anthropic", + "claude-3-7-sonnet-latest": "anthropic", + "claude-3-7-sonnet-20250219": "anthropic", + "claude-3-5-sonnet-latest": "anthropic", + "claude-3-5-sonnet-20241022": "anthropic", + "claude-3-5-sonnet-20240620": "anthropic", }; /** diff --git a/lib/agent/AnthropicCUAClient.ts b/lib/agent/AnthropicCUAClient.ts index 65b6e2df0..d1d67d5b6 100644 --- a/lib/agent/AnthropicCUAClient.ts +++ b/lib/agent/AnthropicCUAClient.ts @@ -42,6 +42,7 @@ export class AnthropicCUAClient extends AgentClient { // Process client options this.apiKey = (clientOptions?.apiKey as string) || process.env.ANTHROPIC_API_KEY || ""; + console.log("USING API KEY", this.apiKey); this.baseURL = (clientOptions?.baseURL as string) || undefined; // Get thinking budget if specified @@ -58,7 +59,7 @@ export class AnthropicCUAClient extends AgentClient { }; if (this.baseURL) { - this.clientOptions.baseUrl = this.baseURL; + this.clientOptions.baseURL = this.baseURL; } // Initialize the Anthropic client @@ -406,6 +407,11 @@ export class AnthropicCUAClient extends AgentClient { ? { type: "enabled" as const, budget_tokens: this.thinkingBudget } : undefined; + // Determine the appropriate computer type and beta flag based on model + const { computerType, betaFlag } = this.getComputerUseConfigForModel( + this.modelName, + ); + // Create the request parameters const requestParams: Record = { model: this.modelName, @@ -413,14 +419,14 @@ export class AnthropicCUAClient extends AgentClient { messages: messages, tools: [ { - type: "computer_20250124", // Use the latest version for Claude 3.7 Sonnet + type: computerType, name: "computer", display_width_px: this.currentViewport.width, display_height_px: this.currentViewport.height, display_number: 1, }, ], - betas: ["computer-use-2025-01-24"], + betas: [betaFlag], }; // Add system parameter if provided @@ -474,6 +480,43 @@ export class AnthropicCUAClient extends AgentClient { } } + /** + * Get the appropriate computer type and beta flag for a given model + */ + private getComputerUseConfigForModel(modelName: string): { + computerType: string; + betaFlag: string; + } { + // Claude 4 models and Claude Sonnet 3.7 use computer_20250124 with computer-use-2025-01-24 + if ( + modelName.includes("claude-3-5-sonnet") || + modelName.includes("claude-3-5") + ) { + return { + computerType: "computer_20241022", + betaFlag: "computer-use-2024-10-22", + }; + } + + // Claude 4 models and Claude Sonnet 3.7 use computer_20250124 with computer-use-2025-01-24 + if ( + modelName.includes("claude-4") || + modelName.includes("claude-3-7") || + modelName.includes("claude-3-7-sonnet") + ) { + return { + computerType: "computer_20250124", + betaFlag: "computer-use-2025-01-24", + }; + } + + // Default fallback for other models + return { + computerType: "computer_20250124", + betaFlag: "computer-use-2025-01-24", + }; + } + async takeAction( toolUseItems: ToolUseItem[], logger: (message: LogLine) => void, diff --git a/lib/agent/OpenAICUAClient.ts b/lib/agent/OpenAICUAClient.ts index f578751b8..b91cacab5 100644 --- a/lib/agent/OpenAICUAClient.ts +++ b/lib/agent/OpenAICUAClient.ts @@ -1,17 +1,17 @@ -import OpenAI from "openai"; -import { LogLine } from "../../types/log"; import { AgentAction, + AgentExecutionOptions, AgentResult, AgentType, - AgentExecutionOptions, - ResponseInputItem, - ResponseItem, ComputerCallItem, FunctionCallItem, + ResponseInputItem, + ResponseItem, } from "@/types/agent"; -import { AgentClient } from "./AgentClient"; import { AgentScreenshotProviderError } from "@/types/stagehandErrors"; +import OpenAI, { ClientOptions } from "openai"; +import { LogLine } from "../../types/log"; +import { AgentClient } from "./AgentClient"; /** * Client for OpenAI's Computer Use Assistant API @@ -34,7 +34,7 @@ export class OpenAICUAClient extends AgentClient { type: AgentType, modelName: string, userProvidedInstructions?: string, - clientOptions?: Record, + clientOptions?: ClientOptions, ) { super(type, modelName, userProvidedInstructions); @@ -43,13 +43,11 @@ export class OpenAICUAClient extends AgentClient { (clientOptions?.apiKey as string) || process.env.OPENAI_API_KEY || ""; this.organization = (clientOptions?.organization as string) || process.env.OPENAI_ORG; + this.baseURL = (clientOptions?.baseURL as string) || undefined; // Get environment if specified - if ( - clientOptions?.environment && - typeof clientOptions.environment === "string" - ) { - this.environment = clientOptions.environment; + if (this.baseURL) { + this.clientOptions.baseURL = this.baseURL; } // Store client options for reference diff --git a/lib/llm/LLMProvider.ts b/lib/llm/LLMProvider.ts index fc11c5753..a4529f4e9 100644 --- a/lib/llm/LLMProvider.ts +++ b/lib/llm/LLMProvider.ts @@ -1,8 +1,21 @@ +import { AISDKCustomProvider, AISDKProvider } from "@/types/llm"; import { UnsupportedAISDKModelProviderError, UnsupportedModelError, UnsupportedModelProviderError, } from "@/types/stagehandErrors"; +import { anthropic, createAnthropic } from "@ai-sdk/anthropic"; +import { azure, createAzure } from "@ai-sdk/azure"; +import { cerebras, createCerebras } from "@ai-sdk/cerebras"; +import { createDeepSeek, deepseek } from "@ai-sdk/deepseek"; +import { createGoogleGenerativeAI, google } from "@ai-sdk/google"; +import { createGroq, groq } from "@ai-sdk/groq"; +import { createMistral, mistral } from "@ai-sdk/mistral"; +import { createOpenAI, openai } from "@ai-sdk/openai"; +import { createPerplexity, perplexity } from "@ai-sdk/perplexity"; +import { createTogetherAI, togetherai } from "@ai-sdk/togetherai"; +import { createXai, xai } from "@ai-sdk/xai"; +import { ollama } from "ollama-ai-provider"; import { LogLine } from "../../types/log"; import { AvailableModel, @@ -17,19 +30,6 @@ import { GoogleClient } from "./GoogleClient"; import { GroqClient } from "./GroqClient"; import { LLMClient } from "./LLMClient"; import { OpenAIClient } from "./OpenAIClient"; -import { openai, createOpenAI } from "@ai-sdk/openai"; -import { anthropic, createAnthropic } from "@ai-sdk/anthropic"; -import { google, createGoogleGenerativeAI } from "@ai-sdk/google"; -import { xai, createXai } from "@ai-sdk/xai"; -import { azure, createAzure } from "@ai-sdk/azure"; -import { groq, createGroq } from "@ai-sdk/groq"; -import { cerebras, createCerebras } from "@ai-sdk/cerebras"; -import { togetherai, createTogetherAI } from "@ai-sdk/togetherai"; -import { mistral, createMistral } from "@ai-sdk/mistral"; -import { deepseek, createDeepSeek } from "@ai-sdk/deepseek"; -import { perplexity, createPerplexity } from "@ai-sdk/perplexity"; -import { ollama } from "ollama-ai-provider"; -import { AISDKProvider, AISDKCustomProvider } from "@/types/llm"; const AISDKProviders: Record = { openai, @@ -80,6 +80,10 @@ const modelToProviderMap: { [key in AvailableModel]: ModelProvider } = { "claude-3-5-sonnet-20241022": "anthropic", "claude-3-7-sonnet-20250219": "anthropic", "claude-3-7-sonnet-latest": "anthropic", + "claude-opus-4-0": "anthropic", + "claude-opus-4-20250514": "anthropic", + "claude-sonnet-4-0": "anthropic", + "claude-sonnet-4-20250514": "anthropic", "cerebras-llama-3.3-70b": "cerebras", "cerebras-llama-3.1-8b": "cerebras", "groq-llama-3.3-70b-versatile": "groq", From 026a5981d76a7732651a791513cc5b3651ae6edb Mon Sep 17 00:00:00 2001 From: Sameel Date: Wed, 9 Jul 2025 16:40:32 -1000 Subject: [PATCH 2/6] remove api key log --- lib/agent/AnthropicCUAClient.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/agent/AnthropicCUAClient.ts b/lib/agent/AnthropicCUAClient.ts index d1d67d5b6..baaa66ebc 100644 --- a/lib/agent/AnthropicCUAClient.ts +++ b/lib/agent/AnthropicCUAClient.ts @@ -42,7 +42,6 @@ export class AnthropicCUAClient extends AgentClient { // Process client options this.apiKey = (clientOptions?.apiKey as string) || process.env.ANTHROPIC_API_KEY || ""; - console.log("USING API KEY", this.apiKey); this.baseURL = (clientOptions?.baseURL as string) || undefined; // Get thinking budget if specified From c79bf7315e23d246c88e18758926b978dd502f7e Mon Sep 17 00:00:00 2001 From: Sameel Date: Wed, 9 Jul 2025 19:40:53 -0700 Subject: [PATCH 3/6] Update lib/agent/AnthropicCUAClient.ts Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- lib/agent/AnthropicCUAClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/agent/AnthropicCUAClient.ts b/lib/agent/AnthropicCUAClient.ts index baaa66ebc..cc733a86a 100644 --- a/lib/agent/AnthropicCUAClient.ts +++ b/lib/agent/AnthropicCUAClient.ts @@ -486,7 +486,7 @@ export class AnthropicCUAClient extends AgentClient { computerType: string; betaFlag: string; } { - // Claude 4 models and Claude Sonnet 3.7 use computer_20250124 with computer-use-2025-01-24 + // Claude 3.5 models use computer_20241022 with computer-use-2024-10-22 if ( modelName.includes("claude-3-5-sonnet") || modelName.includes("claude-3-5") From b71a12ee2d5de3844a4c8351522f61e9e3a172d0 Mon Sep 17 00:00:00 2001 From: Sameel Date: Wed, 9 Jul 2025 19:41:15 -0700 Subject: [PATCH 4/6] Update lib/agent/OpenAICUAClient.ts Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- lib/agent/OpenAICUAClient.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/agent/OpenAICUAClient.ts b/lib/agent/OpenAICUAClient.ts index b91cacab5..062746c9f 100644 --- a/lib/agent/OpenAICUAClient.ts +++ b/lib/agent/OpenAICUAClient.ts @@ -46,10 +46,18 @@ export class OpenAICUAClient extends AgentClient { this.baseURL = (clientOptions?.baseURL as string) || undefined; // Get environment if specified + // Store client options for reference + this.clientOptions = { + apiKey: this.apiKey, + }; + if (this.baseURL) { this.clientOptions.baseURL = this.baseURL; } + // Initialize the OpenAI client + this.client = new OpenAI(this.clientOptions); + // Store client options for reference this.clientOptions = { apiKey: this.apiKey, From df1dda44017101d4a96ae43d4e0bfc3b2902d276 Mon Sep 17 00:00:00 2001 From: Sameel Date: Wed, 9 Jul 2025 16:43:18 -1000 Subject: [PATCH 5/6] fix client options type --- lib/agent/AgentClient.ts | 3 +-- lib/agent/AnthropicCUAClient.ts | 12 ++---------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/lib/agent/AgentClient.ts b/lib/agent/AgentClient.ts index 014ac8c7c..ac0aa83af 100644 --- a/lib/agent/AgentClient.ts +++ b/lib/agent/AgentClient.ts @@ -4,7 +4,6 @@ import { AgentType, AgentExecutionOptions, } from "@/types/agent"; -import { ClientOptions } from "@anthropic-ai/sdk"; /** * Abstract base class for agent clients @@ -13,7 +12,7 @@ import { ClientOptions } from "@anthropic-ai/sdk"; export abstract class AgentClient { public type: AgentType; public modelName: string; - public clientOptions: ClientOptions; + public clientOptions: Record; public userProvidedInstructions?: string; constructor( diff --git a/lib/agent/AnthropicCUAClient.ts b/lib/agent/AnthropicCUAClient.ts index baaa66ebc..30916ccc9 100644 --- a/lib/agent/AnthropicCUAClient.ts +++ b/lib/agent/AnthropicCUAClient.ts @@ -1,4 +1,4 @@ -import Anthropic from "@anthropic-ai/sdk"; +import Anthropic, { ClientOptions } from "@anthropic-ai/sdk"; import { LogLine } from "@/types/log"; import { AgentAction, @@ -35,7 +35,7 @@ export class AnthropicCUAClient extends AgentClient { type: AgentType, modelName: string, userProvidedInstructions?: string, - clientOptions?: Record, + clientOptions?: ClientOptions, ) { super(type, modelName, userProvidedInstructions); @@ -44,14 +44,6 @@ export class AnthropicCUAClient extends AgentClient { (clientOptions?.apiKey as string) || process.env.ANTHROPIC_API_KEY || ""; this.baseURL = (clientOptions?.baseURL as string) || undefined; - // Get thinking budget if specified - if ( - clientOptions?.thinkingBudget && - typeof clientOptions.thinkingBudget === "number" - ) { - this.thinkingBudget = clientOptions.thinkingBudget; - } - // Store client options for reference this.clientOptions = { apiKey: this.apiKey, From b79c7924bd1282ed15db1f95552c22c9bb782840 Mon Sep 17 00:00:00 2001 From: Sameel Date: Wed, 9 Jul 2025 16:45:59 -1000 Subject: [PATCH 6/6] changeset --- .changeset/solid-emus-sin.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/solid-emus-sin.md diff --git a/.changeset/solid-emus-sin.md b/.changeset/solid-emus-sin.md new file mode 100644 index 000000000..9b5343e32 --- /dev/null +++ b/.changeset/solid-emus-sin.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Added support for new Anthropic computer-use models. Also added support for passing in new Anthropic models into Stagehand via the legacy (non AI SDK) format until we fully migrate.