[feat]: adding openrouter as a client to run evals on grok 4

evals/args.ts

@@ -93,7 +93,7 @@ function buildUsage(detailed = false): string {
         providerDefault,
       )})       [${chalk.yellow("OPENAI")}, ${chalk.yellow(
         "ANTHROPIC",
-      )}, ${chalk.yellow("GOOGLE")}, ${chalk.yellow("TOGETHER")}, ${chalk.yellow(
+      )}, ${chalk.yellow("GOOGLE")}, ${chalk.yellow("OPENROUTER")}, ${chalk.yellow("TOGETHER")}, ${chalk.yellow(
         "GROQ",
       )}, ${chalk.yellow("CEREBRAS")}]
 

evals/index.eval.ts

@@ -39,6 +39,7 @@ import { groq } from "@ai-sdk/groq";
 import { cerebras } from "@ai-sdk/cerebras";
 import { openai } from "@ai-sdk/openai";
 import { AISdkClient } from "@/examples/external_clients/aisdk";
+import { OpenRouterClient } from "@/lib/llm/OpenRouterClient";
 dotenv.config();
 
 /**
@@ -351,6 +352,17 @@ const generateFilteredTestcases = (): Testcase[] => {
                 ),
               ),
             });
+          } else if (input.modelName.startsWith("x-ai/")) {
+            // Handle OpenRouter models (e.g., x-ai/grok-4) using native OpenRouterClient
+            llmClient = new OpenRouterClient({
+              logger: logger.log.bind(logger),
+              enableCaching: false,
+              cache: undefined,
+              modelName: input.modelName as AvailableModel,
+              clientOptions: {
+                apiKey: process.env.OPENROUTER_API_KEY,
+              },
+            });
           } else if (input.modelName.includes("/")) {
             llmClient = new CustomOpenAIClient({
               modelName: input.modelName as AvailableModel,
@@ -361,6 +373,10 @@ const generateFilteredTestcases = (): Testcase[] => {
                 }),
               ),
             });
+          } else {
+            throw new StagehandEvalError(
+              `Unsupported model: ${input.modelName}. Please add support for this model in the evals.`,
+            );
           }
           const taskInput = await initStagehand({
             logger,

evals/taskConfig.ts

@@ -35,6 +35,8 @@ const ALL_EVAL_MODELS = [
   "o3",
   "o3-mini",
   "o4-mini",
+  // OPENROUTER
+  "x-ai/grok-4",
   // TOGETHER - META
   "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
   "meta-llama/Llama-3.3-70B-Instruct-Turbo",
@@ -136,6 +138,8 @@ const filterModelByProvider = (model: string, provider: string): boolean => {
     return modelLower.startsWith("claude");
   } else if (provider === "google") {
     return modelLower.startsWith("gemini");
+  } else if (provider === "openrouter") {
+    return modelLower.startsWith("x-ai/");
   } else if (provider === "together") {
     return (
       modelLower.startsWith("meta-llama") ||

lib/llm/LLMProvider.ts

@@ -17,6 +17,7 @@ import { GoogleClient } from "./GoogleClient";
 import { GroqClient } from "./GroqClient";
 import { LLMClient } from "./LLMClient";
 import { OpenAIClient } from "./OpenAIClient";
+import { OpenRouterClient } from "./OpenRouterClient";
 import { openai, createOpenAI } from "@ai-sdk/openai";
 import { anthropic, createAnthropic } from "@ai-sdk/anthropic";
 import { google, createGoogleGenerativeAI } from "@ai-sdk/google";
@@ -91,6 +92,7 @@ const modelToProviderMap: { [key in AvailableModel]: ModelProvider } = {
   "gemini-2.0-flash": "google",
   "gemini-2.5-flash-preview-04-17": "google",
   "gemini-2.5-pro-preview-03-25": "google",
+  "x-ai/grok-4": "openrouter",
 };
 
 function getAISDKLanguageModel(
@@ -221,6 +223,14 @@ export class LLMProvider {
           modelName: availableModel,
           clientOptions,
         });
+      case "openrouter":
+        return new OpenRouterClient({
+          logger: this.logger,
+          enableCaching: this.enableCaching,
+          cache: this.cache,
+          modelName: availableModel,
+          clientOptions,
+        });
       default:
         throw new UnsupportedModelProviderError([
           ...new Set(Object.values(modelToProviderMap)),