[chore] remove and update evals (#886)

# why
- some of these evals were written when stagehand had fundamentally
different behaviour, so they needed to be updated to be useful
evaluations
# what changed
- removed some 'un-passable' evals, and updated others to reflect
current recommended stagehand usage patterns
# test plan
- this is it

Author: seanmcguire12

evals/evals.config.json

@@ -105,14 +105,6 @@
       "name": "costar",
       "categories": ["experimental"]
     },
-    {
-      "name": "expedia",
-      "categories": ["experimental"]
-    },
-    {
-      "name": "expedia_search",
-      "categories": ["experimental"]
-    },
     {
       "name": "extract_aigrant_companies",
       "categories": ["regression"]
@@ -133,10 +125,6 @@
       "name": "extract_snowshoeing_destinations",
       "categories": ["experimental"]
     },
-    {
-      "name": "google_jobs",
-      "categories": ["experimental"]
-    },
     {
       "name": "homedepot",
       "categories": ["experimental"]

evals/tasks/expedia.ts

@@ -11,15 +11,15 @@ export const extract_partners: EvalFunction = async ({
     await stagehand.page.goto("https://ramp.com");
 
     await stagehand.page.act({
-      action: "move down to the bottom of the page.",
+      action: "scroll to the bottom of the page",
     });
 
     await stagehand.page.act({
       action: "Close the popup.",
     });
 
     await stagehand.page.act({
-      action: "Find and click on the link that leads to the partners page.",
+      action: "click on the link that leads to the partners page.",
     });
 
     const partners = await stagehand.page.extract({

evals/tasks/expedia_search.ts

@@ -24,9 +24,6 @@ export const extract_press_releases: EvalFunction = async ({
   try {
     await stagehand.page.goto(
       "https://browserbase.github.io/stagehand-eval-sites/sites/press-releases/",
-      {
-        waitUntil: "networkidle",
-      },
     );
     await new Promise((resolve) => setTimeout(resolve, 5000));
 

evals/tasks/extract_partners.ts

@@ -9,21 +9,17 @@ export const homedepot: EvalFunction = async ({
 }) => {
   try {
     await stagehand.page.goto("https://www.homedepot.com/");
-    await stagehand.page.act("search for gas grills");
+    await stagehand.page.act("enter 'gas grills' in the search bar");
+    await stagehand.page.act("press enter");
     await stagehand.page.act("click on the best selling gas grill");
     await stagehand.page.act("click on the Product Details");
-    await stagehand.page.act("find the Primary Burner BTU");
 
     const productSpecs = await stagehand.page.extract({
       instruction: "Extract the Primary exact Burner BTU of the product",
       schema: z.object({
-        productSpecs: z
-          .array(
-            z.object({
-              burnerBTU: z.string().describe("Primary Burner BTU exact value"),
-            }),
-          )
-          .describe("Gas grill Primary Burner BTU exact value"),
+        productSpecs: z.object({
+          burnerBTU: z.number().describe("Primary Burner BTU exact value"),
+        }),
       }),
     });
 
@@ -38,11 +34,7 @@ export const homedepot: EvalFunction = async ({
       },
     });
 
-    if (
-      !productSpecs ||
-      !productSpecs.productSpecs ||
-      productSpecs.productSpecs.length !== 1
-    ) {
+    if (!productSpecs || !productSpecs.productSpecs) {
       await stagehand.close();
 
       return {
@@ -54,14 +46,12 @@ export const homedepot: EvalFunction = async ({
       };
     }
 
-    const hasFourZerosAndOne4 =
-      (productSpecs.productSpecs[0].burnerBTU.match(/0/g) || []).length === 4 &&
-      (productSpecs.productSpecs[0].burnerBTU.match(/4/g) || []).length === 1;
+    const isLargerThan1000 = productSpecs.productSpecs.burnerBTU >= 10000;
 
     await stagehand.close();
 
     return {
-      _success: hasFourZerosAndOne4,
+      _success: isLargerThan1000,
       productSpecs,
       debugUrl,
       sessionUrl,

evals/tasks/extract_press_releases.ts

@@ -21,9 +21,7 @@ export const os_dropdown: EvalFunction = async ({
       "choose 'Smog Check Technician' from the 'License Type' dropdown",
     );
     const selectedOption = await page
-      .locator(
-        "xpath=/html/body/form/div[1]/div[3]/article/div[2]/div[1]/select[2] >> option:checked",
-      )
+      .locator("#licenseType >> option:checked")
       .textContent();
 
     if (selectedOption === "Smog Check Technician") {

evals/tasks/google_jobs.ts

@@ -7,19 +7,12 @@ export const rakuten_jp: EvalFunction = async ({
   logger,
 }) => {
   await stagehand.page.goto("https://www.rakuten.co.jp/");
-  await stagehand.page.act({ action: "click on online supermarket" });
-
-  await stagehand.page.act({ action: "if there is a popup, close it" });
-
-  await stagehand.page.act({
-    action: "navigate to Inageya Online Supermarket",
-  });
-  await stagehand.page.act({ action: "click the search bar input" });
-  await stagehand.page.act({ action: "search for '香菜'" });
 
+  await stagehand.page.act({ action: "type '香菜' into the search bar" });
+  await stagehand.page.act({ action: "press enter" });
   const url = stagehand.page.url();
   const successUrl =
-    "https://netsuper.rakuten.co.jp/inageya/search/?keyword=%E9%A6%99%E8%8F%9C";
+    "https://search.rakuten.co.jp/search/mall/%E9%A6%99%E8%8F%9C/";
 
   await stagehand.close();
 

evals/tasks/homedepot.ts

@@ -14,6 +14,13 @@ export const ted_talk: EvalFunction = async ({
       waitUntil: "domcontentloaded",
     },
   );
+
+  await stagehand.page.act({
+    action: "scroll 10% down the page",
+  });
+
+  await new Promise((resolve) => setTimeout(resolve, 5000));
+
   await stagehand.page.act({
     action:
       "Click the link that takes you to the page about the 'Culture' topic",