iframes

.changeset/sixty-bikes-grab.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand-lib": minor
+---
+
+iframe support

docs/reference/act.mdx

@@ -45,6 +45,10 @@ You can pass an `ObserveResult` to `act()` to perform the suggested action, whic
     Variables to use in the action. Variables in the action string are referenced using %variable_name%
   </ParamField>
 
+  <ParamField path="iframes" type="boolean" optional>
+    Set `iframes: true` if the target element exists within an iframe
+  </ParamField>
+
   <ParamField path="domSettleTimeoutMs" type="number" optional>
     Timeout in milliseconds for waiting for the DOM to settle
   </ParamField>

docs/reference/extract.mdx

@@ -107,6 +107,10 @@ const apartments = await stagehand.page.extract({
     Defines the structure of the data to extract
   </ParamField>
 
+  <ParamField path="iframes" type="boolean" optional>
+      Set `iframes: true` if the extraction content exists within an iframe.
+  </ParamField>
+
   <ParamField path="useTextExtract" type="boolean" deprecated>
     This field is now **deprecated** and has no effect.
   </ParamField>

docs/reference/observe.mdx

@@ -37,6 +37,10 @@ Observe can also return a suggested action for the candidate element by setting
     Returns an observe result object that contains a suggested action for the candidate element. The suggestion includes method, and arguments (if any). Defaults to `true`.
   </ParamField>
 
+  <ParamField path="iframes" type="boolean" optional>
+    Set `iframes: true` if content from iframes should be included in the observation.
+  </ParamField>
+
   <ParamField path="modelName" type="AvailableModel" optional>
     Specifies the model to use
   </ParamField>

evals/evals.config.json

@@ -346,6 +346,22 @@
     {
       "name": "login",
       "categories": ["act", "regression"]
+    },
+    {
+      "name": "iframe_hn",
+      "categories": ["extract"]
+    },
+    {
+      "name": "iframe_same_proc",
+      "categories": ["act"]
+    },
+    {
+      "name": "iframe_form_filling",
+      "categories": ["act"]
+    },
+    {
+      "name": "iframes_nested",
+      "categories": ["act"]
     }
   ]
 }

evals/initStagehand.ts

@@ -38,6 +38,7 @@ const StagehandConfig = {
   enableCaching,
   domSettleTimeoutMs: 30_000,
   disablePino: true,
+  experimental: true,
   browserbaseSessionCreateParams: {
     projectId: process.env.BROWSERBASE_PROJECT_ID!,
     browserSettings: {

evals/tasks/iframe_form_filling.ts

@@ -0,0 +1,70 @@
+import { EvalFunction } from "@/types/evals";
+
+export const iframe_form_filling: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  const page = stagehand.page;
+  await page.goto(
+    "https://browserbase.github.io/stagehand-eval-sites/sites/iframe-form-filling/",
+  );
+
+  await page.act({
+    action: "type 'nunya' into the 'first name' field",
+    iframes: true,
+  });
+  await page.act({
+    action: "type 'business' into the 'last name' field",
+    iframes: true,
+  });
+  await page.act({
+    action: "type '[email protected]' into the 'email' field",
+    iframes: true,
+  });
+  await page.act({
+    action: "click 'phone' as the preferred contact method",
+    iframes: true,
+  });
+  await page.act({
+    action: "type 'yooooooooooooooo' into the message box",
+    iframes: true,
+  });
+
+  const iframe = page.frameLocator("iframe");
+
+  const firstNameValue: string = await iframe
+    .locator('input[placeholder="Jane"]')
+    .inputValue();
+
+  const lastNameValue: string = await iframe
+    .locator('input[placeholder="Doe"]')
+    .inputValue();
+
+  const emailValue: string = await iframe
+    .locator('input[placeholder="[email protected]"]')
+    .inputValue();
+
+  const contactValue: boolean = await iframe
+    .locator("xpath=/html/body/main/section[1]/form/fieldset/label[2]/input")
+    .isChecked();
+
+  const messageValue: string = await iframe
+    .locator('textarea[placeholder="Say hello…"]')
+    .inputValue();
+
+  const passed: boolean =
+    firstNameValue.toLowerCase().trim() === "nunya" &&
+    lastNameValue.toLowerCase().trim() === "business" &&
+    emailValue.toLowerCase() === "[email protected]" &&
+    messageValue.toLowerCase() === "yooooooooooooooo" &&
+    contactValue;
+
+  return {
+    _success: passed,
+    logs: logger.getLogs(),
+    debugUrl,
+    sessionUrl,
+  };
+};

evals/tasks/iframe_hn.ts

@@ -0,0 +1,48 @@
+import { EvalFunction } from "@/types/evals";
+import { z } from "zod";
+
+export const iframe_hn: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  const page = stagehand.page;
+  await page.goto(
+    "https://browserbase.github.io/stagehand-eval-sites/sites/iframe-hn/",
+  );
+
+  const result = await page.extract({
+    instruction: "extract the title of the first hackernews story",
+    schema: z.object({
+      story_title: z.string(),
+    }),
+    iframes: true,
+  });
+
+  await stagehand.close();
+
+  const title = result.story_title.toLowerCase();
+  const expectedTitleSubstring = "overengineered anchor links";
+
+  if (!title.includes(expectedTitleSubstring)) {
+    logger.error({
+      message: `Extracted title: ${title} does not contain expected substring: ${expectedTitleSubstring}`,
+      level: 0,
+    });
+    return {
+      _success: false,
+      error: `Extracted title: ${title} does not contain expected substring: ${expectedTitleSubstring}`,
+      logs: logger.getLogs(),
+      debugUrl,
+      sessionUrl,
+    };
+  }
+
+  return {
+    _success: true,
+    logs: logger.getLogs(),
+    debugUrl,
+    sessionUrl,
+  };
+};

evals/tasks/iframe_same_proc.ts

@@ -0,0 +1,45 @@
+import { EvalFunction } from "@/types/evals";
+
+export const iframe_same_proc: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  const page = stagehand.page;
+  await page.goto(
+    "https://browserbase.github.io/stagehand-eval-sites/sites/iframe-same-proc/",
+  );
+
+  await page.act({
+    action: "type 'stagehand' into the 'your name' field",
+    iframes: true,
+  });
+
+  // overly specific prompting is okay here. we are just trying to evaluate whether
+  // we are properly traversing iframes
+  await page.act({
+    action:
+      "select 'Green' from the favorite colour dropdown. Ensure the word 'Green' is capitalized. Choose the selectOption playwright method.",
+    iframes: true,
+  });
+
+  const iframe = page.frameLocator("iframe");
+
+  const nameValue: string = await iframe
+    .locator('input[placeholder="Alice"]')
+    .inputValue();
+
+  const colorValue: string = await iframe.locator("select").inputValue();
+
+  const passed: boolean =
+    nameValue.toLowerCase().trim() === "stagehand" &&
+    colorValue.toLowerCase().trim() === "green";
+
+  return {
+    _success: passed,
+    logs: logger.getLogs(),
+    debugUrl,
+    sessionUrl,
+  };
+};

evals/tasks/iframes_nested.ts

@@ -0,0 +1,49 @@
+import { EvalFunction } from "@/types/evals";
+import { FrameLocator } from "@playwright/test";
+
+export const iframes_nested: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  stagehand,
+  logger,
+}) => {
+  const page = stagehand.page;
+  try {
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/nested-iframes/",
+    );
+
+    await page.act({
+      action: "type 'stagehand' into the 'username' field",
+      iframes: true,
+    });
+
+    const inner: FrameLocator = page
+      .frameLocator("iframe.lvl1") // level 1
+      .frameLocator("iframe.lvl2") // level 2
+      .frameLocator("iframe.lvl3"); // level 3 – form lives here
+
+    const usernameText = await inner
+      .locator('input[name="username"]')
+      .inputValue();
+
+    const passed: boolean = usernameText.toLowerCase().trim() === "stagehand";
+
+    return {
+      _success: passed,
+      logs: logger.getLogs(),
+      debugUrl,
+      sessionUrl,
+    };
+  } catch (error) {
+    return {
+      _success: false,
+      logs: logger.getLogs(),
+      debugUrl,
+      sessionUrl,
+      error,
+    };
+  } finally {
+    await stagehand.close();
+  }
+};