expose a11y

examples/example.ts

@@ -17,6 +17,9 @@ async function example() {
   /**
    * Add your code here!
    */
+  // delete before pushing to main
+  const tree = await stagehand.page.getAccessibilityTree();
+  console.log(tree);
   await stagehand.close();
 }
 

lib/StagehandPage.ts

@@ -16,8 +16,9 @@ import { StagehandObserveHandler } from "./handlers/observeHandler";
 import { ActOptions, ActResult, GotoOptions, Stagehand } from "./index";
 import { LLMClient } from "./llm/LLMClient";
 import { StagehandContext } from "./StagehandContext";
-import { EnhancedContext } from "../types/context";
+import { EnhancedContext, TreeResult } from "../types/context";
 import { clearOverlays } from "./utils";
+import { getAccessibilityTree } from "./a11y/utils";
 
 const BROWSERBASE_REGION_DOMAIN = {
   "us-west-2": "wss://connect.usw2.browserbase.com",
@@ -58,6 +59,7 @@ export class StagehandPage {
           (prop === ("act" as keyof Page) ||
             prop === ("extract" as keyof Page) ||
             prop === ("observe" as keyof Page) ||
+            prop === ("getAccessibilityTree" as keyof Page) ||
             prop === ("on" as keyof Page))
         ) {
           return () => {
@@ -248,6 +250,12 @@ export class StagehandPage {
           };
         }
 
+        if (prop === "getAccessibilityTree") {
+          return async () => {
+            return await this.getAccessibilityTree();
+          };
+        }
+
         // Handle goto specially
         if (prop === "goto") {
           return async (url: string, options: GotoOptions) => {
@@ -679,7 +687,7 @@ export class StagehandPage {
 
   async observe(
     instructionOrOptions?: string | ObserveOptions,
-  ): Promise<ObserveResult[]> {
+  ): Promise<TreeResult | ObserveResult[]> {
     if (!this.observeHandler) {
       throw new Error("Observe handler not initialized");
     }
@@ -829,4 +837,8 @@ export class StagehandPage {
   async disableCDP(domain: string): Promise<void> {
     await this.sendCDP(`${domain}.disable`, {});
   }
+
+  async getAccessibilityTree(): Promise<TreeResult> {
+    return getAccessibilityTree(this, this.stagehand.logger);
+  }
 }

lib/a11y/utils.ts

@@ -244,6 +244,9 @@ export async function buildHierarchicalTree(
 
 /**
  * Retrieves the full accessibility tree via CDP and transforms it into a hierarchical structure.
+ *
+ * DO NOT USE THIS FUNCTION DIRECTLY.
+ * Instead, use `StagehandPage.getAccessibilityTree()`
  */
 export async function getAccessibilityTree(
   page: StagehandPage,

lib/handlers/observeHandler.ts

@@ -4,10 +4,7 @@ import { observe } from "../inference";
 import { LLMClient } from "../llm/LLMClient";
 import { StagehandPage } from "../StagehandPage";
 import { generateId, drawObserveOverlay } from "../utils";
-import {
-  getAccessibilityTree,
-  getXPathByResolvedObjectId,
-} from "../a11y/utils";
+import { getXPathByResolvedObjectId } from "../a11y/utils";
 import { AccessibilityNode } from "../../types/context";
 
 export class StagehandObserveHandler {
@@ -67,7 +64,12 @@ export class StagehandObserveHandler {
     drawOverlay?: boolean;
   }) {
     if (!instruction) {
-      instruction = `Find elements that can be used for any future actions in the page. These may be navigation links, related pages, section/subsection links, buttons, or other interactive elements. Be comprehensive: if there are multiple elements that may be relevant for future actions, return all of them.`;
+      this.logger({
+        category: "observation",
+        message: "No instruction provided, returning hybrid tree",
+        level: 1,
+      });
+      return await this.stagehandPage.getAccessibilityTree();
     }
 
     this.logger({
@@ -88,7 +90,7 @@ export class StagehandObserveHandler {
     const useAccessibilityTree = !onlyVisible;
     if (useAccessibilityTree) {
       await this.stagehandPage._waitForSettledDom();
-      const tree = await getAccessibilityTree(this.stagehandPage, this.logger);
+      const tree = await this.stagehandPage.getAccessibilityTree();
       this.logger({
         category: "observation",
         message: "Getting accessibility tree data",

types/page.ts

@@ -12,6 +12,7 @@ import type {
   ObserveOptions,
   ObserveResult,
 } from "./stagehand";
+import { TreeResult as TreeResultType } from "./context";
 
 export const defaultExtractSchema = z.object({
   extraction: z.string(),
@@ -21,6 +22,9 @@ export const pageTextSchema = z.object({
   page_text: z.string(),
 });
 
+// Need to re-export TreeResult to make it into dist/types
+export type TreeResult = TreeResultType;
+
 export interface Page extends Omit<PlaywrightPage, "on"> {
   act(action: string): Promise<ActResult>;
   act(options: ActOptions): Promise<ActResult>;
@@ -34,13 +38,15 @@ export interface Page extends Omit<PlaywrightPage, "on"> {
   ): Promise<ExtractResult<T>>;
   extract(): Promise<ExtractResult<typeof pageTextSchema>>;
 
-  observe(): Promise<ObserveResult[]>;
+  observe(): Promise<TreeResult>;
   observe(instruction: string): Promise<ObserveResult[]>;
   observe(options?: ObserveOptions): Promise<ObserveResult[]>;
 
   on: {
     (event: "popup", listener: (page: Page) => unknown): Page;
   } & PlaywrightPage["on"];
+
+  getAccessibilityTree(): Promise<TreeResult>;
 }
 
 // Empty type for now, but will be used in the future