Support Dropdowns

examples/form_filling_jotform.ts

@@ -0,0 +1,36 @@
+import { Stagehand } from "@/dist";
+// import { GroqClient } from "@/lib/llm/GroqClient";
+// import { CerebrasClient } from "../lib/llm/CerebrasClient";
+import StagehandConfig from "@/stagehand.config";
+
+async function example() {
+  const stagehand = new Stagehand({
+    ...StagehandConfig,
+    env: "LOCAL",
+    modelName: "gpt-4o",
+    modelClientOptions: {
+      apiKey: process.env.OPENAI_API_KEY,
+    },
+  });
+  await stagehand.init();
+  const page = stagehand.page;
+  await page.goto("https://form.jotform.com/251018791931458");
+  await page.act("enter john in first name");
+  await page.act("enter doe in last name");
+  await page.act("enter 43 in age");
+  await page.act("enter 1234567890 in phone number");
+  await page.act("enter [email protected] in email");
+  await page.act("select male in gender dropdown");
+  await page.act("select no in are you taking any medication dropdown");
+  await page.act("select yes in medication allergies dropdown");
+  await page.act("select no in chronic medical conditions");
+  await page.act("select yes in tobacco products");
+  await page.act("select india in country of origin");
+  await page.act("select ep in visa status");
+  await page.act("submit form");
+  await stagehand.close();
+}
+
+(async () => {
+  await example();
+})();

lib/prompt.ts

@@ -202,7 +202,8 @@ export function buildActObservePrompt(
   ONLY return one action. If multiple actions are relevant, return the most relevant one. 
   If the user is asking to scroll to a position on the page, e.g., 'halfway' or 0.75, etc, you must return the argument formatted as the correct percentage, e.g., '50%' or '75%', etc.
   If the user is asking to scroll to the next chunk/previous chunk, choose the nextChunk/prevChunk method. No arguments are required here.
-  If the action implies a key press, e.g., 'press enter', 'press a', 'press space', etc., always choose the press method with the appropriate key as argument — e.g. 'a', 'Enter', 'Space'. Do not choose a click action on an on-screen keyboard. Capitalize the first character like 'Enter', 'Tab', 'Escape' only for special keys.`;
+  If the action implies a key press, e.g., 'press enter', 'press a', 'press space', etc., always choose the press method with the appropriate key as argument — e.g. 'a', 'Enter', 'Space'. Do not choose a click action on an on-screen keyboard. Capitalize the first character like 'Enter', 'Tab', 'Escape' only for special keys.
+  For form elements, if the action involves selecting an option from a dropdown or combobox, use 'selectOption' with the option value or label as the argument.`;
 
   // Add variable names (not values) to the instruction if any
   if (variables && Object.keys(variables).length > 0) {

package.json

@@ -18,7 +18,8 @@
     "ai-sdk-client": "npm run build && tsx examples/ai_sdk_example.ts",
     "actionable_observe_example": "npm run build && tsx examples/actionable_observe_example.ts",
     "form-filling-sensible-cerebras": "npm run build && tsx examples/form_filling_sensible_cerebras.ts",
-    "form-filling-sensible-openai": "npm run build && tsx examples/form_filling_sensible_openai.ts",
+    "form-filling-sensible-openai": "npm run build && tsx examples/form_filling_sensible.ts",
+    "form-filling-jotform": "npm run build && tsx examples/form_filling_jotform.ts",
     "google-enter": "npm run build && tsx examples/google_enter.ts",
     "try-wordle": "npm run build && tsx examples/try_wordle.ts",
     "format": "prettier --write .",

types/act.ts

@@ -39,6 +39,7 @@ export enum SupportedPlaywrightAction {
   SCROLL = "scrollTo",
   NEXT_CHUNK = "nextChunk",
   PREV_CHUNK = "prevChunk",
+  SELECTOPTION = "selectOption",
 }
 
 /**