robustness to failures, gpt-5-mini as a judge for webarenalite

Author: patricebechard

browsergym/core/src/browsergym/core/env.py

@@ -1,6 +1,8 @@
 import copy
 import logging
 import re
+import json
+import os
 import time
 from abc import ABC
 from pathlib import Path
@@ -10,7 +12,7 @@
 import numpy as np
 import playwright.sync_api
 
-from . import _get_global_playwright
+from . import _get_global_playwright, _set_global_playwright
 from .action.base import execute_python_code
 from .action.highlevel import HighLevelActionSet
 from .chat import Chat
@@ -256,7 +258,21 @@ def override_property(task, env, property):
         # use the global Playwright instance
         pw: playwright.sync_api.Playwright = _get_global_playwright()
         # important: change playwright's test id attribute from "data-testid" to "bid"
-        pw.selectors.set_test_id_attribute(BROWSERGYM_ID_ATTRIBUTE)
+        try:
+            pw.selectors.set_test_id_attribute(BROWSERGYM_ID_ATTRIBUTE)
+        except RuntimeError as exc:
+            if "no running event loop" not in str(exc):
+                raise
+            logger.warning(
+                "Playwright global instance lost its event loop; restarting Playwright."
+            )
+            try:
+                pw.stop()
+            except Exception:
+                logger.debug("Failed to stop stale Playwright instance.", exc_info=True)
+            pw = playwright.sync_api.sync_playwright().start()
+            _set_global_playwright(pw)
+            pw.selectors.set_test_id_attribute(BROWSERGYM_ID_ATTRIBUTE)
         args = [
             (
                 f"--window-size={viewport['width']},{viewport['height']}"
@@ -281,6 +297,7 @@ def override_property(task, env, property):
         )
 
         # create a new browser context for pages
+        extra_http_headers = json.loads(os.getenv("EXTRA_HTTP_HEADERS", "{}"))
         self.context = self.browser.new_context(
             no_viewport=True if self.resizeable_window else None,
             viewport=viewport if not self.resizeable_window else None,
@@ -291,6 +308,7 @@ def override_property(task, env, property):
             locale=locale,
             timezone_id=timezone_id,
             ignore_https_errors=True,
+            extra_http_headers=extra_http_headers,
             # will raise an Exception if above args are overriden
             **self.pw_context_kwargs,
         )

browsergym/webarenalite/src/browsergym/webarenalite/helper_functions.py

@@ -247,7 +247,8 @@ def llm_fuzzy_match(pred: str, reference: str, question: str) -> float:
     ]
 
     raw_response = generate_from_openai_chat_completion(
-        model="gpt-4-1106-preview",
+        # model="gpt-4-1106-preview",
+        model="gpt-5-mini",
         messages=messages,
         temperature=0,
         max_tokens=768,
@@ -283,7 +284,8 @@ def llm_ua_match(pred: str, reference: str, question: str) -> float:
     ]
 
     raw_response = generate_from_openai_chat_completion(
-        model="gpt-4-1106-preview",
+        # model="gpt-4-1106-preview",
+        model="gpt-5-mini",
         messages=messages,
         temperature=0,
         max_tokens=768,