From a3d6e6218cc5028b85d507895be9f95eb406d8e0 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Mon, 28 Apr 2025 13:11:08 -0400 Subject: [PATCH 1/3] Update CDP Mode --- seleniumbase/core/browser_launcher.py | 4 + seleniumbase/plugins/sb_manager.py | 6 +- seleniumbase/undetected/cdp_driver/browser.py | 125 +++++++----------- .../undetected/cdp_driver/cdp_util.py | 5 + seleniumbase/undetected/cdp_driver/tab.py | 16 ++- 5 files changed, 78 insertions(+), 78 deletions(-) diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index 1483fcafd3f..7c90647f34d 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -546,12 +546,15 @@ def uc_open_with_cdp_mode(driver, url=None): headless = False headed = None xvfb = None + binary_location = None if hasattr(sb_config, "headless"): headless = sb_config.headless if hasattr(sb_config, "headed"): headed = sb_config.headed if hasattr(sb_config, "xvfb"): xvfb = sb_config.xvfb + if hasattr(sb_config, "binary_location"): + binary_location = sb_config.binary_location loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -562,6 +565,7 @@ def uc_open_with_cdp_mode(driver, url=None): headless=headless, headed=headed, xvfb=xvfb, + browser_executable_path=binary_location, ) ) loop.run_until_complete(driver.cdp_base.wait(0)) diff --git a/seleniumbase/plugins/sb_manager.py b/seleniumbase/plugins/sb_manager.py index 510c3d9e03d..70491d40cfa 100644 --- a/seleniumbase/plugins/sb_manager.py +++ b/seleniumbase/plugins/sb_manager.py @@ -1234,7 +1234,8 @@ def SB( sb._has_failure = False # This may change with suppress(Exception): - stack_base = traceback.format_stack()[0].split(os.sep)[-1] + stack_base = traceback.format_stack()[0].split("with SB(")[0] + stack_base = stack_base.split(os.sep)[-1] test_base = stack_base.split(", in ")[0] filename = test_base.split('"')[0] methodname = ".line_" + test_base.split(", line ")[-1] @@ -1251,7 +1252,8 @@ def SB( c1 = colorama.Fore.GREEN b1 = colorama.Style.BRIGHT cr = colorama.Style.RESET_ALL - stack_base = traceback.format_stack()[0].split(os.sep)[-1] + stack_base = traceback.format_stack()[0].split("with SB(")[0] + stack_base = stack_base.split(os.sep)[-1] test_name = stack_base.split(", in ")[0].replace('", line ', ":") test_name += ":SB" start_text = "=== {%s} starts ===" % test_name diff --git a/seleniumbase/undetected/cdp_driver/browser.py b/seleniumbase/undetected/cdp_driver/browser.py index d5c59d68bd4..e3e193ec122 100644 --- a/seleniumbase/undetected/cdp_driver/browser.py +++ b/seleniumbase/undetected/cdp_driver/browser.py @@ -46,9 +46,8 @@ def deconstruct_browser(): logger.debug( "Problem removing data dir %s\n" "Consider checking whether it's there " - "and remove it by hand\nerror: %s", - _.config.user_data_dir, - e, + "and remove it by hand\nerror: %s" + % (_.config.user_data_dir, e) ) break time.sleep(0.15) @@ -190,7 +189,7 @@ async def wait(self, time: Union[float, int] = 1) -> Browser: sleep = wait """Alias for wait""" - def _handle_target_update( + async def _handle_target_update( self, event: Union[ cdp.target.TargetInfoChanged, @@ -224,21 +223,21 @@ def _handle_target_update( current_tab.target = target_info elif isinstance(event, cdp.target.TargetCreated): target_info: cdp.target.TargetInfo = event.target_info - from .tab import Tab - - new_target = Tab( - ( - f"ws://{self.config.host}:{self.config.port}" - f"/devtools/{target_info.type_ or 'page'}" - f"/{target_info.target_id}" - ), - target=target_info, - browser=self, - ) - self.targets.append(new_target) - logger.debug( - "Target #%d created => %s", len(self.targets), new_target + websocket_url = ( + f"ws://{self.config.host}:{self.config.port}" + f"/devtools/{target_info.type_ or 'page'}" + f"/{target_info.target_id}" ) + async with tab.Tab( + websocket_url=websocket_url, + target=target_info, + browser=self + ) as new_target: + self.targets.append(new_target) + logger.debug( + "Target #%d created => %s" + % (len(self.targets), new_target) + ) elif isinstance(event, cdp.target.TargetDestroyed): current_tab = next( filter( @@ -287,62 +286,42 @@ async def get( connection: tab.Tab = next( filter(lambda item: item.type_ == "page", self.targets) ) - if hasattr(sb_config, "_cdp_locale") and sb_config._cdp_locale: - await connection.send(cdp.page.navigate("about:blank")) - if ( - hasattr(sb_config, "_cdp_user_agent") - and sb_config._cdp_user_agent - ): - pass - elif ( - hasattr(sb_config, "_cdp_platform") - and sb_config._cdp_platform - ): - pass - else: - await connection.set_locale(sb_config._cdp_locale) - if hasattr(sb_config, "_cdp_timezone") and sb_config._cdp_timezone: - await connection.send(cdp.page.navigate("about:blank")) - await connection.set_timezone(sb_config._cdp_timezone) + _cdp_timezone = None + _cdp_user_agent = "" + _cdp_locale = None + _cdp_platform = None + _cdp_geolocation = None + if ( + hasattr(sb_config, "_cdp_timezone") and sb_config._cdp_timezone + ): + _cdp_timezone = sb_config._cdp_timezone if ( hasattr(sb_config, "_cdp_user_agent") and sb_config._cdp_user_agent ): - await connection.send(cdp.page.navigate("about:blank")) - if hasattr(sb_config, "_cdp_locale") and sb_config._cdp_locale: - _cdp_platform = None - if ( - hasattr(sb_config, "_cdp_platform") - and sb_config._cdp_platform - ): - _cdp_platform = sb_config._cdp_platform - await connection.set_user_agent( - sb_config._cdp_user_agent, - sb_config._cdp_locale, - _cdp_platform, - ) - else: - await connection.set_user_agent(sb_config._cdp_user_agent) - elif ( - hasattr(sb_config, "_cdp_platform") and sb_config._cdp_platform - ): - await connection.send(cdp.page.navigate("about:blank")) - if hasattr(sb_config, "_cdp_locale") and sb_config._cdp_locale: - _cdp_platform = sb_config._cdp_platform - await connection.set_user_agent( - accept_language=sb_config._cdp_locale, - platform=_cdp_platform, - ) - else: - await connection.set_user_agent( - platform=sb_config._cdp_platform - ) + _cdp_user_agent = sb_config._cdp_user_agent + if hasattr(sb_config, "_cdp_locale") and sb_config._cdp_locale: + _cdp_locale = sb_config._cdp_locale + if hasattr(sb_config, "_cdp_platform") and sb_config._cdp_platform: + _cdp_platform = sb_config._cdp_platform if ( hasattr(sb_config, "_cdp_geolocation") and sb_config._cdp_geolocation ): + _cdp_geolocation = sb_config._cdp_geolocation + if _cdp_timezone: + await connection.send(cdp.page.navigate("about:blank")) + await connection.set_timezone(_cdp_timezone) + if _cdp_user_agent or _cdp_locale or _cdp_platform: + await connection.send(cdp.page.navigate("about:blank")) + await connection.set_user_agent( + user_agent=_cdp_user_agent, + accept_language=_cdp_locale, + platform=_cdp_platform, + ) + if _cdp_geolocation: await connection.send(cdp.page.navigate("about:blank")) - await connection.set_geolocation(sb_config._cdp_geolocation) + await connection.set_geolocation(_cdp_geolocation) # Use the tab to navigate to new url frame_id, loader_id, *_ = await connection.send( cdp.page.navigate(url) @@ -376,8 +355,8 @@ async def start(self=None) -> Browser: self.config.port = util.free_port() if not connect_existing: logger.debug( - "BROWSER EXECUTABLE PATH: %s", - self.config.browser_executable_path, + "BROWSER EXECUTABLE PATH: %s" + % self.config.browser_executable_path, ) if not pathlib.Path(self.config.browser_executable_path).exists(): raise FileNotFoundError( @@ -782,10 +761,8 @@ async def save(self, file: PathLike = ".session.dat", pattern: str = ".*"): for cookie in cookies: for match in pattern.finditer(str(cookie.__dict__)): logger.debug( - "Saved cookie for matching pattern '%s' => (%s: %s)", - pattern.pattern, - cookie.name, - cookie.value, + "Saved cookie for matching pattern '%s' => (%s: %s)" + % (pattern.pattern, cookie.name, cookie.value) ) included_cookies.append(cookie) break @@ -822,10 +799,8 @@ async def load(self, file: PathLike = ".session.dat", pattern: str = ".*"): for match in pattern.finditer(str(cookie.__dict__)): included_cookies.append(cookie) logger.debug( - "Loaded cookie for matching pattern '%s' => (%s: %s)", - pattern.pattern, - cookie.name, - cookie.value, + "Loaded cookie for matching pattern '%s' => (%s: %s)" + % (pattern.pattern, cookie.name, cookie.value) ) break await connection.send(cdp.network.set_cookies(included_cookies)) diff --git a/seleniumbase/undetected/cdp_driver/cdp_util.py b/seleniumbase/undetected/cdp_driver/cdp_util.py index b2a5bc16f8a..a50ae3d43a6 100644 --- a/seleniumbase/undetected/cdp_driver/cdp_util.py +++ b/seleniumbase/undetected/cdp_driver/cdp_util.py @@ -325,6 +325,11 @@ async def start( ad_block_dir = os.path.join(DOWNLOADS_FOLDER, "ad_block") __unzip_to_new_folder(ad_block_zip, ad_block_dir) extension_dir = __add_chrome_ext_dir(extension_dir, ad_block_dir) + if ( + "binary_location" in kwargs + and not browser_executable_path + ): + browser_executable_path = kwargs["binary_location"] if not config: config = Config( user_data_dir, diff --git a/seleniumbase/undetected/cdp_driver/tab.py b/seleniumbase/undetected/cdp_driver/tab.py index f46d2583470..fc6988e25ec 100644 --- a/seleniumbase/undetected/cdp_driver/tab.py +++ b/seleniumbase/undetected/cdp_driver/tab.py @@ -6,6 +6,7 @@ import pathlib import urllib.parse import warnings +from seleniumbase import config as sb_config from typing import Dict, List, Union, Optional, Tuple from . import browser as cdp_browser from . import element @@ -137,6 +138,14 @@ def __init__( self._dom = None self._window_id = None + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.aclose() + if exc_type and exc_val: + raise exc_type(exc_val) + @property def inspector_url(self): """ @@ -348,7 +357,12 @@ async def get( if new_window and not new_tab: new_tab = True if new_tab: - return await self.browser.get(url, new_tab, new_window) + if hasattr(sb_config, "incognito") and sb_config.incognito: + return await self.browser.get( + url, new_tab=False, new_window=True + ) + else: + return await self.browser.get(url, new_tab, new_window) else: frame_id, loader_id, *_ = await self.send(cdp.page.navigate(url)) await self From 03d7dc0c1b1a00675e586c42ae73a09890aa1716 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Mon, 28 Apr 2025 13:11:25 -0400 Subject: [PATCH 2/3] Refresh Python dependencies --- mkdocs_build/requirements.txt | 2 +- requirements.txt | 7 ++++--- setup.py | 7 ++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt index de38a770118..6fd0f591cc1 100644 --- a/mkdocs_build/requirements.txt +++ b/mkdocs_build/requirements.txt @@ -2,7 +2,7 @@ # Minimum Python version: 3.9 (for generating docs only) regex>=2024.11.6 -pymdown-extensions>=10.14.3 +pymdown-extensions>=10.15 pipdeptree>=2.26.1 python-dateutil>=2.8.2 Markdown==3.8 diff --git a/requirements.txt b/requirements.txt index c8a3a9e12aa..e9c8f342e2f 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,11 @@ -pip>=25.0.1 +pip>=25.0.1;python_version<"3.9" +pip>=25.1;python_version>="3.9" packaging>=25.0 setuptools~=70.2;python_version<"3.10" -setuptools>=79.0.1;python_version>="3.10" +setuptools>=80.0.0;python_version>="3.10" wheel>=0.45.1 attrs>=25.3.0 -certifi>=2025.1.31 +certifi>=2025.4.26 exceptiongroup>=1.2.2 websockets~=13.1;python_version<"3.9" websockets>=15.0.1;python_version>="3.9" diff --git a/setup.py b/setup.py index 754f45b6b08..d1f12df9ddb 100755 --- a/setup.py +++ b/setup.py @@ -147,13 +147,14 @@ ], python_requires=">=3.8", install_requires=[ - 'pip>=25.0.1', + 'pip>=25.0.1;python_version<"3.9"', + 'pip>=25.1;python_version>="3.9"', 'packaging>=25.0', 'setuptools~=70.2;python_version<"3.10"', # Newer ones had issues - 'setuptools>=79.0.1;python_version>="3.10"', + 'setuptools>=80.0.0;python_version>="3.10"', 'wheel>=0.45.1', 'attrs>=25.3.0', - "certifi>=2025.1.31", + "certifi>=2025.4.26", "exceptiongroup>=1.2.2", 'websockets~=13.1;python_version<"3.9"', 'websockets>=15.0.1;python_version>="3.9"', From fb4d4725fdb9acf7b2ea16f212bc6ed8c913c334 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Mon, 28 Apr 2025 13:11:35 -0400 Subject: [PATCH 3/3] Version 4.37.8 --- seleniumbase/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index eeed4ff3f51..419cb6fa478 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.37.7" +__version__ = "4.37.8"