From c500466ecb885ad02d3cad5037d976bc9c9c1b68 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Wed, 23 Apr 2025 13:08:02 -0400 Subject: [PATCH 1/3] Improve CDP multithreading --- seleniumbase/undetected/__init__.py | 15 ++++++++++++--- seleniumbase/undetected/cdp_driver/connection.py | 9 ++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/seleniumbase/undetected/__init__.py b/seleniumbase/undetected/__init__.py index 365ffc5ba32..963790b4e33 100644 --- a/seleniumbase/undetected/__init__.py +++ b/seleniumbase/undetected/__init__.py @@ -402,7 +402,8 @@ def get(self, url): def add_cdp_listener(self, event_name, callback): if ( - self.reactor + hasattr(self, "reactor") + and self.reactor and self.reactor is not None and isinstance(self.reactor, Reactor) ): @@ -411,7 +412,11 @@ def add_cdp_listener(self, event_name, callback): return False def clear_cdp_listeners(self): - if self.reactor and isinstance(self.reactor, Reactor): + if ( + hasattr(self, "reactor") + and self.reactor + and isinstance(self.reactor, Reactor) + ): self.reactor.handlers.clear() def window_new(self, url=None): @@ -581,7 +586,11 @@ def quit(self): finally: with suppress(Exception): self.service._terminate_process() - if self.reactor and hasattr(self.reactor, "event"): + if ( + hasattr(self, "reactor") + and self.reactor + and hasattr(self.reactor, "event") + ): logger.debug("Shutting down Reactor") with suppress(Exception): self.reactor.event.set() diff --git a/seleniumbase/undetected/cdp_driver/connection.py b/seleniumbase/undetected/cdp_driver/connection.py index 0629938394b..6d3fa24c839 100644 --- a/seleniumbase/undetected/cdp_driver/connection.py +++ b/seleniumbase/undetected/cdp_driver/connection.py @@ -292,9 +292,6 @@ async def aclose(self): Closes the websocket connection. Shouldn't be called manually by users. """ if self.websocket and self.websocket.state is not State.CLOSED: - if self.listener and self.listener.running: - self.listener.cancel() - self.enabled_domains.clear() try: await self.websocket.close() except Exception: @@ -302,6 +299,9 @@ async def aclose(self): "\n❌ Error closing websocket connection to %s", self.websocket_url ) + if self.listener and self.listener.running: + self.listener.cancel() + self.enabled_domains.clear() logger.debug( "\n❌ Closed websocket connection to %s", self.websocket_url ) @@ -549,8 +549,7 @@ async def listener_loop(self): except asyncio.TimeoutError: self.idle.set() # Pause for a moment. - # await asyncio.sleep(self.time_before_considered_idle / 10) - await asyncio.sleep(0.015) + await asyncio.sleep(self.time_before_considered_idle / 10) continue except (Exception,) as e: logger.debug( From ee8b82df82b3013417db91e3ba052cd0102ad090 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Wed, 23 Apr 2025 13:08:26 -0400 Subject: [PATCH 2/3] Update examples --- examples/cdp_mode/ReadMe.md | 3 ++- examples/cdp_mode/raw_pixelscan.py | 16 ++++++++++++---- examples/cdp_mode/raw_theaters.py | 1 + examples/cdp_mode/raw_walmart.py | 3 ++- examples/presenter/uc_presentation_4.py | 3 ++- examples/raw_cdp_logging.py | 2 +- examples/raw_pixelscan.py | 14 ++++++++++---- 7 files changed, 30 insertions(+), 12 deletions(-) diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index 54f8e2f0f40..af776c0679c 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -286,6 +286,7 @@ with SB(uc=True, test=True, ad_block=True) as sb: sb.activate_cdp_mode(url) sb.sleep(2.5) sb.cdp.click_if_visible('[data-automation-id*="close-mark"]') + sb.sleep(0.3) sb.cdp.mouse_click('input[aria-label="Search"]') sb.sleep(1.2) search = "Settlers of Catan Board Game" @@ -300,7 +301,7 @@ with SB(uc=True, test=True, ad_block=True) as sb: for item in items: if required_text in item.text: description = item.querySelector( - '[data-automation-id="product-price"] + span' + '[data-automation-id="product-title"]' ) if description and description.text not in unique_item_text: unique_item_text.append(description.text) diff --git a/examples/cdp_mode/raw_pixelscan.py b/examples/cdp_mode/raw_pixelscan.py index c7802b664f4..c0fefb50cb4 100644 --- a/examples/cdp_mode/raw_pixelscan.py +++ b/examples/cdp_mode/raw_pixelscan.py @@ -2,12 +2,20 @@ with SB(uc=True, incognito=True, test=True) as sb: sb.activate_cdp_mode("https://pixelscan.net/") - sb.sleep(3) - sb.remove_elements(".bg-bannerBg") # Remove the top banner - sb.remove_elements("pxlscn-ad1") # Remove the ad banner + sb.sleep(2) + sb.click('button[class*="startButton"]') + sb.sleep(6) + sb.remove_elements(".bg-bannerBg") # Remove top banner + sb.remove_elements("pxlscn-ad1") # Remove an ad banner + sb.remove_elements("pxlscn-ad2") # Remove an ad banner sb.remove_elements("jdiv") # Remove chat widgets + sb.sleep(14) not_masking_text = "You are not masking your fingerprint" - sb.assert_text(not_masking_text, "pxlscn-fingerprint-masking") + sb.assert_text( + not_masking_text, + "pxlscn-fingerprint-masking", + timeout=20, + ) no_automation_detected = "No automation framework detected" sb.assert_text(no_automation_detected, "pxlscn-bot-detection") consistent_selector = 'div.bg-consistentBg [alt="Good"]' diff --git a/examples/cdp_mode/raw_theaters.py b/examples/cdp_mode/raw_theaters.py index 2481ed3f5b1..bb5d51c7c6d 100644 --- a/examples/cdp_mode/raw_theaters.py +++ b/examples/cdp_mode/raw_theaters.py @@ -5,6 +5,7 @@ url = "https://architectureofcities.com/roman-theaters" sb.activate_cdp_mode(url) sb.cdp.click_if_visible("#cn-close-notice") + sb.cdp.click_if_visible('span:contains("Continue")') sb.sleep(1) print("*** " + sb.cdp.get_text("h1") + " ***") for item in sb.cdp.find_elements("h3"): diff --git a/examples/cdp_mode/raw_walmart.py b/examples/cdp_mode/raw_walmart.py index 3123fd04db7..a343f1d9677 100644 --- a/examples/cdp_mode/raw_walmart.py +++ b/examples/cdp_mode/raw_walmart.py @@ -5,6 +5,7 @@ sb.activate_cdp_mode(url) sb.sleep(2.5) sb.cdp.click_if_visible('[data-automation-id*="close-mark"]') + sb.sleep(0.3) sb.cdp.mouse_click('input[aria-label="Search"]') sb.sleep(1.2) search = "Settlers of Catan Board Game" @@ -19,7 +20,7 @@ for item in items: if required_text in item.text: description = item.querySelector( - '[data-automation-id="product-price"] + span' + '[data-automation-id="product-title"]' ) if description and description.text not in unique_item_text: unique_item_text.append(description.text) diff --git a/examples/presenter/uc_presentation_4.py b/examples/presenter/uc_presentation_4.py index 1195c9bcb31..75efcef771d 100644 --- a/examples/presenter/uc_presentation_4.py +++ b/examples/presenter/uc_presentation_4.py @@ -521,6 +521,7 @@ def test_presentation_4(self): sb.activate_cdp_mode(url) sb.sleep(2.5) sb.cdp.click_if_visible('[data-automation-id*="close-mark"]') + sb.sleep(0.3) sb.cdp.mouse_click('input[aria-label="Search"]') sb.sleep(1.2) search = "Settlers of Catan Board Game" @@ -535,7 +536,7 @@ def test_presentation_4(self): for item in items: if required_text in item.text: description = item.querySelector( - '[data-automation-id="product-price"] + span' + '[data-automation-id="product-title"]' ) if ( description diff --git a/examples/raw_cdp_logging.py b/examples/raw_cdp_logging.py index 23ae0b177a2..51885103cc8 100644 --- a/examples/raw_cdp_logging.py +++ b/examples/raw_cdp_logging.py @@ -6,7 +6,7 @@ url = "seleniumbase.io/apps/turnstile" driver.uc_open_with_reconnect(url, 2) driver.uc_gui_handle_captcha() - driver.sleep(3) + driver.sleep(2) pprint(driver.get_log("performance")) finally: driver.quit() diff --git a/examples/raw_pixelscan.py b/examples/raw_pixelscan.py index b6be9bb0390..df8883a2d45 100644 --- a/examples/raw_pixelscan.py +++ b/examples/raw_pixelscan.py @@ -1,12 +1,18 @@ from seleniumbase import SB with SB(uc=True, incognito=True, test=True) as sb: - sb.driver.uc_open_with_reconnect("https://pixelscan.net/", 7) - sb.remove_elements(".bg-bannerBg") # Remove the top banner - sb.remove_elements("pxlscn-ad1") # Remove the ad banner + sb.driver.uc_open_with_reconnect("https://pixelscan.net/", 2) + sb.uc_click('button[class*="startButton"]', reconnect_time=20) + sb.remove_elements(".bg-bannerBg") # Remove top banner + sb.remove_elements("pxlscn-ad1") # Remove an ad banner + sb.remove_elements("pxlscn-ad2") # Remove an ad banner sb.remove_elements("jdiv") # Remove chat widgets no_automation_detected = "No automation framework detected" - sb.assert_text(no_automation_detected, "pxlscn-bot-detection") + sb.assert_text( + no_automation_detected, + "pxlscn-bot-detection", + timeout=20, + ) not_masking_text = "You are not masking your fingerprint" sb.assert_text(not_masking_text, "pxlscn-fingerprint-masking") consistent_selector = 'div.bg-consistentBg [alt="Good"]' From 2d5c8103767090201f0740d94942808a987b1261 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Wed, 23 Apr 2025 13:08:40 -0400 Subject: [PATCH 3/3] Version 4.37.6 --- seleniumbase/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index d9897558f82..254f4065e3e 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.37.5" +__version__ = "4.37.6"