Merge pull request #32 from ClericPy/dev

2.3.3

Author: ClericPy

ichrome/__init__.py

@@ -6,7 +6,7 @@
 from .logs import logger
 from .sync_utils import Chrome, Tab
 
-__version__ = "2.3.2"
+__version__ = "2.3.3"
 __tips__ = "[github]: https://github.com/ClericPy/ichrome\n[cdp]: https://chromedevtools.github.io/devtools-protocol/\n[cmd args]: https://peter.sh/experiments/chromium-command-line-switches/"
 __all__ = [
     'Chrome', 'ChromeDaemon', 'Tab', 'Tag', 'AsyncChrome', 'AsyncTab', 'logger',

ichrome/__main__.py

@@ -46,8 +46,7 @@ def main():
         "-cp",
         "--chrome-path",
         "--chrome_path",
-        help=
-        "chrome executable file path, default to null for automatic searching",
+        help="chrome executable file path, default to null(automatic searching)",
         default="")
     parser.add_argument("-H",
                         "--host",
@@ -211,17 +210,26 @@ def main():
     log_level = getattr(args, 'log_level', None)
     if log_level:
         logger.setLevel(log_level)
-    if args.start_url == 'about:blank' or not args.start_url:
+    if kwargs['start_url'] == 'about:blank' or not kwargs['start_url']:
+        # reset start_url from extra_config
         for config in kwargs['extra_config']:
             if re.match('^https?://', config):
                 kwargs['start_url'] = config
                 kwargs['extra_config'].remove(config)
                 break
+
     if '--dump-dom' in extra_config or args.crawl:
         logger.setLevel(60)
         from .debugger import crawl_once
+        if kwargs['start_url'] == 'about:blank' or not kwargs['start_url']:
+            kwargs['start_url'] = kwargs['extra_config'].pop(0)
+        if kwargs['start_url'] != 'about:blank' and not kwargs[
+                'start_url'].startswith('http'):
+            kwargs['start_url'] = 'http://' + kwargs['start_url']
         kwargs['headless'] = getattr(args, 'headless', True)
-        asyncio.run(crawl_once(**kwargs))
+        kwargs['disable_image'] = True
+        kwargs['timeout'] = max([5, args.timeout])
+        print(asyncio.run(crawl_once(**kwargs)), flush=True)
     elif args.clear_cache:
         from .debugger import clear_cache_handler
         kwargs['headless'] = getattr(args, 'headless', True)

ichrome/async_utils.py

@@ -993,6 +993,23 @@ async def reset_history(self, timeout=NotSet) -> bool:
         result = await self.send('Page.resetNavigationHistory', timeout=timeout)
         return self.check_error('reset_history', result)
 
+    async def setBlockedURLs(self, urls: List[str], timeout=NotSet):
+        """(Network.setBlockedURLs) Blocks URLs from loading. [EXPERIMENTAL].
+        :param urls: URL patterns to block. Wildcards ('*') are allowed.
+        :type urls: List[str]
+
+        Demo::
+
+            await tab.setBlockedURLs(urls=['*.jpg', '*.png'])
+
+        WARNING: This method is EXPERIMENTAL, the official suggestion is using Fetch.enable, even Fetch is also EXPERIMENTAL, and wait events to control the requests (continue / abort / modify), especially block urls with resourceType: Document, Stylesheet, Image, Media, Font, Script, TextTrack, XHR, Fetch, EventSource, WebSocket, Manifest, SignedExchange, Ping, CSPViolationReport, Other.
+        https://chromedevtools.github.io/devtools-protocol/tot/Fetch/#method-enable
+
+
+        """
+        await self.enable('Network')
+        return await self.send('Network.setBlockedURLs', urls=urls, timeout=timeout)
+
     async def set_url(self,
                       url: Optional[str] = None,
                       referrer: Optional[str] = None,

ichrome/daemon.py

@@ -30,12 +30,12 @@ class ChromeDaemon(object):
         host="127.0.0.1",     --remote-debugging-address, default to 127.0.0.1
         port,                 --remote-debugging-port, default to 9222
         headless,             --headless and --hide-scrollbars, default to False
-        user_agent,           --user-agent, default to 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36'
+        user_agent,           --user-agent, default to None (with the original UA)
         proxy,                --proxy-server, default to None
-        user_data_dir,        user_data_dir to save the user data, default to ~/ichrome_user_data. These string will ignore user_data_dir arg: {'null', 'None', '/dev/null', "''", '""'}
+        user_data_dir,        user_data_dir to save the user data, default to ~/ichrome_user_data. These strings will ignore user_data_dir arg: {'null', 'None', '/dev/null', "''", '""'}
         disable_image,        disable image for loading performance, default to False
-        start_url,            start url while launching chrome, default to about:blank
-        max_deaths,           max deaths in 5 secs, auto restart `max_deaths` times if crash fast in 5 secs. default to 1 for without auto-restart
+        start_url,            start url while launching chrome, default to None
+        max_deaths,           max deaths in 5 secs, auto restart `max_deaths` times if crash fast in 5 secs. default to 1 (without auto-restart)
         timeout,              timeout to connect the remote server, default to 1 for localhost
         debug,                set logger level to DEBUG
         proc_check_interval,  check chrome process alive every interval seconds
@@ -67,7 +67,7 @@ class ChromeDaemon(object):
     """
 
     port_in_using: set = set()
-    PC_UA = "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Mobile Safari/537.36"
+    PC_UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36"
     MAC_OS_UA = (
         "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_0_1) Version/8.0.1a Safari/728.28.19"
     )
@@ -86,7 +86,7 @@ def __init__(
         proxy=None,
         user_data_dir=None,
         disable_image=False,
-        start_url="about:blank",
+        start_url="",
         extra_config=None,
         max_deaths=1,
         daemon=True,
@@ -109,10 +109,13 @@ def __init__(
         self.ready = False
         self.proc = None
         self.host = host
-        self.port = port
+        if port is None:
+            self.port = self.get_free_port(host=self.host)
+        else:
+            self.port = port
         self.server = f"http://{self.host}:{self.port}"
         self.chrome_path = chrome_path
-        self.UA = self.PC_UA if user_agent is None else user_agent
+        self.UA = user_agent
         self.headless = headless
         self.proxy = proxy
         self.disable_image = disable_image
@@ -265,7 +268,7 @@ def proc_ok(self):
     def connection_ok(self):
         url = self.server + "/json"
         for _ in range(2):
-            r = self.req.get(url, timeout=self._timeout)
+            r = self.req.head(url, timeout=self._timeout)
             if r.x and r.ok:
                 self.ready = True
                 self.port_in_using.add(self.port)
@@ -531,7 +534,7 @@ def __init__(
         proxy=None,
         user_data_dir=None,
         disable_image=False,
-        start_url="about:blank",
+        start_url="",
         extra_config=None,
         max_deaths=1,
         daemon=True,
@@ -604,7 +607,7 @@ async def launch_chrome(self):
     async def check_connection(self):
         url = self.server + "/json"
         for _ in range(int(self._timeout) + 1):
-            r = await self.req.get(url, timeout=self._timeout)
+            r = await self.req.head(url, timeout=self._timeout)
             if r and r.ok:
                 self.ready = True
                 self.port_in_using.add(self.port)
@@ -719,7 +722,7 @@ async def __aenter__(self):
     async def create_chrome_workers(self):
         for port in range(self.start_port, self.start_port + self.workers):
             logger.debug("ChromeDaemon cmd args: port=%s, %s" %
-                        (port, self.kwargs))
+                         (port, self.kwargs))
             self.daemons.append(await
                                 AsyncChromeDaemon(port=port,
                                                   daemon=True,

ichrome/debugger.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import asyncio
 import atexit
+import os
 from functools import wraps
 from inspect import isawaitable
 from typing import Set
@@ -68,8 +69,6 @@ def __getattr__(self, name):
 def quit_while_daemon_missing(daemon):
     # quit the whole program while missing daemon process for daemon debugger
     if not daemon.get_proc(daemon.port):
-        import os
-        print(f'{daemon} missing process, exit.')
         os._exit(1)
 
 
@@ -210,17 +209,27 @@ def launch(*args, **kwargs):
     return Daemon(*args, **kwargs)
 
 
-def get_a_tab(host='127.0.0.1', port=None) -> AsyncTab:
+def connect_a_chrome(host='127.0.0.1', port=None, **daemon_kwargs) -> Chrome:
     if not port:
         for port in ChromeDaemon.port_in_using:
-            return Chrome(port=port).get_tab()
-        port = 9222
+            return Chrome(host=host, port=port)
+        port = ChromeDaemon.get_free_port(host=host)
     try:
-        return Chrome(host=host, port=port).get_tab()
+        return Chrome(host=host, port=port)
     except RuntimeError:
         # no existing port, launch a new chrome, and auto quit if chrome process missed.
-        launch()
-        return Chrome(host=host, port=port).get_tab()
+        d = launch(host=host, port=port, **daemon_kwargs)
+        return Chrome(host=host, port=d.port)
+
+
+def get_a_tab(host='127.0.0.1', port=None, **daemon_kwargs) -> AsyncTab:
+    chrome = connect_a_chrome(host=host, port=port, **daemon_kwargs)
+    return chrome.get_tab()
+
+
+def get_a_new_tab(host='127.0.0.1', port=None, **daemon_kwargs) -> AsyncTab:
+    chrome = connect_a_chrome(host=host, port=port, **daemon_kwargs)
+    return chrome.new_tab()
 
 
 def show_all_log():
@@ -235,14 +244,13 @@ def mute_all_log():
 
 def stop_all_daemons():
     if Daemon.daemons:
-        logger.info(f'auto shutdown {Daemon.daemons}')
+        logger.debug(f'auto shutdown {Daemon.daemons}')
         for daemon in Daemon.daemons:
             daemon.stop()
 
 
 def shutdown():
     stop_all_daemons()
-    import os
     os._exit(0)
 
 
@@ -257,7 +265,7 @@ def _filter_function(r):
                          indent=2)
         req_type = get_data_value(r, 'params.type')
         doc_url = get_data_value(r, 'params.documentURL')
-        print(f'{doc_url} - {req_type}\n{req}', end=f'\n{"="*40}\n')
+        print(f'{doc_url} - {req_type}\n{req}', end=f'\n{"="*40}\n', flush=True)
 
     # listen network flow in 60 s
     timeout = timeout
@@ -276,13 +284,13 @@ async def crawl_once(**kwargs):
     async with AsyncChromeDaemon(**kwargs) as cd:
         async with AsyncChrome(
                 host=kwargs.get('host', '127.0.0.1'),
-                port=kwargs.get('port', 9222),
+                port=cd.port,
                 timeout=cd._timeout or 2,
         ) as chrome:
             async with chrome.connect_tab(0, auto_close=True) as tab:
                 await tab.set_url(url, timeout=cd._timeout)
                 html = await tab.get_html(timeout=cd._timeout)
-                print(html)
+                return html
 
 
 async def clear_cache_handler(**kwargs):