diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 0b0732b..cd5855e 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b15' +__version__ = '4.0.0b16' diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 7f882c0..5d2ed0c 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -7,7 +7,7 @@ from time import sleep, perf_counter from .chromium_driver import BrowserDriver, ChromiumDriver from .._commons.tools import stop_process_on_port, raise_error -from .._units.download_manager import DownloadManager +from .._units.downloader import DownloadManager __ERROR__ = 'error' diff --git a/DrissionPage/_base/browser.pyi b/DrissionPage/_base/browser.pyi index 963e3f7..3bf21be 100644 --- a/DrissionPage/_base/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -7,7 +7,7 @@ from typing import List, Optional, Union from .chromium_driver import BrowserDriver, ChromiumDriver from .._pages.chromium_page import ChromiumPage -from .._units.download_manager import DownloadManager +from .._units.downloader import DownloadManager class Browser(object): diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index f1963f5..f8f4b4f 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -9,7 +9,7 @@ from threading import Thread, Event from time import perf_counter from requests import get -from websocket import (WebSocketTimeoutException, WebSocketException, WebSocketConnectionClosedException, \ +from websocket import (WebSocketTimeoutException, WebSocketException, WebSocketConnectionClosedException, create_connection) @@ -70,7 +70,7 @@ class ChromiumDriver(object): self.method_results[ws_id] = Queue() try: self._ws.send(message_json) - except OSError: + except (OSError, WebSocketConnectionClosedException): self.method_results.pop(ws_id, None) return None @@ -190,14 +190,14 @@ class ChromiumDriver(object): event = self.event_queue.get_nowait() function = self.event_handlers.get(event['method']) if function: - if self._debug: - print(f'开始执行 {function.__name__}') + # if self._debug: + # print(f'开始执行 {function.__name__}') try: function(**event['params']) except: pass - if self._debug: - print(f'执行 {function.__name__}完毕') + # if self._debug: + # print(f'执行 {function.__name__}完毕') self.event_handlers.clear() self.method_results.clear() diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_commons/browser.py index e5431b2..4e8237c 100644 --- a/DrissionPage/_commons/browser.py +++ b/DrissionPage/_commons/browser.py @@ -24,10 +24,10 @@ def connect_browser(option): :param option: ChromiumOptions对象 :return: 返回是否接管的浏览器 """ - debugger_address = option.debugger_address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') + address = option.address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') chrome_path = option.browser_path - ip, port = debugger_address.split(':') + ip, port = address.split(':') if ip != '127.0.0.1' or port_is_using(ip, port) or option.is_existing_only: test_connect(ip, port) option._headless = False @@ -86,7 +86,7 @@ def get_launch_args(opt): result.add(i) if not has_user_path and not opt.system_user_path: - port = opt.debugger_address.split(':')[-1] if opt.debugger_address else '0' + port = opt.address.split(':')[-1] if opt.address else '0' path = Path(gettempdir()) / 'DrissionPage' / f'userData_{port}' path.mkdir(parents=True, exist_ok=True) opt.set_user_data_path(path) diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index fdd2edf..b8e2eec 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -4,6 +4,7 @@ @Contact : g1879@qq.com """ from pathlib import Path +from re import search from shutil import rmtree from tempfile import gettempdir, TemporaryDirectory from threading import Lock @@ -36,12 +37,13 @@ class ChromiumOptions(object): self._extensions = options.get('extensions', []) self._prefs = options.get('prefs', {}) self._flags = options.get('flags', {}) - self._debugger_address = options.get('debugger_address', None) + self._address = options.get('address', None) self._load_mode = options.get('load_mode', 'normal') - self._proxy = om.proxies.get('http', None) self._system_user_path = options.get('system_user_path', False) self._existing_only = options.get('existing_only', False) + self._proxy = om.proxies.get('http', None) or om.proxies.get('https', None) + user_path = user = False for arg in self._arguments: if arg.startswith('--user-data-dir='): @@ -54,14 +56,14 @@ class ChromiumOptions(object): break timeouts = om.timeouts - self._timeouts = {'implicit': timeouts['implicit'], + self._timeouts = {'base': timeouts['base'], 'pageLoad': timeouts['page_load'], 'script': timeouts['script']} self._auto_port = options.get('auto_port', False) if self._auto_port: port, path = PortFinder().get_port() - self._debugger_address = f'127.0.0.1:{port}' + self._address = f'127.0.0.1:{port}' self.set_argument('--user-data-dir', path) others = om.others @@ -77,8 +79,8 @@ class ChromiumOptions(object): self._extensions = [] self._prefs = {} self._flags = {} - self._timeouts = {'implicit': 10, 'pageLoad': 30, 'script': 30} - self._debugger_address = '127.0.0.1:9222' + self._timeouts = {'base': 10, 'pageLoad': 30, 'script': 30} + self._address = '127.0.0.1:9222' self._load_mode = 'normal' self._proxy = None self._auto_port = False @@ -125,12 +127,17 @@ class ChromiumOptions(object): @property def debugger_address(self): """返回浏览器地址,ip:port""" - return self._debugger_address + return self._address @debugger_address.setter def debugger_address(self, address): """设置浏览器地址,格式ip:port""" - self.set_debugger_address(address) + self.set_address(address) + + @property + def address(self): + """返回浏览器地址,ip:port""" + return self._address @property def arguments(self): @@ -275,15 +282,16 @@ class ChromiumOptions(object): self.clear_file_flags = True return self - def set_timeouts(self, implicit=None, pageLoad=None, script=None): + def set_timeouts(self, base=None, pageLoad=None, script=None, implicit=None): """设置超时时间,单位为秒 - :param implicit: 默认超时时间 + :param base: 默认超时时间 :param pageLoad: 页面加载超时时间 :param script: 脚本运行超时时间 :return: 当前对象 """ - if implicit is not None: - self._timeouts['implicit'] = implicit + base = base if base is not None else implicit + if base is not None: + self._timeouts['base'] = base if pageLoad is not None: self._timeouts['pageLoad'] = pageLoad if script is not None: @@ -352,6 +360,10 @@ class ChromiumOptions(object): :param proxy: 代理url和端口 :return: 当前对象 """ + if search(r'.*?:.*?@.*?\..*', proxy): + print('你似乎在设置使用账号密码的代理,暂时不支持这种代理,可自行用插件实现需求。') + if not proxy.lower().startswith('socks'): + print('你似乎在设置使用socks代理,暂时不支持这种代理,可自行用插件实现需求。') self._proxy = proxy return self.set_argument('--proxy-server', proxy) @@ -368,25 +380,26 @@ class ChromiumOptions(object): self._load_mode = value.lower() return self - def set_paths(self, browser_path=None, local_port=None, debugger_address=None, download_path=None, - user_data_path=None, cache_path=None): + def set_paths(self, browser_path=None, local_port=None, address=None, download_path=None, + user_data_path=None, cache_path=None, debugger_address=None): """快捷的路径设置函数 :param browser_path: 浏览器可执行文件路径 :param local_port: 本地端口号 - :param debugger_address: 调试浏览器地址,例:127.0.0.1:9222 + :param address: 调试浏览器地址,例:127.0.0.1:9222 :param download_path: 下载文件路径 :param user_data_path: 用户数据路径 :param cache_path: 缓存路径 :return: 当前对象 """ + address = address or debugger_address if browser_path is not None: self.set_browser_path(browser_path) if local_port is not None: self.set_local_port(local_port) - if debugger_address is not None: - self.set_debugger_address(debugger_address) + if address is not None: + self.set_address(address) if download_path is not None: self.set_download_path(download_path) @@ -404,17 +417,17 @@ class ChromiumOptions(object): :param port: 端口号 :return: 当前对象 """ - self._debugger_address = f'127.0.0.1:{port}' + self._address = f'127.0.0.1:{port}' self._auto_port = False return self - def set_debugger_address(self, address): + def set_address(self, address): """设置浏览器地址,格式'ip:port' :param address: 浏览器地址 :return: 当前对象 """ address = address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') - self._debugger_address = address + self._address = address return self def set_browser_path(self, path): @@ -507,7 +520,7 @@ class ChromiumOptions(object): om = OptionsManager(self.ini_path or str(Path(__file__).parent / 'configs.ini')) # 设置chromium_options - attrs = ('debugger_address', 'browser_path', 'arguments', 'extensions', 'user', 'load_mode', + attrs = ('address', 'browser_path', 'arguments', 'extensions', 'user', 'load_mode', 'auto_port', 'system_user_path', 'existing_only', 'flags') for i in attrs: om.set_item('chromium_options', i, self.__getattribute__(f'_{i}')) @@ -517,7 +530,7 @@ class ChromiumOptions(object): # 设置路径 om.set_item('paths', 'download_path', self._download_path or '') # 设置timeout - om.set_item('timeouts', 'implicit', self._timeouts['implicit']) + om.set_item('timeouts', 'base', self._timeouts['base']) om.set_item('timeouts', 'page_load', self._timeouts['pageLoad']) om.set_item('timeouts', 'script', self._timeouts['script']) # 设置重试 diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index 9a1b0d4..ec843ca 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -19,7 +19,7 @@ class ChromiumOptions(object): self._load_mode: str = ... self._timeouts: dict = ... self._proxy: str = ... - self._debugger_address: str = ... + self._address: str = ... self._extensions: list = ... self._prefs: dict = ... self._flags: dict = ... @@ -54,7 +54,7 @@ class ChromiumOptions(object): def proxy(self) -> str: ... @property - def debugger_address(self) -> str: ... + def address(self) -> str: ... @property def arguments(self) -> list: ... @@ -100,7 +100,7 @@ class ChromiumOptions(object): def clear_flags_in_file(self) -> ChromiumOptions: ... - def set_timeouts(self, implicit: float = None, pageLoad: float = None, + def set_timeouts(self, base: float = None, pageLoad: float = None, script: float = None) -> ChromiumOptions: ... def set_user(self, user: str = 'Default') -> ChromiumOptions: ... @@ -125,7 +125,7 @@ class ChromiumOptions(object): def set_local_port(self, port: Union[str, int]) -> ChromiumOptions: ... - def set_debugger_address(self, address: str) -> ChromiumOptions: ... + def set_address(self, address: str) -> ChromiumOptions: ... def set_download_path(self, path: Union[str, Path]) -> ChromiumOptions: ... @@ -134,8 +134,8 @@ class ChromiumOptions(object): def set_cache_path(self, path: Union[str, Path]) -> ChromiumOptions: ... def set_paths(self, browser_path: Union[str, Path] = None, local_port: Union[int, str] = None, - debugger_address: str = None, download_path: Union[str, Path] = None, - user_data_path: Union[str, Path] = None, cache_path: Union[str, Path] = None) -> ChromiumOptions: ... + address: str = None, download_path: Union[str, Path] = None, user_data_path: Union[str, Path] = None, + cache_path: Union[str, Path] = None) -> ChromiumOptions: ... def use_system_user_path(self, on_off: bool = True) -> ChromiumOptions: ... diff --git a/DrissionPage/_configs/configs.ini b/DrissionPage/_configs/configs.ini index 3f0645f..7a22f7a 100644 --- a/DrissionPage/_configs/configs.ini +++ b/DrissionPage/_configs/configs.ini @@ -2,7 +2,7 @@ download_path = [chromium_options] -debugger_address = 127.0.0.1:9222 +address = 127.0.0.1:9222 browser_path = chrome arguments = ['--no-first-run', '--disable-infobars', '--disable-popup-blocking'] extensions = [] @@ -18,12 +18,12 @@ existing_only = False headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8', 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'connection': 'keep-alive', 'accept-charset': 'GB2312,utf-8;q=0.7,*;q=0.7'} [timeouts] -implicit = 10 +base = 10 page_load = 30 script = 30 [proxies] -http = +http = https = [others] diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index 728c6a1..88b396f 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -76,7 +76,7 @@ class SessionOptions(object): self._max_redirects = options['max_redirects'] self.set_proxies(om.proxies.get('http', None), om.proxies.get('https', None)) - self._timeout = om.timeouts.get('implicit', 10) + self._timeout = om.timeouts.get('base', 10) self._download_path = om.paths.get('download_path', None) or None others = om.others @@ -379,7 +379,7 @@ class SessionOptions(object): om.set_item('session_options', i, options[i]) om.set_item('paths', 'download_path', self.download_path or '') - om.set_item('timeouts', 'implicit', self.timeout) + om.set_item('timeouts', 'base', self.timeout) om.set_item('proxies', 'http', self.proxies.get('http', None)) om.set_item('proxies', 'https', self.proxies.get('https', None)) om.set_item('others', 'retry_times', self.retry_times) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 3c7dab8..25d9952 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -19,7 +19,7 @@ from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_f from .._units.clicker import Clicker from .._units.rect import ElementRect from .._units.scroller import ElementScroller -from .._units.select_element import SelectElement +from .._units.selector import SelectElement from .._units.setter import ChromiumElementSetter from .._units.states import ElementStates, ShadowRootStates from .._units.waiter import ElementWaiter diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 9bb0cb4..7725ccd 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -16,7 +16,7 @@ from .._pages.web_page import WebPage from .._units.clicker import Clicker from .._units.rect import ElementRect from .._units.scroller import ElementScroller -from .._units.select_element import SelectElement +from .._units.selector import SelectElement from .._units.setter import ChromiumElementSetter from .._units.states import ShadowRootStates, ElementStates from .._units.waiter import ElementWaiter diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index f338d5b..832c9de 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -17,8 +17,8 @@ from .._commons.web import location_in_viewport from .._elements.chromium_element import ChromiumElement, run_js, make_chromium_ele from .._elements.none_element import NoneElement from .._elements.session_element import make_session_ele -from .._units.action_chains import ActionChains -from .._units.network_listener import NetworkListener +from .._units.actions import Actions +from .._units.listener import Listener from .._units.rect import TabRect from .._units.screencast import Screencast from .._units.scroller import PageScroller @@ -46,7 +46,6 @@ class ChromiumBase(BasePage): self._set = None self._screencast = None self._actions = None - self._listener = None self._states = None self._has_alert = False self._ready_state = None @@ -57,6 +56,8 @@ class ChromiumBase(BasePage): self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc self._download_path = None self._load_end_time = 0 + if not hasattr(self, '_listener'): + self._listener = None if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): address = f'127.0.0.1:{address}' @@ -313,7 +314,7 @@ class ChromiumBase(BasePage): def actions(self): """返回用于执行动作链的对象""" if self._actions is None: - self._actions = ActionChains(self) + self._actions = Actions(self) self.wait.load_complete() return self._actions @@ -321,7 +322,7 @@ class ChromiumBase(BasePage): def listen(self): """返回用于聆听数据包的对象""" if self._listener is None: - self._listener = NetworkListener(self) + self._listener = Listener(self) return self._listener @property @@ -1054,20 +1055,21 @@ class ChromiumBase(BasePage): class Timeout(object): """用于保存d模式timeout信息的类""" - def __init__(self, page, implicit=None, page_load=None, script=None): + def __init__(self, page, base=None, page_load=None, script=None, implicit=None): """ :param page: ChromiumBase页面 - :param implicit: 默认超时时间 + :param base: 默认超时时间 :param page_load: 页面加载超时时间 :param script: js超时时间 """ self._page = page - self.implicit = 10 if implicit is None else implicit + base = base if base is not None else implicit + self.base = 10 if base is None else base self.page_load = 30 if page_load is None else page_load self.script = 30 if script is None else script def __repr__(self): - return str({'implicit': self.implicit, 'page_load': self.page_load, 'script': self.script}) + return str({'base': self.base, 'page_load': self.page_load, 'script': self.script}) class Alert(object): diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index eb5b437..3ba30f9 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -14,8 +14,8 @@ from .._elements.none_element import NoneElement from .._elements.session_element import SessionElement from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage -from .._units.action_chains import ActionChains -from .._units.network_listener import NetworkListener +from .._units.actions import Actions +from .._units.listener import Listener from .._units.rect import TabRect from .._units.screencast import Screencast from .._units.scroller import Scroller, PageScroller @@ -48,8 +48,8 @@ class ChromiumBase(BasePage): self._wait: BaseWaiter = ... self._set: ChromiumBaseSetter = ... self._screencast: Screencast = ... - self._actions: ActionChains = ... - self._listener: NetworkListener = ... + self._actions: Actions = ... + self._listener: Listener = ... self._states: PageStates = ... self._alert: Alert = ... self._has_alert: bool = ... @@ -152,10 +152,10 @@ class ChromiumBase(BasePage): def screencast(self) -> Screencast: ... @property - def actions(self) -> ActionChains: ... + def actions(self) -> Actions: ... @property - def listen(self) -> NetworkListener: ... + def listen(self) -> Listener: ... @property def states(self) -> PageStates: ... @@ -237,9 +237,9 @@ class ChromiumBase(BasePage): class Timeout(object): - def __init__(self, page: ChromiumBase, implicit=None, page_load=None, script=None): + def __init__(self, page: ChromiumBase, base=None, page_load=None, script=None): self._page: ChromiumBase = ... - self.implicit: float = ... + self.base: float = ... self.page_load: float = ... self.script: float = ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index d5c2824..5716eb6 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -9,6 +9,7 @@ from time import sleep, perf_counter from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase +from .._units.listener import FrameListener from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter @@ -122,12 +123,16 @@ class ChromiumFrame(ChromiumBase): self._is_diff_domain = False self.doc_ele = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) self._frame_id = node['frameId'] + if self._listener: + self._listener._to_target(self._target_page.tab_id, self.address, self) super().__init__(self.address, self._target_page.tab_id, self._target_page.timeout) self._debug = debug self.driver._debug = d_debug else: self._is_diff_domain = True + if self._listener: + self._listener._to_target(node['frameId'], self.address, self) super().__init__(self.address, node['frameId'], self._target_page.timeout) end_time = perf_counter() + self.timeouts.page_load while perf_counter() < end_time: @@ -251,6 +256,13 @@ class ChromiumFrame(ChromiumBase): self._rect = FrameRect(self) return self._rect + @property + def listen(self): + """返回用于聆听数据包的对象""" + if self._listener is None: + self._listener = FrameListener(self) + return self._listener + # ----------挂件---------- @property diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index f203667..b7d6892 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -11,10 +11,11 @@ from .chromium_page import ChromiumPage from .chromium_tab import ChromiumTab from .web_page import WebPage from .._elements.chromium_element import ChromiumElement -from .._units.states import FrameStates +from .._units.listener import FrameListener from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter +from .._units.states import FrameStates from .._units.waiter import FrameWaiter @@ -33,6 +34,7 @@ class ChromiumFrame(ChromiumBase): self._states: FrameStates = ... self._reloading: bool = ... self._rect: FrameRect = ... + self._listener: FrameListener = ... def __call__(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> Union[ChromiumElement, str]: ... @@ -83,6 +85,9 @@ class ChromiumFrame(ChromiumBase): @property def rect(self) -> FrameRect: ... + @property + def listen(self) -> FrameListener: ... + @property def _obj_id(self) -> str: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 0814dce..0fd998b 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -32,7 +32,7 @@ class ChromiumPage(ChromiumBase): address = self._handle_options(addr_or_opts) self._run_browser() super().__init__(address, tab_id) - self.set.timeouts(implicit=timeout) + self.set.timeouts(base=timeout) self._page_init() def _handle_options(self, addr_or_opts): @@ -48,7 +48,7 @@ class ChromiumPage(ChromiumBase): elif isinstance(addr_or_opts, str): self._chromium_options = ChromiumOptions() - self._chromium_options.set_debugger_address(addr_or_opts) + self._chromium_options.set_address(addr_or_opts) elif isinstance(addr_or_opts, int): self._chromium_options = ChromiumOptions() @@ -57,36 +57,36 @@ class ChromiumPage(ChromiumBase): else: raise TypeError('只能接收ip:port格式或ChromiumOptions类型参数。') - return self._chromium_options.debugger_address + return self._chromium_options.address def _run_browser(self): """连接浏览器""" is_exist = connect_browser(self._chromium_options) try: - ws = get(f'http://{self._chromium_options.debugger_address}/json/version', headers={'Connection': 'close'}) + ws = get(f'http://{self._chromium_options.address}/json/version', headers={'Connection': 'close'}) if not ws: raise BrowserConnectError('\n浏览器连接失败,请检查是否启用全局代理。如是,须设置不代理127.0.0.1地址。') except : raise BrowserConnectError('\n浏览器连接失败,请检查是否启用全局代理。如是,须设置不代理127.0.0.1地址。') ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] - self._browser = Browser(self._chromium_options.debugger_address, ws, self) + self._browser = Browser(self._chromium_options.address, ws, self) if (is_exist and self._chromium_options._headless is False and 'headless' in self._browser.run_cdp('Browser.getVersion')['userAgent'].lower()): self._browser.quit(3) connect_browser(self._chromium_options) - ws = get(f'http://{self._chromium_options.debugger_address}/json/version', headers={'Connection': 'close'}) + ws = get(f'http://{self._chromium_options.address}/json/version', headers={'Connection': 'close'}) ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] - self._browser = Browser(self._chromium_options.debugger_address, ws, self) + self._browser = Browser(self._chromium_options.address, ws, self) def _d_set_runtime_settings(self): """设置运行时用到的属性""" self._timeouts = Timeout(self, page_load=self._chromium_options.timeouts['pageLoad'], script=self._chromium_options.timeouts['script'], - implicit=self._chromium_options.timeouts['implicit']) - if self._chromium_options.timeouts['implicit'] is not None: - self._timeout = self._chromium_options.timeouts['implicit'] + base=self._chromium_options.timeouts['base']) + if self._chromium_options.timeouts['base'] is not None: + self._timeout = self._chromium_options.timeouts['base'] self._load_mode = self._chromium_options.load_mode self._download_path = None if self._chromium_options.download_path is None \ else str(Path(self._chromium_options.download_path).absolute()) diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 64eee41..126153f 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -174,7 +174,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): @property def timeout(self): """返回通用timeout设置""" - return self.timeouts.implicit + return self.timeouts.base @timeout.setter def timeout(self, second): @@ -182,7 +182,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): :param second: 秒数 :return: None """ - self.set.timeouts(implicit=second) + self.set.timeouts(base=second) def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None, **kwargs): """跳转到一个url diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index eaca23b..2f75d8e 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -32,7 +32,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): super().__init__(session_or_options=session_or_options) if not chromium_options: chromium_options = ChromiumOptions(read_file=chromium_options) - chromium_options.set_timeouts(implicit=self._timeout).set_paths(download_path=self.download_path) + chromium_options.set_timeouts(base=self._timeout).set_paths(download_path=self.download_path) super(SessionPage, self).__init__(addr_or_opts=chromium_options, timeout=timeout) self.change_mode(self._mode, go=False, copy_cookies=False) @@ -138,7 +138,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): @property def timeout(self): """返回通用timeout设置""" - return self.timeouts.implicit + return self.timeouts.base @timeout.setter def timeout(self, second): @@ -146,7 +146,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): :param second: 秒数 :return: None """ - self.set.timeouts(implicit=second) + self.set.timeouts(base=second) def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None, **kwargs): """跳转到一个url diff --git a/DrissionPage/_units/action_chains.py b/DrissionPage/_units/actions.py similarity index 99% rename from DrissionPage/_units/action_chains.py rename to DrissionPage/_units/actions.py index df74134..42cee3c 100644 --- a/DrissionPage/_units/action_chains.py +++ b/DrissionPage/_units/actions.py @@ -9,7 +9,7 @@ from .._commons.keys import modifierBit, keyDescriptionForString from .._commons.web import location_in_viewport -class ActionChains: +class Actions: """用于实现动作链的类""" def __init__(self, page): diff --git a/DrissionPage/_units/action_chains.pyi b/DrissionPage/_units/actions.pyi similarity index 65% rename from DrissionPage/_units/action_chains.pyi rename to DrissionPage/_units/actions.pyi index 1c66af6..2f5df4e 100644 --- a/DrissionPage/_units/action_chains.pyi +++ b/DrissionPage/_units/actions.pyi @@ -10,7 +10,7 @@ from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase -class ActionChains: +class Actions: def __init__(self, page: ChromiumBase): self.page: ChromiumBase = ... @@ -20,53 +20,53 @@ class ActionChains: self.curr_y: int = ... def move_to(self, ele_or_loc: Union[ChromiumElement, Tuple[int, int], str], - offset_x: int = 0, offset_y: int = 0, duration: float = .5) -> ActionChains: ... + offset_x: int = 0, offset_y: int = 0, duration: float = .5) -> Actions: ... - def move(self, offset_x: int = 0, offset_y: int = 0, duration: float = .5) -> ActionChains: ... + def move(self, offset_x: int = 0, offset_y: int = 0, duration: float = .5) -> Actions: ... - def click(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def click(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def r_click(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def r_click(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def m_click(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def m_click(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def db_click(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def db_click(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def hold(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def hold(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def release(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def release(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def r_hold(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def r_hold(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def r_release(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def r_release(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def m_hold(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def m_hold(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def m_release(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def m_release(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... def _hold(self, on_ele: Union[ChromiumElement, str] = None, button: str = 'left', - count: int = 1) -> ActionChains: ... + count: int = 1) -> Actions: ... - def _release(self, button: str) -> ActionChains: ... + def _release(self, button: str) -> Actions: ... def scroll(self, delta_x: int = 0, delta_y: int = 0, - on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def up(self, pixel: int) -> ActionChains: ... + def up(self, pixel: int) -> Actions: ... - def down(self, pixel: int) -> ActionChains: ... + def down(self, pixel: int) -> Actions: ... - def left(self, pixel: int) -> ActionChains: ... + def left(self, pixel: int) -> Actions: ... - def right(self, pixel: int) -> ActionChains: ... + def right(self, pixel: int) -> Actions: ... - def key_down(self, key: str) -> ActionChains: ... + def key_down(self, key: str) -> Actions: ... - def key_up(self, key: str) -> ActionChains: ... + def key_up(self, key: str) -> Actions: ... - def type(self, text: Union[str, list, tuple]) -> ActionChains: ... + def type(self, text: Union[str, list, tuple]) -> Actions: ... - def wait(self, second: float) -> ActionChains: ... + def wait(self, second: float) -> Actions: ... def _get_key_data(self, key: str, action: str) -> dict: ... diff --git a/DrissionPage/_units/download_manager.py b/DrissionPage/_units/downloader.py similarity index 100% rename from DrissionPage/_units/download_manager.py rename to DrissionPage/_units/downloader.py diff --git a/DrissionPage/_units/download_manager.pyi b/DrissionPage/_units/downloader.pyi similarity index 100% rename from DrissionPage/_units/download_manager.pyi rename to DrissionPage/_units/downloader.pyi diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/listener.py similarity index 88% rename from DrissionPage/_units/network_listener.py rename to DrissionPage/_units/listener.py index ea57ae8..3323049 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/listener.py @@ -14,7 +14,7 @@ from requests.structures import CaseInsensitiveDict from .._base.chromium_driver import ChromiumDriver -class NetworkListener(object): +class Listener(object): """监听器基类""" def __init__(self, page): @@ -22,6 +22,8 @@ class NetworkListener(object): :param page: ChromiumBase对象 """ self._page = page + self._address = page.address + self._target_id = page._target_id self._driver = None self._caught = None # 临存捕捉到的数据 @@ -77,7 +79,7 @@ class NetworkListener(object): if self.listening: return - self._driver = ChromiumDriver(self._page.tab_id, 'page', self._page.address) + self._driver = ChromiumDriver(self._target_id, 'page', self._address) self._driver.run('Network.enable') self.listening = True @@ -102,7 +104,7 @@ class NetworkListener(object): fail = False else: - end = perf_counter() + count + end = perf_counter() + timeout while True: if perf_counter() > end: fail = True @@ -179,6 +181,26 @@ class NetworkListener(object): self._extra_info_ids = {} self._caught.queue.clear() + def _to_target(self, target_id, address, page): + """切换监听的页面对象 + :param target_id: 新页面对象_target_id + :param address: 新页面对象address + :param page: 新页面对象 + :return: None + """ + self._target_id = target_id + self._address = address + self._page = page + debug = False + if self._driver: + debug = self._driver._debug + self._driver.stop() + if self.listening: + self._driver = ChromiumDriver(self._target_id, 'page', self._address) + self._driver._debug = debug + self._driver.run('Network.enable') + self._set_callback() + def _set_callback(self): """设置监听请求的回调函数""" self._driver.set_callback('Network.requestWillBeSent', self._requestWillBeSent) @@ -190,8 +212,6 @@ class NetworkListener(object): def _requestWillBeSent(self, **kwargs): """接收到请求时的回调函数""" - if kwargs.get('frameId', self._page._frame_id) != self._page._frame_id: - return p = None if not self._targets: if not self._method or kwargs['request']['method'] in self._method: @@ -210,9 +230,6 @@ class NetworkListener(object): not self._method or kwargs['request']['method'] in self._method): p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, target)) p._raw_request = kwargs - if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): - p._raw_post_data = self._driver.run('Network.getRequestPostData', - requestId=rid)['postData'] break self._extra_info_ids.setdefault(kwargs['requestId'], {})['obj'] = p if p else False @@ -223,8 +240,6 @@ class NetworkListener(object): def _response_received(self, **kwargs): """接收到返回信息时处理方法""" - if kwargs.get('frameId', self._page._frame_id) != self._page._frame_id: - return request = self._request_ids.get(kwargs['requestId'], None) if request: request._raw_response = kwargs['response'] @@ -238,7 +253,7 @@ class NetworkListener(object): if obj is False: self._extra_info_ids.pop(kwargs['requestId'], None) elif isinstance(obj, DataPacket): - obj._requestExtraInfo = r['request'] + obj._requestExtraInfo = r.get('request', None) obj._responseExtraInfo = kwargs self._extra_info_ids.pop(kwargs['requestId'], None) else: @@ -246,16 +261,21 @@ class NetworkListener(object): def _loading_finished(self, **kwargs): """请求完成时处理方法""" - r_id = kwargs['requestId'] - dp = self._request_ids.get(r_id) - if dp: - r = self._driver.run('Network.getResponseBody', requestId=r_id) + rid = kwargs['requestId'] + packet = self._request_ids.get(rid) + if packet: + r = self._driver.run('Network.getResponseBody', requestId=rid) if 'body' in r: - dp._raw_body = r['body'] - dp._base64_body = r['base64Encoded'] + packet._raw_body = r['body'] + packet._base64_body = r['base64Encoded'] else: - dp._raw_body = '' - dp._base64_body = False + packet._raw_body = '' + packet._base64_body = False + + if (packet._raw_request['request'].get('hasPostData', None) + and not packet._raw_request['request'].get('postData', None)): + r = self._driver.run('Network.getRequestPostData', requestId=rid, _timeout=1) + packet._raw_post_data = r.get('postData', None) r = self._extra_info_ids.get(kwargs['requestId'], None) if r: @@ -268,10 +288,10 @@ class NetworkListener(object): obj._responseExtraInfo = response self._extra_info_ids.pop(kwargs['requestId'], None) - self._request_ids.pop(r_id, None) + self._request_ids.pop(rid, None) - if dp: - self._caught.put(dp) + if packet: + self._caught.put(packet) def _loading_failed(self, **kwargs): """请求失败时的回调方法""" @@ -299,6 +319,20 @@ class NetworkListener(object): self._caught.put(dp) +class FrameListener(Listener): + def _requestWillBeSent(self, **kwargs): + """接收到请求时的回调函数""" + if not self._page._is_diff_domain and kwargs.get('frameId', None) != self._page._frame_id: + return + super()._requestWillBeSent(**kwargs) + + def _response_received(self, **kwargs): + """接收到返回信息时处理方法""" + if not self._page._is_diff_domain and kwargs.get('frameId', None) != self._page._frame_id: + return + super()._response_received(**kwargs) + + class DataPacket(object): """返回的数据包管理类""" diff --git a/DrissionPage/_units/network_listener.pyi b/DrissionPage/_units/listener.pyi similarity index 93% rename from DrissionPage/_units/network_listener.pyi rename to DrissionPage/_units/listener.pyi index 8ef48cd..2c3a280 100644 --- a/DrissionPage/_units/network_listener.pyi +++ b/DrissionPage/_units/listener.pyi @@ -10,11 +10,14 @@ from requests.structures import CaseInsensitiveDict from .._base.chromium_driver import ChromiumDriver from .._pages.chromium_base import ChromiumBase +from .._pages.chromium_frame import ChromiumFrame -class NetworkListener(object): +class Listener(object): def __init__(self, page: ChromiumBase): self._page: ChromiumBase = ... + self._address: str = ... + self._target_id: str = ... self._targets: Union[str, dict] = ... self._method: set = ... self._caught: Queue = ... @@ -44,6 +47,8 @@ class NetworkListener(object): def clear(self) -> None: ... + def _to_target(self, target_id: str, address: str, page: ChromiumBase) -> None: ... + def start(self, targets: Union[str, List[str], Tuple, bool, None] = None, is_regex: bool = False, method: Union[str, list, tuple, set] = None) \ -> Union[DataPacket, Dict[str, List[DataPacket]], False]: ... @@ -66,6 +71,12 @@ class NetworkListener(object): def _set_callback(self) -> None: ... +class FrameListener(Listener): + def __init__(self, page: ChromiumFrame, is_diff: bool): + self._page: ChromiumFrame = ... + self._is_diff: bool = ... + + class DataPacket(object): """返回的数据包管理类""" diff --git a/DrissionPage/_units/select_element.py b/DrissionPage/_units/selector.py similarity index 100% rename from DrissionPage/_units/select_element.py rename to DrissionPage/_units/selector.py diff --git a/DrissionPage/_units/select_element.pyi b/DrissionPage/_units/selector.pyi similarity index 100% rename from DrissionPage/_units/select_element.pyi rename to DrissionPage/_units/selector.pyi diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index d51d2eb..d0bd70c 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -47,16 +47,17 @@ class ChromiumBaseSetter(BasePageSetter): """设置连接失败重连间隔""" self._page.retry_interval = interval - def timeouts(self, implicit=None, page_load=None, script=None): + def timeouts(self, base=None, page_load=None, script=None, implicit=None): """设置超时时间,单位为秒 - :param implicit: 查找元素超时时间 + :param base: 基本等待时间,除页面加载和脚本超时,其它等待默认使用 :param page_load: 页面加载超时时间 :param script: 脚本运行超时时间 :return: None """ - if implicit is not None: - self._page.timeouts.implicit = implicit - self._page._timeout = implicit + base = base if base is not None else implicit + if base is not None: + self._page.timeouts.base = base + self._page._timeout = base if page_load is not None: self._page.timeouts.page_load = page_load diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index b936565..a254288 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -45,7 +45,7 @@ class ChromiumBaseSetter(BasePageSetter): def retry_interval(self, interval: float) -> None: ... - def timeouts(self, implicit: float = None, page_load: float = None, script: float = None) -> None: ... + def timeouts(self, base: float = None, page_load: float = None, script: float = None) -> None: ... def user_agent(self, ua: str, platform: str = None) -> None: ... diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 3cbca57..0d09ed3 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -130,14 +130,6 @@ class BaseWaiter(object): """ return self._change('title', text, exclude, timeout, raise_err) - def data_packets(self, count=1, timeout=None, fix_count: bool = True): - """等待符合要求的数据包到达指定数量 - :param count: 需要捕捉的数据包数量 - :param timeout: 超时时间,为None无限等待 - :param fix_count: 是否必须满足总数要求,发生超时,为True返回False,为False返回已捕捉到的数据包 - :return: count为1时返回数据包对象,大于1时返回列表,超时且fix_count为True时返回False""" - return self._driver.listen.wait(count, timeout, fix_count) - def _change(self, arg, text, exclude=False, timeout=None, raise_err=None): """等待指定属性变成包含或不包含指定文本 :param arg: 要被匹配的属性 @@ -189,6 +181,16 @@ class BaseWaiter(object): else: return False + # -----------即将废弃----------- + + def data_packets(self, count=1, timeout=None, fix_count: bool = True): + """等待符合要求的数据包到达指定数量 + :param count: 需要捕捉的数据包数量 + :param timeout: 超时时间,为None无限等待 + :param fix_count: 是否必须满足总数要求,发生超时,为True返回False,为False返回已捕捉到的数据包 + :return: count为1时返回数据包对象,大于1时返回列表,超时且fix_count为True时返回False""" + return self._driver.listen.wait(count, timeout, fix_count) + class TabWaiter(BaseWaiter): diff --git a/DrissionPage/_units/waiter.pyi b/DrissionPage/_units/waiter.pyi index d353178..f8634b1 100644 --- a/DrissionPage/_units/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -5,8 +5,8 @@ """ from typing import Union, List -from .download_manager import DownloadMission -from .network_listener import DataPacket +from .downloader import DownloadMission +from .listener import DataPacket from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame @@ -47,9 +47,6 @@ class BaseWaiter(object): def title_change(self, text: str, exclude: bool = False, timeout: float = None, raise_err: bool = None) -> bool: ... - def data_packets(self, count: int = 1, timeout: float = None, - fix_count: bool = True) -> Union[List[DataPacket], DataPacket, None]: ... - def _change(self, arg: str, text: str, exclude: bool = False, timeout: float = None, raise_err: bool = None) -> bool: ... diff --git a/DrissionPage/common.py b/DrissionPage/common.py index 88f9e57..32fabb2 100644 --- a/DrissionPage/common.py +++ b/DrissionPage/common.py @@ -8,6 +8,6 @@ from ._commons.keys import Keys from ._commons.settings import Settings from ._commons.tools import wait_until, configs_to_here from ._elements.session_element import make_session_ele -from ._units.action_chains import ActionChains +from ._units.actions import Actions -__all__ = ['make_session_ele', 'ActionChains', 'Keys', 'By', 'Settings', 'wait_until', 'configs_to_here'] +__all__ = ['make_session_ele', 'Actions', 'Keys', 'By', 'Settings', 'wait_until', 'configs_to_here'] diff --git a/setup.py b/setup.py index 7b3053d..acb5c6a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b15", + version="4.0.0b16", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.",