From f97a7c80dee11e8e30525ef025b44e1961b3fc95 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 11 Aug 2023 14:19:01 +0800 Subject: [PATCH 001/182] =?UTF-8?q?BasePage=E8=A1=A5=E4=B8=8Auser=5Fagent?= =?UTF-8?q?=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/base.py | 4 ++++ DrissionPage/base.pyi | 3 +++ 2 files changed, 7 insertions(+) diff --git a/DrissionPage/base.py b/DrissionPage/base.py index 54b9a7c..7923bb2 100644 --- a/DrissionPage/base.py +++ b/DrissionPage/base.py @@ -428,6 +428,10 @@ class BasePage(BaseParser): def json(self): return + @property + def user_agent(self): + pass + @abstractmethod def get_cookies(self, as_dict=False, all_info=False): return {} diff --git a/DrissionPage/base.pyi b/DrissionPage/base.pyi index eda767f..d24d305 100644 --- a/DrissionPage/base.pyi +++ b/DrissionPage/base.pyi @@ -195,6 +195,9 @@ class BasePage(BaseParser): @property def json(self) -> dict: ... + @property + def user_agent(self) -> str: ... + @abstractmethod def get_cookies(self, as_dict: bool = False, all_info: bool = False) -> Union[list, dict]: ... From 2b1b7f6188e73f599e779f48d504288b4c5af2a4 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 11 Aug 2023 15:47:12 +0800 Subject: [PATCH 002/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=85=B3=E9=97=ADtab?= =?UTF-8?q?=E6=97=B6=E5=B0=8F=E5=87=A0=E7=8E=87=E6=8A=A5=E9=94=99=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_driver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/DrissionPage/chromium_driver.py b/DrissionPage/chromium_driver.py index 09e9cbf..01aede3 100644 --- a/DrissionPage/chromium_driver.py +++ b/DrissionPage/chromium_driver.py @@ -93,6 +93,9 @@ class ChromiumDriver(object): continue + except Exception: + return None + finally: self.method_results.pop(message['id'], None) From dbabe35489f663939913728c0917cc364556c0cb Mon Sep 17 00:00:00 2001 From: g1879 Date: Sat, 19 Aug 2023 23:37:52 +0800 Subject: [PATCH 003/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dpage.set.timeouts()?= =?UTF-8?q?=E4=B8=ADimplicit=E5=8F=82=E6=95=B0=E5=A4=B1=E6=95=88=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/setter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index 1faeaa2..a49bbfd 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -42,6 +42,7 @@ class ChromiumBaseSetter(object): """ if implicit is not None: self._page.timeouts.implicit = implicit + self._page.timeout = implicit if page_load is not None: self._page.timeouts.page_load = page_load From 5f429840f85b501b7296b09d1365882c255caf78 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 22 Aug 2023 09:15:26 +0800 Subject: [PATCH 004/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dset.timeouts()?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=9B=E5=B1=8F=E8=94=BDLinux=E5=92=8CMAC?= =?UTF-8?q?=E7=B3=BB=E7=BB=9F=E4=B8=AD=E8=BF=90=E8=A1=8C=E6=97=B6=E9=82=A3?= =?UTF-8?q?=E4=BA=9B=E5=95=B0=E5=97=A6=E7=9A=84=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/commons/browser.py | 4 ++-- DrissionPage/setter.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DrissionPage/commons/browser.py b/DrissionPage/commons/browser.py index 86b3e9e..339091f 100644 --- a/DrissionPage/commons/browser.py +++ b/DrissionPage/commons/browser.py @@ -5,7 +5,7 @@ """ from json import load, dump from pathlib import Path -from subprocess import Popen +from subprocess import Popen, DEVNULL from tempfile import gettempdir from time import perf_counter, sleep @@ -173,7 +173,7 @@ def _run_browser(port, path: str, args) -> Popen: arguments = [p, f'--remote-debugging-port={port}'] arguments.extend(args) try: - return Popen(arguments, shell=False) + return Popen(arguments, shell=False, stdout=DEVNULL, stderr=DEVNULL) except FileNotFoundError: raise FileNotFoundError('未找到浏览器，请手动指定浏览器可执行文件路径。') diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index a49bbfd..8cce32c 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -42,7 +42,7 @@ class ChromiumBaseSetter(object): """ if implicit is not None: self._page.timeouts.implicit = implicit - self._page.timeout = implicit + self._page._timeout = implicit if page_load is not None: self._page.timeouts.page_load = page_load From 03ae186f069ee1e54f562d4e131d5b6a647849de Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 22 Aug 2023 23:56:03 +0800 Subject: [PATCH 005/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=8B=BC=E5=86=99?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_base.py | 2 +- DrissionPage/chromium_frame.py | 4 ++-- DrissionPage/waiter.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index 283c199..e8bd861 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -159,7 +159,7 @@ class ChromiumBase(BasePage): self._is_reading = False def _wait_loaded(self, timeout=None): - """等待页面加载完成 + """等待页面加载完成，超时触发停止加载 :param timeout: 超时时间 :return: 是否成功，超时返回False """ diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/chromium_frame.py index 34aa115..5fe0031 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/chromium_frame.py @@ -80,7 +80,7 @@ class ChromiumFrame(ChromiumBase): """重新获取document""" debug = self._debug if debug: - print('reload') + print('重新获取document') self._frame_ele = ChromiumElement(self.page, backend_id=self._backend_id) node = self.page.run_cdp('DOM.describeNode', backendNodeId=self._frame_ele.ids.backend_id)['node'] @@ -568,7 +568,7 @@ class ChromiumFrame(ChromiumBase): for t in range(times + 1): err = None - result = self.driver.Page.navigate(url=to_url, frameId=self.frame_id) + result = self.driver.call_method('Page.navigate', url=to_url, frameId=self.frame_id) is_timeout = not self._wait_loaded(timeout) sleep(.5) diff --git a/DrissionPage/waiter.py b/DrissionPage/waiter.py index 25b98a1..0df8fb0 100644 --- a/DrissionPage/waiter.py +++ b/DrissionPage/waiter.py @@ -66,7 +66,7 @@ class ChromiumBaseWaiter(object): return self._loading(timeout=timeout, gap=.002, raise_err=raise_err) def load_complete(self, timeout=None, raise_err=None): - """等待页面开始加载 + """等待页面加载完成 :param timeout: 超时时间，为None时使用页面timeout属性 :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 From fa918f105a1e7f35a368094563aa134bf881fa46 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 24 Aug 2023 10:05:46 +0800 Subject: [PATCH 006/182] =?UTF-8?q?quit()=E4=BC=9A=E7=AD=89=E5=BE=85?= =?UTF-8?q?=E6=B5=8F=E8=A7=88=E5=99=A8=E8=BF=9B=E7=A8=8B=E5=85=B3=E9=97=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_page.py | 14 ++++++++++---- DrissionPage/web_page.py | 3 +-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 936243e..6158afd 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -9,7 +9,6 @@ from .chromium_base import ChromiumBase, Timeout from .chromium_driver import ChromiumDriver from .chromium_tab import ChromiumTab from .commons.browser import connect_browser -from .commons.tools import port_is_using from .configs.chromium_options import ChromiumOptions from .errors import BrowserConnectError from .setter import ChromiumPageSetter @@ -338,9 +337,16 @@ class ChromiumPage(ChromiumBase): """关闭浏览器""" self._tab_obj.call_method('Browser.close') self._tab_obj.stop() - ip, port = self.address.split(':') - while port_is_using(ip, port): - sleep(.1) + + if self.process_id: + from os import popen + from platform import system + txt = f'tasklist | findstr {self.process_id}' if system().lower() == 'windows' \ + else f'ps -ef | grep {self.process_id}' + while True: + p = popen(txt) + if f' {self.process_id} ' not in p.read(): + break def _on_alert_close(self, **kwargs): """alert关闭时触发的方法""" diff --git a/DrissionPage/web_page.py b/DrissionPage/web_page.py index b36b09c..ec8a872 100644 --- a/DrissionPage/web_page.py +++ b/DrissionPage/web_page.py @@ -421,8 +421,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self._response = None self._has_session = None if self._has_driver: - self._tab_obj.call_method('Browser.close') - self._tab_obj.stop() + super(SessionPage, self).quit() self._tab_obj = None self._has_driver = None From 8d89e4f7ef91986aa2eb70520362a7afeaa943df Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 25 Aug 2023 18:32:11 +0800 Subject: [PATCH 007/182] =?UTF-8?q?=E7=94=A8=5Ftarget=5Fid=E5=8F=96?= =?UTF-8?q?=E4=BB=A3tab=5Fid=EF=BC=8Cframe=E5=AF=B9=E8=B1=A1=E8=83=BD?= =?UTF-8?q?=E6=AD=A3=E7=A1=AE=E8=BF=94=E5=9B=9Etab=20id=EF=BC=9B=E4=BF=AE?= =?UTF-8?q?=E5=A4=8Dtab=E6=B2=A1=E6=9C=89=E7=BB=A7=E6=89=BFpage=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E8=B7=AF=E5=BE=84=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=9B?= =?UTF-8?q?page=E5=92=8Ctab=E5=AF=B9=E8=B1=A1=E6=B7=BB=E5=8A=A0set.downloa?= =?UTF-8?q?d=5Fpath?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/base.py | 7 ++++--- DrissionPage/chromium_base.py | 15 ++++++++----- DrissionPage/chromium_base.pyi | 3 +++ DrissionPage/chromium_element.py | 2 +- DrissionPage/chromium_frame.py | 12 ++++++++++- DrissionPage/chromium_frame.pyi | 3 +++ DrissionPage/chromium_page.py | 8 +++---- DrissionPage/chromium_tab.py | 9 ++++++++ DrissionPage/chromium_tab.pyi | 4 ++++ DrissionPage/setter.py | 36 ++++++++++++++++++++++++++++---- DrissionPage/setter.pyi | 17 +++++++++++---- 11 files changed, 94 insertions(+), 22 deletions(-) diff --git a/DrissionPage/base.py b/DrissionPage/base.py index 7923bb2..33aa5c1 100644 --- a/DrissionPage/base.py +++ b/DrissionPage/base.py @@ -363,10 +363,11 @@ class BasePage(BaseParser): """初始化函数""" self._url = None self.timeout = timeout if timeout is not None else 10 - self.retry_times = 3 - self.retry_interval = 2 self._url_available = None - self._download_path = '' + if not hasattr(self, 'retry_times'): + self.retry_times = 3 + if not hasattr(self, 'retry_interval'): + self.retry_interval = 2 self._DownloadKit = None @property diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index e8bd861..ffbfb8b 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -189,7 +189,7 @@ class ChromiumBase(BasePage): def _onFrameStartedLoading(self, **kwargs): """页面开始加载时触发""" - if kwargs['frameId'] == self.tab_id: + if kwargs['frameId'] == self._target_id: self._is_loading = True if self._debug: @@ -199,7 +199,7 @@ class ChromiumBase(BasePage): def _onFrameStoppedLoading(self, **kwargs): """页面加载完成后触发""" - if kwargs['frameId'] == self.tab_id and self._first_run is False and self._is_loading: + if kwargs['frameId'] == self._target_id and self._first_run is False and self._is_loading: if self._debug: print('页面停止加载 FrameStoppedLoading') if self._debug_recorder: @@ -225,7 +225,7 @@ class ChromiumBase(BasePage): def _onFrameNavigated(self, **kwargs): """页面跳转时触发""" - if kwargs['frame'].get('parentId', None) == self.tab_id and self._first_run is False and self._is_loading: + if kwargs['frame'].get('parentId', None) == self._target_id and self._first_run is False and self._is_loading: self._is_loading = True if self._debug: print('navigated') @@ -275,12 +275,12 @@ class ChromiumBase(BasePage): @property def title(self): """返回当前页面title""" - return self.run_cdp_loaded('Target.getTargetInfo', targetId=self.tab_id)['targetInfo']['title'] + return self.run_cdp_loaded('Target.getTargetInfo', targetId=self._target_id)['targetInfo']['title'] @property def url(self): """返回当前页面url""" - return self.run_cdp_loaded('Target.getTargetInfo', targetId=self.tab_id)['targetInfo']['url'] + return self.run_cdp_loaded('Target.getTargetInfo', targetId=self._target_id)['targetInfo']['url'] @property def _browser_url(self): @@ -303,6 +303,11 @@ class ChromiumBase(BasePage): @property def tab_id(self): + """返回当前标签页id""" + return self._target_id + + @property + def _target_id(self): """返回当前标签页id""" return self.driver.id if self.driver.status == 'started' else '' diff --git a/DrissionPage/chromium_base.pyi b/DrissionPage/chromium_base.pyi index ebbbd1b..810ab1a 100644 --- a/DrissionPage/chromium_base.pyi +++ b/DrissionPage/chromium_base.pyi @@ -97,6 +97,9 @@ class ChromiumBase(BasePage): @property def json(self) -> Union[dict, None]: ... + @property + def _target_id(self) -> str: ... + @property def tab_id(self) -> str: ... diff --git a/DrissionPage/chromium_element.py b/DrissionPage/chromium_element.py index 3750000..afcb6d4 100644 --- a/DrissionPage/chromium_element.py +++ b/DrissionPage/chromium_element.py @@ -463,7 +463,7 @@ class ChromiumElement(DrissionElement): node = self.page.run_cdp('DOM.describeNode', backendNodeId=self._backend_id)['node'] frame = node.get('frameId', None) - frame = frame or self.page.tab_id + frame = frame or self.page._target_id try: result = self.page.run_cdp('Page.getResourceContent', frameId=frame, url=src) diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/chromium_frame.py index 5fe0031..d9d797d 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/chromium_frame.py @@ -17,10 +17,15 @@ from .waiter import FrameWaiter class ChromiumFrame(ChromiumBase): def __init__(self, page, ele): + """ + :param page: frame所在的页面对象 + :param ele: frame所在元素 + """ self.page = page self.address = page.address node = page.run_cdp('DOM.describeNode', backendNodeId=ele.ids.backend_id)['node'] self.frame_id = node['frameId'] + self._tab_id = page.tab_id self._backend_id = ele.ids.backend_id self._frame_ele = ele self._states = None @@ -329,6 +334,11 @@ class ChromiumFrame(ChromiumBase): self._wait = FrameWaiter(self) return self._wait + @property + def tab_id(self): + """返回frame所在tab的id""" + return self._tab_id + def refresh(self): """刷新frame页面""" self._check_ok() @@ -616,7 +626,7 @@ class ChromiumFrameIds(object): @property def tab_id(self): """返回当前标签页id""" - return self._frame.page.tab_id + return self._frame._tab_id @property def backend_id(self): diff --git a/DrissionPage/chromium_frame.pyi b/DrissionPage/chromium_frame.pyi index a2bdce8..df14080 100644 --- a/DrissionPage/chromium_frame.pyi +++ b/DrissionPage/chromium_frame.pyi @@ -114,6 +114,9 @@ class ChromiumFrame(ChromiumBase): @property def wait(self) -> FrameWaiter: ... + @property + def tab_id(self) -> str: ... + def refresh(self) -> None: ... def attr(self, attr: str) -> Union[str, None]: ... diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 6158afd..e6f3dff 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -99,10 +99,6 @@ class ChromiumPage(ChromiumBase): self._rect = None self._main_tab = self.tab_id - # try: - # self.download_set.by_browser() - # except CDPError: - # pass self._process_id = None r = self.browser_driver.call_method('SystemInfo.getProcessInfo') @@ -440,6 +436,10 @@ class ChromiumTabRect(object): return self._page.browser_driver.call_method('Browser.getWindowForTarget', targetId=self._page.tab_id)['bounds'] +class BrowserDownloadManager(object): + def __init__(self, page): + self._page = page + self.frames = {} # class BaseDownloadSetter(DownloadSetter): # """用于设置下载参数的类""" # diff --git a/DrissionPage/chromium_tab.py b/DrissionPage/chromium_tab.py index 9279600..6fdfa38 100644 --- a/DrissionPage/chromium_tab.py +++ b/DrissionPage/chromium_tab.py @@ -8,6 +8,7 @@ from copy import copy from .chromium_base import ChromiumBase from .commons.web import set_session_cookies, set_browser_cookies from .session_page import SessionPage +from .setter import TabSetter from .setter import WebPageTabSetter @@ -28,6 +29,7 @@ class ChromiumTab(ChromiumBase): self.retry_times = self.page.retry_times self.retry_interval = self.page.retry_interval self._page_load_strategy = self.page.page_load_strategy + self._download_path = self.page.download_path def close(self): """关闭当前标签页""" @@ -38,6 +40,13 @@ class ChromiumTab(ChromiumBase): """返回获取窗口坐标和大小的对象""" return self.page.rect + @property + def set(self): + """返回用于等待的对象""" + if self._set is None: + self._set = TabSetter(self) + return self._set + class WebPageTab(SessionPage, ChromiumTab): def __init__(self, page, tab_id): diff --git a/DrissionPage/chromium_tab.pyi b/DrissionPage/chromium_tab.pyi index 04f3ad6..fecc3fb 100644 --- a/DrissionPage/chromium_tab.pyi +++ b/DrissionPage/chromium_tab.pyi @@ -13,6 +13,7 @@ from .chromium_frame import ChromiumFrame from .chromium_page import ChromiumPage, ChromiumTabRect from .session_element import SessionElement from .session_page import SessionPage +from .setter import TabSetter from .setter import WebPageTabSetter from .web_page import WebPage @@ -29,6 +30,9 @@ class ChromiumTab(ChromiumBase): @property def rect(self) -> ChromiumTabRect: ... + @property + def set(self) -> TabSetter: ... + class WebPageTab(SessionPage, ChromiumTab): def __init__(self, page: WebPage, tab_id: str): diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index 8cce32c..8343cc2 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -118,7 +118,23 @@ class ChromiumBaseSetter(object): self._page.run_cdp('Network.setExtraHTTPHeaders', headers=headers) -class ChromiumPageSetter(ChromiumBaseSetter): +class DownloadSetter(object): + def download_path(self, path): + """设置下载路径 + :param path: 下载路径 + :return: None + """ + self._page._download_path = str(path) + if self._page._DownloadKit: + self._page._DownloadKit.set.goal_path(path) + + +class TabSetter(ChromiumBaseSetter, DownloadSetter): + def __init__(self, page): + super().__init__(page) + + +class ChromiumPageSetter(ChromiumBaseSetter, DownloadSetter): def main_tab(self, tab_id=None): """设置主tab :param tab_id: 标签页id，不传入则设置当前tab @@ -143,8 +159,11 @@ class ChromiumPageSetter(ChromiumBaseSetter): self._page._control_session.get(f'http://{self._page.address}/json/activate/{tab_or_id}') -class SessionPageSetter(object): +class SessionPageSetter(DownloadSetter): def __init__(self, page): + """ + :param page: SessionPage对象 + """ self._page = page def retry_times(self, times): @@ -155,6 +174,15 @@ class SessionPageSetter(object): """设置连接失败时重连间隔""" self._page.retry_interval = interval + def download_path(self, path): + """设置下载路径 + :param path: 下载路径 + :return: None + """ + self._page._download_path = str(path) + if self._page._DownloadKit: + self._page._DownloadKit.set.goal_path(path) + def timeout(self, second): """设置连接超时时间 :param second: 秒数 @@ -274,7 +302,7 @@ class SessionPageSetter(object): self._page.session.mount(url, adapter) -class WebPageSetter(ChromiumPageSetter): +class WebPageSetter(ChromiumPageSetter, DownloadSetter): def __init__(self, page): super().__init__(page) self._session_setter = SessionPageSetter(self._page) @@ -308,7 +336,7 @@ class WebPageSetter(ChromiumPageSetter): self._chromium_setter.user_agent(ua, platform) -class WebPageTabSetter(ChromiumBaseSetter): +class WebPageTabSetter(ChromiumBaseSetter, DownloadSetter): def __init__(self, page): super().__init__(page) self._session_setter = SessionPageSetter(self._page) diff --git a/DrissionPage/setter.pyi b/DrissionPage/setter.pyi index e750130..dfddd59 100644 --- a/DrissionPage/setter.pyi +++ b/DrissionPage/setter.pyi @@ -4,6 +4,7 @@ @Contact : g1879@qq.com """ from http.cookiejar import Cookie +from pathlib import Path from typing import Union, Tuple from requests.adapters import HTTPAdapter @@ -50,7 +51,15 @@ class ChromiumBaseSetter(object): def upload_files(self, files: Union[str, list, tuple]) -> None: ... -class ChromiumPageSetter(ChromiumBaseSetter): +class DownloadSetter(object): + def download_path(self, path: Union[str, Path]) -> None: ... + + +class TabSetter(ChromiumBaseSetter, DownloadSetter): + def __init__(self, page): ... + + +class ChromiumPageSetter(ChromiumBaseSetter, DownloadSetter): _page: ChromiumPage = ... def main_tab(self, tab_id: str = None) -> None: ... @@ -61,7 +70,7 @@ class ChromiumPageSetter(ChromiumBaseSetter): def tab_to_front(self, tab_or_id: Union[str, ChromiumTab] = None) -> None: ... -class SessionPageSetter(object): +class SessionPageSetter(DownloadSetter): def __init__(self, page: SessionPage): self._page: SessionPage = ... @@ -102,7 +111,7 @@ class SessionPageSetter(object): def add_adapter(self, url: str, adapter: HTTPAdapter) -> None: ... -class WebPageSetter(ChromiumPageSetter): +class WebPageSetter(ChromiumPageSetter, DownloadSetter): _page: WebPage = ... _session_setter: SessionPageSetter = ... _chromium_setter: ChromiumPageSetter = ... @@ -114,7 +123,7 @@ class WebPageSetter(ChromiumPageSetter): def cookies(self, cookies) -> None: ... -class WebPageTabSetter(ChromiumBaseSetter): +class WebPageTabSetter(ChromiumBaseSetter, DownloadSetter): _page: WebPage = ... _session_setter: SessionPageSetter = ... _chromium_setter: ChromiumBaseSetter = ... From fd5e8aa89d3d6a00c7e81e57ac125c1721e8282f Mon Sep 17 00:00:00 2001 From: g1879 Date: Sat, 26 Aug 2023 08:46:25 +0800 Subject: [PATCH 008/182] =?UTF-8?q?ChromiumFrame=E5=A2=9E=E5=8A=A0=5Ftarge?= =?UTF-8?q?t=5Fpage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_driver.py | 23 ++++++++-------- DrissionPage/chromium_driver.pyi | 2 +- DrissionPage/chromium_frame.py | 47 +++++++++++++++++++------------- DrissionPage/chromium_frame.pyi | 3 ++ DrissionPage/chromium_page.py | 6 +++- 5 files changed, 49 insertions(+), 32 deletions(-) diff --git a/DrissionPage/chromium_driver.py b/DrissionPage/chromium_driver.py index 01aede3..fde230f 100644 --- a/DrissionPage/chromium_driver.py +++ b/DrissionPage/chromium_driver.py @@ -25,7 +25,7 @@ class ChromiumDriver(object): self.id = tab_id self.address = address self.type = tab_type - self.debug = False + self._debug = False self.has_alert = False self._websocket_url = f'ws://{address}/devtools/{tab_type}/{tab_id}' @@ -57,11 +57,12 @@ class ChromiumDriver(object): message_json = dumps(message) - if self.debug: - if self.debug is True or (isinstance(self.debug, str) and message.get('method', '').startswith(self.debug)): + if self._debug: + if self._debug is True or ( + isinstance(self._debug, str) and message.get('method', '').startswith(self._debug)): print(f'发> {message_json}') - elif isinstance(self.debug, (list, tuple, set)): - for m in self.debug: + elif isinstance(self._debug, (list, tuple, set)): + for m in self._debug: if message.get('method', '').startswith(m): print(f'发> {message_json}') break @@ -112,12 +113,12 @@ class ChromiumDriver(object): self.stop() return - if self.debug: - if self.debug is True or 'id' in mes or (isinstance(self.debug, str) - and mes.get('method', '').startswith(self.debug)): + if self._debug: + if self._debug is True or 'id' in mes or (isinstance(self._debug, str) + and mes.get('method', '').startswith(self._debug)): print(f'<收 {message_json}') - elif isinstance(self.debug, (list, tuple, set)): - for m in self.debug: + elif isinstance(self._debug, (list, tuple, set)): + for m in self._debug: if mes.get('method', '').startswith(m): print(f'<收 {message_json}') break @@ -129,7 +130,7 @@ class ChromiumDriver(object): if mes["id"] in self.method_results: self.method_results[mes['id']].put(mes) - elif self.debug: + elif self._debug: print(f'未知信息：{mes}') def _handle_event_loop(self): diff --git a/DrissionPage/chromium_driver.pyi b/DrissionPage/chromium_driver.pyi index 0c63041..9ba7feb 100644 --- a/DrissionPage/chromium_driver.pyi +++ b/DrissionPage/chromium_driver.pyi @@ -23,7 +23,7 @@ class ChromiumDriver(object): id: str address: str type: str - debug: bool + _debug: bool has_alert: bool _websocket_url: str _cur_id: int diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/chromium_frame.py index d9d797d..a5fbebe 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/chromium_frame.py @@ -21,7 +21,14 @@ class ChromiumFrame(ChromiumBase): :param page: frame所在的页面对象 :param ele: frame所在元素 """ - self.page = page + page_type = str(type(page)) + if 'ChromiumPage' in page_type or 'WebPage' in page: + self.page = self._target_page = self.tab = page + else: # Tab、Frame + self.page = page.page + self._target_page = page + self.tab = page.tab if 'ChromiumFrame' in page_type else page + self.address = page.address node = page.run_cdp('DOM.describeNode', backendNodeId=ele.ids.backend_id)['node'] self.frame_id = node['frameId'] @@ -32,7 +39,7 @@ class ChromiumFrame(ChromiumBase): if self._is_inner_frame(): self._is_diff_domain = False - self.doc_ele = ChromiumElement(self.page, backend_id=node['contentDocument']['backendNodeId']) + self.doc_ele = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) super().__init__(page.address, page.tab_id, page.timeout) else: self._is_diff_domain = True @@ -65,8 +72,8 @@ class ChromiumFrame(ChromiumBase): def _runtime_settings(self): """重写设置浏览器运行参数方法""" - self._timeouts = self.page.timeouts - self._page_load_strategy = self.page.page_load_strategy + self._timeouts = self._target_page.timeouts + self._page_load_strategy = self._target_page.page_load_strategy def _driver_init(self, tab_id): """避免出现服务器500错误 @@ -87,18 +94,18 @@ class ChromiumFrame(ChromiumBase): if debug: print('重新获取document') - self._frame_ele = ChromiumElement(self.page, backend_id=self._backend_id) - node = self.page.run_cdp('DOM.describeNode', backendNodeId=self._frame_ele.ids.backend_id)['node'] + self._frame_ele = ChromiumElement(self._target_page, backend_id=self._backend_id) + node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self._frame_ele.ids.backend_id)['node'] if self._is_inner_frame(): self._is_diff_domain = False - self.doc_ele = ChromiumElement(self.page, backend_id=node['contentDocument']['backendNodeId']) - super().__init__(self.address, self.page.tab_id, self.page.timeout) + self.doc_ele = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) + super().__init__(self.address, self._target_page.tab_id, self._target_page.timeout) self._debug = debug else: self._is_diff_domain = True self._tab_obj.stop() - super().__init__(self.address, self.frame_id, self.page.timeout) + super().__init__(self.address, self.frame_id, self._target_page.timeout) obj_id = super().run_js('document;', as_expr=True)['objectId'] self.doc_ele = ChromiumElement(self, obj_id=obj_id) self._debug = debug @@ -109,7 +116,7 @@ class ChromiumFrame(ChromiumBase): self._reload() try: - self.page.run_cdp('DOM.describeNode', nodeId=self.ids.node_id) + self._target_page.run_cdp('DOM.describeNode', nodeId=self.ids.node_id) except Exception: self._reload() # sleep(2) @@ -126,8 +133,9 @@ class ChromiumFrame(ChromiumBase): while self.is_alive and perf_counter() < end_time: try: if self._is_diff_domain is False: - node = self.page.run_cdp('DOM.describeNode', backendNodeId=self.ids.backend_id)['node'] - self.doc_ele = ChromiumElement(self.page, backend_id=node['contentDocument']['backendNodeId']) + node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self.ids.backend_id)['node'] + self.doc_ele = ChromiumElement(self._target_page, + backend_id=node['contentDocument']['backendNodeId']) else: b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] @@ -207,7 +215,8 @@ class ChromiumFrame(ChromiumBase): """返回元素outerHTML文本""" self._check_ok() tag = self.tag - out_html = self.page.run_cdp('DOM.getOuterHTML', backendNodeId=self.frame_ele.ids.backend_id)['outerHTML'] + out_html = self._target_page.run_cdp('DOM.getOuterHTML', backendNodeId=self.frame_ele.ids.backend_id)[ + 'outerHTML'] sign = search(rf'<{tag}.*?>', out_html).group(0) return f'{sign}{self.inner_html}' @@ -296,7 +305,7 @@ class ChromiumFrame(ChromiumBase): except ContextLossError: try: node = self.run_cdp('DOM.describeNode', backendNodeId=self.frame_ele.ids.backend_id)['node'] - doc = ChromiumElement(self.page, backend_id=node['contentDocument']['backendNodeId']) + doc = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) return doc.run_js('return this.readyState;') except: pass @@ -525,7 +534,7 @@ class ChromiumFrame(ChromiumBase): cx, cy = ele.locations.viewport_location w, h = ele.size img_data = f'data:image/{pic_type};base64,{self.frame_ele.get_screenshot(as_base64=True)}' - body = self.page('t:body') + body = self._target_page('t:body') first_child = body('c::first-child') if not isinstance(first_child, ChromiumElement): first_child = first_child.frame_ele @@ -540,9 +549,9 @@ class ChromiumFrame(ChromiumBase): new_ele.scroll.to_see(True) top = int(self.frame_ele.style('border-top').split('px')[0]) left = int(self.frame_ele.style('border-left').split('px')[0]) - r = self.page.get_screenshot(path=path, as_bytes=as_bytes, as_base64=as_base64, - left_top=(cx + left, cy + top), right_bottom=(cx + w + left, cy + h + top)) - self.page.remove_ele(new_ele) + r = self._target_page.get_screenshot(path=path, as_bytes=as_bytes, as_base64=as_base64, + left_top=(cx + left, cy + top), right_bottom=(cx + w + left, cy + h + top)) + self._target_page.remove_ele(new_ele) return r def _find_elements(self, loc_or_ele, timeout=None, single=True, relative=False, raise_err=None): @@ -610,7 +619,7 @@ class ChromiumFrame(ChromiumBase): def _is_inner_frame(self): """返回当前frame是否同域""" - return self.frame_id in str(self.page.run_cdp('Page.getFrameTree')['frameTree']) + return self.frame_id in str(self._target_page.run_cdp('Page.getFrameTree')['frameTree']) def _check_alive(self): """检测iframe是否有效线程方法""" diff --git a/DrissionPage/chromium_frame.pyi b/DrissionPage/chromium_frame.pyi index df14080..f4cb6ba 100644 --- a/DrissionPage/chromium_frame.pyi +++ b/DrissionPage/chromium_frame.pyi @@ -16,6 +16,9 @@ class ChromiumFrame(ChromiumBase): def __init__(self, page: ChromiumBase, ele: ChromiumElement): self.page: ChromiumBase = ... + self._target_page = ... + self.tab = ... + self._tab_id: str = ... self.frame_id: str = ... self._frame_ele: ChromiumElement = ... self._backend_id: str = ... diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index e6f3dff..6678cb7 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -343,6 +343,7 @@ class ChromiumPage(ChromiumBase): p = popen(txt) if f' {self.process_id} ' not in p.read(): break + sleep(.2) def _on_alert_close(self, **kwargs): """alert关闭时触发的方法""" @@ -438,8 +439,11 @@ class ChromiumTabRect(object): class BrowserDownloadManager(object): def __init__(self, page): - self._page = page + self.page = page self.frames = {} + self.pause = False + + # class BaseDownloadSetter(DownloadSetter): # """用于设置下载参数的类""" # From 42e8217c8e73a84125ff03ed5bb814ea5b683e44 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 28 Aug 2023 00:07:51 +0800 Subject: [PATCH 009/182] =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E7=B1=BB=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=5Fpage=E5=B1=9E=E6=80=A7=E4=BB=A5=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E6=80=BB=E9=A1=B5=E9=9D=A2=E5=AF=B9=E8=B1=A1=EF=BC=9B=E7=BB=A7?= =?UTF-8?q?=E7=BB=AD=E5=AE=8C=E5=96=84=E6=B5=8F=E8=A7=88=E5=99=A8=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E5=8A=9F=E8=83=BD=EF=BC=8C=E6=9C=AA=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_base.py | 8 ++++++++ DrissionPage/chromium_base.pyi | 4 ++++ DrissionPage/chromium_frame.py | 12 ++++++++++-- DrissionPage/chromium_frame.pyi | 14 +++++++++++--- DrissionPage/chromium_page.py | 25 ++++++++++++++++++++++--- DrissionPage/chromium_page.pyi | 15 +++++++++++++-- DrissionPage/chromium_tab.py | 7 ++++++- DrissionPage/chromium_tab.pyi | 10 ++++++++-- DrissionPage/setter.py | 9 +++++++++ DrissionPage/setter.pyi | 2 ++ 10 files changed, 93 insertions(+), 13 deletions(-) diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index ffbfb8b..6e08aba 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -111,6 +111,7 @@ class ChromiumBase(BasePage): self._tab_obj.set_listener('DOM.documentUpdated', self._onDocumentUpdated) self._tab_obj.set_listener('Page.loadEventFired', self._onLoadEventFired) self._tab_obj.set_listener('Page.frameNavigated', self._onFrameNavigated) + self._tab_obj.set_listener('Page.downloadWillBegin', self._onDownloadWillBegin) def _get_document(self): """刷新cdp使用的document数据""" @@ -242,6 +243,9 @@ class ChromiumBase(BasePage): self.run_cdp('Page.setInterceptFileChooserDialog', enabled=False) self._upload_list = None + def _onDownloadWillBegin(self, **kwargs): + self._page._dl_mgr.add_mission(kwargs['guid'], self.download_path, kwargs['suggestedFilename']) + def __call__(self, loc_or_str, timeout=None): """在内部查找元素 例：ele = page('@id=ele_id') @@ -251,6 +255,10 @@ class ChromiumBase(BasePage): """ return self.ele(loc_or_str, timeout) + @property + def page(self): + return self._page + @property def driver(self): """返回用于控制浏览器的ChromiumDriver对象""" diff --git a/DrissionPage/chromium_base.pyi b/DrissionPage/chromium_base.pyi index 810ab1a..94463f1 100644 --- a/DrissionPage/chromium_base.pyi +++ b/DrissionPage/chromium_base.pyi @@ -13,6 +13,7 @@ from .base import BasePage from .chromium_driver import ChromiumDriver from .chromium_element import ChromiumElement, ChromiumScroll from .chromium_frame import ChromiumFrame +from .chromium_page import ChromiumPage from .commons.constants import NoneElement from .network_listener import NetworkListener from .session_element import SessionElement @@ -25,6 +26,7 @@ class ChromiumBase(BasePage): address: Union[str, int], tab_id: str = None, timeout: float = None): + self._page: ChromiumPage = ... self._control_session: Session = ... self.address: str = ... self._tab_obj: ChromiumDriver = ... @@ -66,6 +68,8 @@ class ChromiumBase(BasePage): def _onFileChooserOpened(self, **kwargs): ... + def _onDownloadWillBegin(self, **kwargs): ... + def _set_start_options(self, address, none) -> None: ... def _set_runtime_settings(self) -> None: ... diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/chromium_frame.py index a5fbebe..001d49f 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/chromium_frame.py @@ -23,9 +23,9 @@ class ChromiumFrame(ChromiumBase): """ page_type = str(type(page)) if 'ChromiumPage' in page_type or 'WebPage' in page: - self.page = self._target_page = self.tab = page + self._page = self._target_page = self.tab = page else: # Tab、Frame - self.page = page.page + self._page = page.page self._target_page = page self.tab = page.tab if 'ChromiumFrame' in page_type else page @@ -189,6 +189,10 @@ class ChromiumFrame(ChromiumBase): print('页面停止加载 FrameStoppedLoading') self._get_new_document() + @property + def page(self): + return self._page + @property def ids(self): return self._ids @@ -348,6 +352,10 @@ class ChromiumFrame(ChromiumBase): """返回frame所在tab的id""" return self._tab_id + @property + def download_path(self): + return self.tab.download_path + def refresh(self): """刷新frame页面""" self._check_ok() diff --git a/DrissionPage/chromium_frame.pyi b/DrissionPage/chromium_frame.pyi index f4cb6ba..1a798f6 100644 --- a/DrissionPage/chromium_frame.pyi +++ b/DrissionPage/chromium_frame.pyi @@ -6,6 +6,8 @@ from pathlib import Path from typing import Union, Tuple, List, Any +from DrissionPage import ChromiumPage, WebPage +from .chromium_tab import ChromiumTab from .chromium_base import ChromiumBase, ChromiumPageScroll from .chromium_element import ChromiumElement, Locations, ChromiumElementStates from .setter import ChromiumFrameSetter @@ -15,9 +17,9 @@ from .waiter import FrameWaiter class ChromiumFrame(ChromiumBase): def __init__(self, page: ChromiumBase, ele: ChromiumElement): - self.page: ChromiumBase = ... - self._target_page = ... - self.tab = ... + self._page: ChromiumPage = ... + self._target_page: ChromiumBase = ... + self.tab: ChromiumTab = ... self._tab_id: str = ... self.frame_id: str = ... self._frame_ele: ChromiumElement = ... @@ -51,6 +53,9 @@ class ChromiumFrame(ChromiumBase): def _onFrameDetached(self, **kwargs): ... + @property + def page(self) -> Union[ChromiumPage, WebPage]: ... + @property def ids(self) -> ChromiumFrameIds: ... @@ -120,6 +125,9 @@ class ChromiumFrame(ChromiumBase): @property def tab_id(self) -> str: ... + @property + def download_path(self) -> str: ... + def refresh(self) -> None: ... def attr(self, attr: str) -> Union[str, None]: ... diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 6678cb7..8d7e52b 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -3,6 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ +from shutil import move from time import perf_counter, sleep from .chromium_base import ChromiumBase, Timeout @@ -25,6 +26,8 @@ class ChromiumPage(ChromiumBase): :param timeout: 超时时间 """ super().__init__(addr_driver_opts, tab_id, timeout) + self._page = self + self._dl_mgr = BrowserDownloadManager(self) def _set_start_options(self, addr_driver_opts, none): """设置浏览器启动属性 @@ -439,9 +442,25 @@ class ChromiumTabRect(object): class BrowserDownloadManager(object): def __init__(self, page): - self.page = page - self.frames = {} - self.pause = False + self._page = page + page.set.download_path(page.download_path) + self._page.browser_driver.set_listener('Browser.downloadProgress', self._onDownloadProgress) + self._missions = {} + + def add_mission(self, guid, path, name): + print(name) + self._missions[guid] = {'path': path, 'name': name} + + def _onDownloadProgress(self, **kwargs): + # todo: 处理同名文件、处理后缀 + if kwargs['state'] == 'completed' and kwargs['guid'] in self._missions: + guid = kwargs['guid'] + path = self._missions[guid]['path'] + name = self._missions[guid]['name'] + form_path = f'{self._page.download_path}\\{guid}' + to_path = f'{path}\\{name}' + move(form_path, to_path) + self._missions.pop(guid) # class BaseDownloadSetter(DownloadSetter): diff --git a/DrissionPage/chromium_page.pyi b/DrissionPage/chromium_page.pyi index 916d85c..1f5f1c3 100644 --- a/DrissionPage/chromium_page.pyi +++ b/DrissionPage/chromium_page.pyi @@ -5,11 +5,11 @@ """ from typing import Union, Tuple, List -from .setter import ChromiumPageSetter from .chromium_base import ChromiumBase from .chromium_driver import ChromiumDriver from .chromium_tab import ChromiumTab from .configs.chromium_options import ChromiumOptions +from .setter import ChromiumPageSetter from .waiter import ChromiumPageWaiter @@ -21,7 +21,7 @@ class ChromiumPage(ChromiumBase): timeout: float = None): self._driver_options: ChromiumOptions = ... self._process_id: str = ... - # self._window_setter: WindowSetter = ... + self._dl_mgr: BrowserDownloadManager = ... self._main_tab: str = ... self._alert: Alert = ... self._browser_driver: ChromiumDriver = ... @@ -125,6 +125,17 @@ class ChromiumTabRect(object): def _get_browser_rect(self) -> dict: ... +class BrowserDownloadManager(object): + _page: ChromiumPage = ... + _missions: dict = ... + + def __init__(self, page: ChromiumPage): ... + + def add_mission(self, guid: str, path: str, name: str) -> None: ... + + def _onDownloadProgress(self, **kwargs) -> None: ... + + # class BaseDownloadSetter(DownloadSetter): # def __init__(self, page: ChromiumPage): # self._page: ChromiumPage = ... diff --git a/DrissionPage/chromium_tab.py b/DrissionPage/chromium_tab.py index 6fdfa38..9f68d5f 100644 --- a/DrissionPage/chromium_tab.py +++ b/DrissionPage/chromium_tab.py @@ -20,7 +20,7 @@ class ChromiumTab(ChromiumBase): :param page: ChromiumPage对象 :param tab_id: 要控制的标签页id，不指定默认为激活的 """ - self.page = page + self._page = page super().__init__(page.address, tab_id, page.timeout) def _set_runtime_settings(self): @@ -35,6 +35,11 @@ class ChromiumTab(ChromiumBase): """关闭当前标签页""" self.page.close_tabs(self.tab_id) + @property + def page(self): + """返回总体page对象""" + return self._page + @property def rect(self): """返回获取窗口坐标和大小的对象""" diff --git a/DrissionPage/chromium_tab.pyi b/DrissionPage/chromium_tab.pyi index fecc3fb..850c1f3 100644 --- a/DrissionPage/chromium_tab.pyi +++ b/DrissionPage/chromium_tab.pyi @@ -21,12 +21,15 @@ from .web_page import WebPage class ChromiumTab(ChromiumBase): def __init__(self, page: ChromiumPage, tab_id: str = None): - self.page: ChromiumPage = ... + self._page: ChromiumPage = ... def _set_runtime_settings(self) -> None: ... def close(self) -> None: ... + @property + def page(self) -> ChromiumPage: ... + @property def rect(self) -> ChromiumTabRect: ... @@ -36,7 +39,7 @@ class ChromiumTab(ChromiumBase): class WebPageTab(SessionPage, ChromiumTab): def __init__(self, page: WebPage, tab_id: str): - self.page: WebPage = ... + self._page: WebPage = ... self._mode: str = ... self._has_driver = ... self._has_session = ... @@ -45,6 +48,9 @@ class WebPageTab(SessionPage, ChromiumTab): loc_or_str: Union[Tuple[str, str], str, ChromiumElement, SessionElement], timeout: float = None) -> Union[ChromiumElement, SessionElement]: ... + @property + def page(self) -> WebPage: ... + @property def url(self) -> Union[str, None]: ... diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index 8343cc2..b46cf9d 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -158,6 +158,15 @@ class ChromiumPageSetter(ChromiumBaseSetter, DownloadSetter): tab_or_id = tab_or_id.tab_id self._page._control_session.get(f'http://{self._page.address}/json/activate/{tab_or_id}') + def download_path(self, path): + """设置下载路径 + :param path: 下载路径 + :return: None + """ + super().download_path(path) + self._page.browser_driver.call_method('Browser.setDownloadBehavior', downloadPath=self._page.download_path, + behavior='allowAndName', eventsEnabled=True) + class SessionPageSetter(DownloadSetter): def __init__(self, page): diff --git a/DrissionPage/setter.pyi b/DrissionPage/setter.pyi index dfddd59..bbf6cec 100644 --- a/DrissionPage/setter.pyi +++ b/DrissionPage/setter.pyi @@ -69,6 +69,8 @@ class ChromiumPageSetter(ChromiumBaseSetter, DownloadSetter): def tab_to_front(self, tab_or_id: Union[str, ChromiumTab] = None) -> None: ... + def download_path(self, path: Union[str, Path]) -> None: ... + class SessionPageSetter(DownloadSetter): def __init__(self, page: SessionPage): From bd54c1f48173e51cf617fdcdeae5c37c0b2e85e2 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 28 Aug 2023 17:09:02 +0800 Subject: [PATCH 010/182] =?UTF-8?q?=E7=BB=A7=E7=BB=AD=E5=AE=8C=E5=96=84?= =?UTF-8?q?=E6=B5=8F=E8=A7=88=E5=99=A8=E4=B8=8B=E8=BD=BD=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=EF=BC=8C=E6=9C=AA=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_base.py | 4 ++++ DrissionPage/chromium_base.pyi | 1 + DrissionPage/chromium_page.py | 6 +++--- DrissionPage/waiter.py | 24 ++++++++++++++++++++++++ DrissionPage/waiter.pyi | 4 +++- 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index 6e08aba..b07db5f 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -64,6 +64,7 @@ class ChromiumBase(BasePage): def _set_runtime_settings(self): self._timeouts = Timeout(self) self._page_load_strategy = 'normal' + self._wait_download_flag = None def _connect_browser(self, tab_id=None): """连接浏览器，在第一次时运行 @@ -244,7 +245,10 @@ class ChromiumBase(BasePage): self._upload_list = None def _onDownloadWillBegin(self, **kwargs): + if self._wait_download_flag is False: + self._page.run_cdp('Browser.cancelDownload', guid=kwargs['guid']) self._page._dl_mgr.add_mission(kwargs['guid'], self.download_path, kwargs['suggestedFilename']) + self._wait_download_flag = {'url': kwargs['url'], 'name': kwargs['suggestedFilename']} def __call__(self, loc_or_str, timeout=None): """在内部查找元素 diff --git a/DrissionPage/chromium_base.pyi b/DrissionPage/chromium_base.pyi index 94463f1..433e7a2 100644 --- a/DrissionPage/chromium_base.pyi +++ b/DrissionPage/chromium_base.pyi @@ -45,6 +45,7 @@ class ChromiumBase(BasePage): self._set: ChromiumBaseSetter = ... self._screencast: Screencast = ... self._listener: NetworkListener = ... + self._wait_download_flag: bool = ... def _connect_browser(self, tab_id: str = None) -> None: ... diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 8d7e52b..9700c26 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -10,6 +10,7 @@ from .chromium_base import ChromiumBase, Timeout from .chromium_driver import ChromiumDriver from .chromium_tab import ChromiumTab from .commons.browser import connect_browser +from .commons.tools import get_usable_path from .configs.chromium_options import ChromiumOptions from .errors import BrowserConnectError from .setter import ChromiumPageSetter @@ -448,17 +449,16 @@ class BrowserDownloadManager(object): self._missions = {} def add_mission(self, guid, path, name): - print(name) self._missions[guid] = {'path': path, 'name': name} def _onDownloadProgress(self, **kwargs): - # todo: 处理同名文件、处理后缀 if kwargs['state'] == 'completed' and kwargs['guid'] in self._missions: guid = kwargs['guid'] path = self._missions[guid]['path'] name = self._missions[guid]['name'] form_path = f'{self._page.download_path}\\{guid}' - to_path = f'{path}\\{name}' + to_path = get_usable_path(f'{path}\\{name}') + move(form_path, to_path) self._missions.pop(guid) diff --git a/DrissionPage/waiter.py b/DrissionPage/waiter.py index 0df8fb0..89cddee 100644 --- a/DrissionPage/waiter.py +++ b/DrissionPage/waiter.py @@ -78,6 +78,26 @@ class ChromiumBaseWaiter(object): while self._driver._upload_list: sleep(.01) + def browser_download_begin(self, timeout=None, cancel=False): + """等待浏览器下载开始，可将其拦截 + :param timeout: 超时时间，None使用页面对象超时时间 + :param cancel: 是否取消该任务 + :return: 成功返回任务信息dict，失败返回False + """ + self._driver._wait_download_flag = False if cancel else True + if timeout is None: + timeout = self._driver.timeout + + r = False + end_time = perf_counter() + timeout + while perf_counter() < end_time: + if not isinstance(self._driver._wait_download_flag, bool): + r = self._driver._wait_download_flag + break + + self._driver._wait_download_flag = None + return r + def url_change(self, text, exclude=False, timeout=None, raise_err=None): """等待url变成包含或不包含指定文本 :param text: 用于识别的文本 @@ -173,6 +193,10 @@ class ChromiumPageWaiter(ChromiumBaseWaiter): else: return False + def browser_downloads_complete(self): + """等待所有下载任务结束""" + pass + class ChromiumElementWaiter(object): """等待元素在dom中某种状态，如删除、显示、隐藏""" diff --git a/DrissionPage/waiter.pyi b/DrissionPage/waiter.pyi index 548c167..2a659ce 100644 --- a/DrissionPage/waiter.pyi +++ b/DrissionPage/waiter.pyi @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union +from typing import Union, Optional from .chromium_base import ChromiumBase from .chromium_element import ChromiumElement @@ -35,6 +35,8 @@ class ChromiumBaseWaiter(object): def upload_paths_inputted(self) -> None: ... + def browser_download_begin(self, timeout: float = None, cancel: bool = False) -> Union[dict, bool]: ... + def url_change(self, text: str, exclude: bool = False, timeout: float = None, raise_err: bool = None) -> bool: ... def title_change(self, text: str, exclude: bool = False, timeout: float = None, raise_err: bool = None) -> bool: ... From b8e2be87992f536747898e9934e4134ee5d3d569 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 28 Aug 2023 22:59:16 +0800 Subject: [PATCH 011/182] =?UTF-8?q?=E5=9F=BA=E6=9C=AC=E5=AE=8C=E6=88=90?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E5=8A=9F=E8=83=BD=E4=BF=AE=E6=94=B9=EF=BC=8C?= =?UTF-8?q?=E5=BE=85=E6=B5=8B=E8=AF=95=E5=92=8C=E5=AE=8C=E5=96=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_base.py | 30 +++- DrissionPage/chromium_base.pyi | 1 + DrissionPage/chromium_page.py | 270 +++------------------------------ DrissionPage/chromium_page.pyi | 63 +------- DrissionPage/commons/tools.py | 1 + DrissionPage/setter.py | 19 ++- DrissionPage/setter.pyi | 14 +- DrissionPage/waiter.py | 20 ++- DrissionPage/waiter.pyi | 10 +- 9 files changed, 86 insertions(+), 342 deletions(-) diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index b07db5f..8c18e5e 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -43,6 +43,8 @@ class ChromiumBase(BasePage): self._set = None self._screencast = None self._listener = None + self._wait_download_flag = None + self._download_rename = None if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): address = f'127.0.0.1:{address}' @@ -64,7 +66,6 @@ class ChromiumBase(BasePage): def _set_runtime_settings(self): self._timeouts = Timeout(self) self._page_load_strategy = 'normal' - self._wait_download_flag = None def _connect_browser(self, tab_id=None): """连接浏览器，在第一次时运行 @@ -190,7 +191,7 @@ class ChromiumBase(BasePage): return False def _onFrameStartedLoading(self, **kwargs): - """页面开始加载时触发""" + """页面开始加载时执行""" if kwargs['frameId'] == self._target_id: self._is_loading = True @@ -200,7 +201,7 @@ class ChromiumBase(BasePage): self._debug_recorder.add_data((perf_counter(), '加载流程', 'FrameStartedLoading')) def _onFrameStoppedLoading(self, **kwargs): - """页面加载完成后触发""" + """页面加载完成后执行""" if kwargs['frameId'] == self._target_id and self._first_run is False and self._is_loading: if self._debug: print('页面停止加载 FrameStoppedLoading') @@ -219,14 +220,14 @@ class ChromiumBase(BasePage): self._get_document() def _onDocumentUpdated(self, **kwargs): - """页面跳转时触发""" + """页面跳转时执行""" if self._debug: print('documentUpdated') if self._debug_recorder: self._debug_recorder.add_data((perf_counter(), '加载流程', 'documentUpdated')) def _onFrameNavigated(self, **kwargs): - """页面跳转时触发""" + """页面跳转时执行""" if kwargs['frame'].get('parentId', None) == self._target_id and self._first_run is False and self._is_loading: self._is_loading = True if self._debug: @@ -235,7 +236,7 @@ class ChromiumBase(BasePage): self._debug_recorder.add_data((perf_counter(), '加载流程', 'navigated')) def _onFileChooserOpened(self, **kwargs): - """文件选择框打开时触发""" + """文件选择框打开时执行""" if self._upload_list: files = self._upload_list if kwargs['mode'] == 'selectMultiple' else self._upload_list[:1] self.run_cdp('DOM.setFileInputFiles', files=files, backendNodeId=kwargs['backendNodeId']) @@ -245,10 +246,23 @@ class ChromiumBase(BasePage): self._upload_list = None def _onDownloadWillBegin(self, **kwargs): + """下载即将开始时执行""" if self._wait_download_flag is False: self._page.run_cdp('Browser.cancelDownload', guid=kwargs['guid']) - self._page._dl_mgr.add_mission(kwargs['guid'], self.download_path, kwargs['suggestedFilename']) - self._wait_download_flag = {'url': kwargs['url'], 'name': kwargs['suggestedFilename']} + + if self._download_rename: + tmp = kwargs['suggestedFilename'].rsplit('.', 1) + ext_name = tmp[-1] if len(tmp) > 1 else '' + tmp = self._download_rename.rsplit('.', 1) + ext_rename = tmp[-1] if len(tmp) > 1 else '' + n = self._download_rename if ext_rename == ext_name else f'{self._download_rename}.{ext_name}' + self._download_rename = None + + else: + n = kwargs['suggestedFilename'] + + self._page._dl_mgr.add_mission(kwargs['guid'], self.download_path, n) + self._wait_download_flag = {'url': kwargs['url'], 'name': n} def __call__(self, loc_or_str, timeout=None): """在内部查找元素 diff --git a/DrissionPage/chromium_base.pyi b/DrissionPage/chromium_base.pyi index 433e7a2..03d102f 100644 --- a/DrissionPage/chromium_base.pyi +++ b/DrissionPage/chromium_base.pyi @@ -46,6 +46,7 @@ class ChromiumBase(BasePage): self._screencast: Screencast = ... self._listener: NetworkListener = ... self._wait_download_flag: bool = ... + self._download_rename: str = ... def _connect_browser(self, tab_id: str = None) -> None: ... diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 9700c26..14bbfbe 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -446,266 +446,38 @@ class BrowserDownloadManager(object): self._page = page page.set.download_path(page.download_path) self._page.browser_driver.set_listener('Browser.downloadProgress', self._onDownloadProgress) + self._page.browser_driver.set_listener('Browser.downloadWillBegin', self._onDownloadWillBegin) self._missions = {} def add_mission(self, guid, path, name): + """添加下载任务信息 + :param guid: guid + :param path: 保存路径 + :param name: 保存文件名 + :return: None + """ self._missions[guid] = {'path': path, 'name': name} + def _onDownloadWillBegin(self, **kwargs): + """用于获取弹出新标签页触发的下载任务""" + sleep(.1) + if kwargs['guid'] not in self._missions: + self.add_mission(kwargs['guid'], self._page.download_path, kwargs['suggestedFilename']) + def _onDownloadProgress(self, **kwargs): - if kwargs['state'] == 'completed' and kwargs['guid'] in self._missions: + """下载状态变化时执行""" + if kwargs['state'] in ('completed', 'canceled') and kwargs['guid'] in self._missions: guid = kwargs['guid'] - path = self._missions[guid]['path'] - name = self._missions[guid]['name'] - form_path = f'{self._page.download_path}\\{guid}' - to_path = get_usable_path(f'{path}\\{name}') + if kwargs['state'] == 'completed': + path = self._missions[guid]['path'] + name = self._missions[guid]['name'] + form_path = f'{self._page.download_path}\\{guid}' + to_path = get_usable_path(f'{path}\\{name}') + move(form_path, to_path) - move(form_path, to_path) self._missions.pop(guid) -# class BaseDownloadSetter(DownloadSetter): -# """用于设置下载参数的类""" -# -# def __init__(self, page): -# """ -# :param page: ChromiumPage对象 -# """ -# super().__init__(page) -# self._behavior = 'allowAndName' -# self._session = None -# self._save_path = '' -# self._rename = None -# self._waiting_download = False -# self._download_begin = False -# self._browser_missions = {} -# self._browser_downloading_count = 0 -# self._show_msg = True -# -# @property -# def session(self): -# """返回用于DownloadKit的Session对象""" -# if self._session is None: -# self._session = Session() -# return self._session -# -# @property -# def browser_missions(self): -# """返回浏览器下载任务""" -# return list(self._browser_missions.values()) -# -# @property -# def DownloadKit_missions(self): -# """返回DownloadKit下载任务""" -# return list(self.DownloadKit.missions.values()) -# -# @property -# def _switched_DownloadKit(self): -# """返回从浏览器同步cookies后的Session对象""" -# self._cookies_to_session() -# return self.DownloadKit -# -# def save_path(self, path): -# """设置下载路径 -# :param path: 下载路径 -# :return: None -# """ -# path = path or '' -# path = Path(path).absolute() -# path.mkdir(parents=True, exist_ok=True) -# path = str(path) -# self._save_path = path -# self._page._download_path = path -# try: -# self._page.browser_driver.Browser.setDownloadBehavior(behavior='allowAndName', downloadPath=path, -# eventsEnabled=True) -# except CDPError: -# warn('\n您的浏览器版本太低，用新标签页下载文件可能崩溃，建议升级。') -# self._page.run_cdp('Page.setDownloadBehavior', behavior='allowAndName', downloadPath=path) -# -# self.DownloadKit.goal_path = path -# -# def rename(self, name): -# """设置浏览器下一个下载任务的文件名 -# :param name: 文件名，不带后缀时自动使用原后缀 -# :return: None -# """ -# self._rename = name -# -# def by_browser(self): -# """设置使用浏览器下载文件""" -# try: -# self._page.browser_driver.Browser.setDownloadBehavior(behavior='allowAndName', eventsEnabled=True, -# downloadPath=self._page.download_path) -# self._page.browser_driver.Browser.downloadWillBegin = self._download_will_begin -# self._page.browser_driver.Browser.downloadProgress = self._download_progress -# except CDPError: -# self._page.driver.Page.setDownloadBehavior(behavior='allowAndName', downloadPath=self._page.download_path) -# self._page.driver.Page.downloadWillBegin = self._download_will_begin -# self._page.driver.Page.downloadProgress = self._download_progress -# -# self._behavior = 'allowAndName' -# -# def by_DownloadKit(self): -# """设置使用DownloadKit下载文件""" -# try: -# self._page.browser_driver.Browser.setDownloadBehavior(behavior='deny', eventsEnabled=True) -# self._page.browser_driver.Browser.downloadWillBegin = self._download_by_DownloadKit -# except CDPError: -# raise RuntimeError('您的浏览器版本太低，不支持此方法，请升级。') -# -# self._behavior = 'deny' -# -# def wait_download_begin(self, timeout=None): -# """等待浏览器下载开始 -# :param timeout: 等待超时时间，为None则使用页面对象timeout属性 -# :return: 是否等到下载开始 -# """ -# self._waiting_download = True -# result = False -# timeout = timeout if timeout is not None else self._page.timeout -# end_time = perf_counter() + timeout -# while perf_counter() < end_time: -# if self._download_begin: -# result = True -# break -# sleep(.05) -# self._download_begin = False -# self._waiting_download = False -# return result -# -# def wait_download_finish(self, timeout=None): -# """等待所有下载结束 -# :param timeout: 超时时间 -# :return: 是否等待到下载完成 -# """ -# timeout = timeout if timeout is not None else self._page.timeout -# end_time = perf_counter() + timeout -# while perf_counter() < end_time: -# if (self._DownloadKit is None or not self.DownloadKit.is_running) and self._browser_downloading_count == 0: -# return True -# sleep(.5) -# return False -# -# def show_msg(self, on_off=True): -# """是否显示下载信息 -# :param on_off: bool表示开或关 -# :return: None -# """ -# self._show_msg = on_off -# -# def _cookies_to_session(self): -# """把driver对象的cookies复制到session对象""" -# ua = self._page.run_cdp('Runtime.evaluate', expression='navigator.userAgent;')['result']['value'] -# self.session.headers.update({"User-Agent": ua}) -# set_session_cookies(self.session, self._page.get_cookies(as_dict=False, all_info=False)) -# -# def _download_by_DownloadKit(self, **kwargs): -# """拦截浏览器下载并用downloadKit下载""" -# url = kwargs['url'] -# if url.startswith('blob:'): -# raise TypeError('bolb:开头的链接无法使用DownloadKit下载，请用浏览器下载功能。') -# -# self._page.browser_driver.Browser.cancelDownload(guid=kwargs['guid']) -# -# if self._rename: -# rename = get_rename(kwargs['suggestedFilename'], self._rename) -# self._rename = None -# else: -# rename = kwargs['suggestedFilename'] -# -# mission = self._page.download.add(file_url=url, goal_path=self._page.download_path, rename=rename) -# Thread(target=self._wait_download_complete, args=(mission,), daemon=False).start() -# -# if self._waiting_download: -# self._download_begin = True -# -# self._browser_downloading_count += 1 -# -# if self._show_msg: -# print(f'(DownloadKit)开始下载：{Path(self._save_path) / rename}') -# -# def _download_will_begin(self, **kwargs): -# """浏览器下载即将开始时调用""" -# if self._rename: -# rename = get_rename(kwargs['suggestedFilename'], self._rename) -# self._rename = None -# else: -# rename = kwargs['suggestedFilename'] -# -# m = BrowserDownloadMission(kwargs['guid'], kwargs['url'], rename) -# self._browser_missions[kwargs['guid']] = m -# aid_path = Path(self._save_path) / rename -# -# if self._show_msg: -# print(f'(Browser)开始下载：{rename}') -# self._browser_downloading_count += 1 -# -# if self._file_exists == 'skip' and aid_path.exists(): -# m.state = 'skipped' -# m.save_path = aid_path.absolute() -# self._page.browser_driver.call_method('Browser.cancelDownload', guid=kwargs['guid']) -# (Path(self._save_path) / kwargs["guid"]).unlink(missing_ok=True) -# return -# -# if self._waiting_download: -# self._download_begin = True -# -# def _download_progress(self, **kwargs): -# """下载状态产生变化时调用""" -# guid = kwargs['guid'] -# m = self._browser_missions.get(guid, None) -# if m: -# m.size = kwargs['totalBytes'] -# m.received = kwargs['receivedBytes'] -# m.state = kwargs['state'] -# -# if m.state == 'completed': -# path = Path(self._save_path) / m.name -# from_path = Path(self._save_path) / guid -# if path.exists(): -# if self._file_exists == 'rename': -# path = get_usable_path(path) -# else: # 'overwrite' -# path.unlink() -# from_path.rename(path) -# m.save_path = path.absolute() -# -# if kwargs['state'] != 'inProgress': -# if self._show_msg and m: -# if kwargs['state'] == 'completed': -# print(f'(Browser)下载完成：{m.save_path}') -# elif m.state != 'skipped': -# print(f'(Browser)下载失败：{m.save_path}') -# else: -# print(f'(Browser)已跳过：{m.save_path}') -# self._browser_downloading_count -= 1 -# -# def _wait_download_complete(self, mission): -# """等待DownloadKit下载完成""" -# mission.wait(show=False) -# if self._show_msg: -# if mission.result == 'skip': -# print(f'(DownloadKit)已跳过：{mission.path}') -# elif not mission.result: -# print(f'(DownloadKit)下载失败：{mission.path}') -# else: -# print(f'(DownloadKit)下载完成：{mission.path}') - - -class BrowserDownloadMission(object): - def __init__(self, guid, url, name): - self.id = guid - self.url = url - self.name = name - self.save_path = None - self.state = None - self.size = None - self.received = None - - def __repr__(self): - return f'' - - class Alert(object): """用于保存alert信息的类""" diff --git a/DrissionPage/chromium_page.pyi b/DrissionPage/chromium_page.pyi index 1f5f1c3..3dab20f 100644 --- a/DrissionPage/chromium_page.pyi +++ b/DrissionPage/chromium_page.pyi @@ -133,70 +133,11 @@ class BrowserDownloadManager(object): def add_mission(self, guid: str, path: str, name: str) -> None: ... + def _onDownloadWillBegin(self, **kwargs) -> None: ... + def _onDownloadProgress(self, **kwargs) -> None: ... -# class BaseDownloadSetter(DownloadSetter): -# def __init__(self, page: ChromiumPage): -# self._page: ChromiumPage = ... -# self._behavior: str = ... -# self._session: Session = ... -# self._save_path: str = ... -# self._rename: str = ... -# self._waiting_download: bool = ... -# self._download_begin: bool = ... -# self._browser_missions: Dict[str, BrowserDownloadMission] = ... -# self._browser_downloading_count: int = ... -# self._show_msg: bool = ... -# -# @property -# def session(self) -> Session: ... -# -# @property -# def browser_missions(self) -> List[BrowserDownloadMission]: ... -# -# @property -# def DownloadKit_missions(self) -> List[Mission]: ... -# -# @property -# def _switched_DownloadKit(self) -> DownloadKit: ... -# -# def save_path(self, path: Union[str, Path]) -> None: ... -# -# def rename(self, name: str) -> None: ... -# -# def by_browser(self) -> None: ... -# -# def by_DownloadKit(self) -> None: ... -# -# def wait_download_begin(self, timeout: float = None) -> bool: ... -# -# def wait_download_finish(self, timeout: float = None) -> bool: ... -# -# def show_msg(self, on_off: bool = True) -> None: ... -# -# def _cookies_to_session(self) -> None: ... -# -# def _download_by_DownloadKit(self, **kwargs) -> None: ... -# -# def _download_will_begin(self, **kwargs) -> None: ... -# -# def _download_progress(self, **kwargs) -> None: ... -# -# def _wait_download_complete(self, mission: Mission) -> None: ... - - -class BrowserDownloadMission(object): - def __init__(self, guid: str, url: str, name: str): - self.id: str = ... - self.url: str = ... - self.name: str = ... - self.save_path: str = ... - self.state: str = ... - self.size: str = ... - self.received: str = ... - - class Alert(object): def __init__(self): diff --git a/DrissionPage/commons/tools.py b/DrissionPage/commons/tools.py index 5adf7ca..dfdab22 100644 --- a/DrissionPage/commons/tools.py +++ b/DrissionPage/commons/tools.py @@ -16,6 +16,7 @@ def get_usable_path(path): """ path = Path(path) parent = path.parent + parent.mkdir(parents=True, exist_ok=True) path = parent / make_valid_name(path.name) name = path.stem if path.is_file() else path.name ext = path.suffix if path.is_file() else '' diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index b46cf9d..dcfd210 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -117,8 +117,6 @@ class ChromiumBaseSetter(object): self._page.run_cdp('Network.enable') self._page.run_cdp('Network.setExtraHTTPHeaders', headers=headers) - -class DownloadSetter(object): def download_path(self, path): """设置下载路径 :param path: 下载路径 @@ -128,13 +126,20 @@ class DownloadSetter(object): if self._page._DownloadKit: self._page._DownloadKit.set.goal_path(path) + def download_file_name(self, name): + """设置下一个被下载文件的名称 + :param name: 文件名，可不含后缀 + :return: None + """ + self._page._download_rename = name -class TabSetter(ChromiumBaseSetter, DownloadSetter): + +class TabSetter(ChromiumBaseSetter): def __init__(self, page): super().__init__(page) -class ChromiumPageSetter(ChromiumBaseSetter, DownloadSetter): +class ChromiumPageSetter(ChromiumBaseSetter): def main_tab(self, tab_id=None): """设置主tab :param tab_id: 标签页id，不传入则设置当前tab @@ -168,7 +173,7 @@ class ChromiumPageSetter(ChromiumBaseSetter, DownloadSetter): behavior='allowAndName', eventsEnabled=True) -class SessionPageSetter(DownloadSetter): +class SessionPageSetter(object): def __init__(self, page): """ :param page: SessionPage对象 @@ -311,7 +316,7 @@ class SessionPageSetter(DownloadSetter): self._page.session.mount(url, adapter) -class WebPageSetter(ChromiumPageSetter, DownloadSetter): +class WebPageSetter(ChromiumPageSetter): def __init__(self, page): super().__init__(page) self._session_setter = SessionPageSetter(self._page) @@ -345,7 +350,7 @@ class WebPageSetter(ChromiumPageSetter, DownloadSetter): self._chromium_setter.user_agent(ua, platform) -class WebPageTabSetter(ChromiumBaseSetter, DownloadSetter): +class WebPageTabSetter(ChromiumBaseSetter): def __init__(self, page): super().__init__(page) self._session_setter = SessionPageSetter(self._page) diff --git a/DrissionPage/setter.pyi b/DrissionPage/setter.pyi index bbf6cec..7e2748a 100644 --- a/DrissionPage/setter.pyi +++ b/DrissionPage/setter.pyi @@ -50,16 +50,16 @@ class ChromiumBaseSetter(object): def upload_files(self, files: Union[str, list, tuple]) -> None: ... - -class DownloadSetter(object): def download_path(self, path: Union[str, Path]) -> None: ... + def download_file_name(self, name: str) -> None: ... -class TabSetter(ChromiumBaseSetter, DownloadSetter): + +class TabSetter(ChromiumBaseSetter): def __init__(self, page): ... -class ChromiumPageSetter(ChromiumBaseSetter, DownloadSetter): +class ChromiumPageSetter(ChromiumBaseSetter): _page: ChromiumPage = ... def main_tab(self, tab_id: str = None) -> None: ... @@ -72,7 +72,7 @@ class ChromiumPageSetter(ChromiumBaseSetter, DownloadSetter): def download_path(self, path: Union[str, Path]) -> None: ... -class SessionPageSetter(DownloadSetter): +class SessionPageSetter(object): def __init__(self, page: SessionPage): self._page: SessionPage = ... @@ -113,7 +113,7 @@ class SessionPageSetter(DownloadSetter): def add_adapter(self, url: str, adapter: HTTPAdapter) -> None: ... -class WebPageSetter(ChromiumPageSetter, DownloadSetter): +class WebPageSetter(ChromiumPageSetter): _page: WebPage = ... _session_setter: SessionPageSetter = ... _chromium_setter: ChromiumPageSetter = ... @@ -125,7 +125,7 @@ class WebPageSetter(ChromiumPageSetter, DownloadSetter): def cookies(self, cookies) -> None: ... -class WebPageTabSetter(ChromiumBaseSetter, DownloadSetter): +class WebPageTabSetter(ChromiumBaseSetter): _page: WebPage = ... _session_setter: SessionPageSetter = ... _chromium_setter: ChromiumBaseSetter = ... diff --git a/DrissionPage/waiter.py b/DrissionPage/waiter.py index 89cddee..93d5d5b 100644 --- a/DrissionPage/waiter.py +++ b/DrissionPage/waiter.py @@ -193,9 +193,23 @@ class ChromiumPageWaiter(ChromiumBaseWaiter): else: return False - def browser_downloads_complete(self): - """等待所有下载任务结束""" - pass + def browser_downloads_complete(self, timeout=None): + """等待所有下载任务结束 + :param timeout: 超时时间，为None时无限等待 + :return: 是否等待成功 + """ + if not timeout: + while self._driver._dl_mgr._missions: + sleep(.5) + return True + + else: + end_time = perf_counter() + timeout + while end_time > perf_counter(): + if not self._driver._dl_mgr._missions: + return True + sleep(.5) + return False if self._driver._dl_mgr._missions else True class ChromiumElementWaiter(object): diff --git a/DrissionPage/waiter.pyi b/DrissionPage/waiter.pyi index 2a659ce..66bbb43 100644 --- a/DrissionPage/waiter.pyi +++ b/DrissionPage/waiter.pyi @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union, Optional +from typing import Union from .chromium_base import ChromiumBase from .chromium_element import ChromiumElement @@ -48,14 +48,10 @@ class ChromiumBaseWaiter(object): class ChromiumPageWaiter(ChromiumBaseWaiter): _driver: ChromiumPage = ... - # _listener: Union[NetworkListener, None] = ... - - # def download_begin(self, timeout: float = 1.5) -> bool: ... - - # def download_finish(self, timeout: float = None) -> bool: ... - def new_tab(self, timeout: float = None, raise_err: bool = None) -> bool: ... + def browser_downloads_complete(self, timeout: float = None) -> bool: ... + class ChromiumElementWaiter(object): def __init__(self, From 8170d53c9706c9cf9494b9987eb092e2796dd983 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 29 Aug 2023 17:39:43 +0800 Subject: [PATCH 012/182] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=80=BB=E8=BE=91=EF=BC=9B=E7=BB=A7?= =?UTF-8?q?=E7=BB=AD=E5=AE=8C=E5=96=84=E4=B8=8B=E8=BD=BD=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/base.py | 14 ++++++-------- DrissionPage/base.pyi | 2 +- DrissionPage/chromium_base.py | 6 ++++-- DrissionPage/chromium_frame.py | 10 +++++++--- DrissionPage/chromium_frame.pyi | 2 +- DrissionPage/chromium_page.py | 24 +++++++++++++++++++++--- DrissionPage/chromium_tab.py | 2 +- DrissionPage/network_listener.pyi | 2 ++ DrissionPage/session_page.py | 5 +++-- DrissionPage/setter.py | 4 ++-- 10 files changed, 48 insertions(+), 23 deletions(-) diff --git a/DrissionPage/base.py b/DrissionPage/base.py index 33aa5c1..c7b89d8 100644 --- a/DrissionPage/base.py +++ b/DrissionPage/base.py @@ -4,7 +4,6 @@ @Contact : g1879@qq.com """ from abc import abstractmethod -from pathlib import Path from re import sub from urllib.parse import quote @@ -359,16 +358,15 @@ class DrissionElement(BaseElement): class BasePage(BaseParser): """页面类的基类""" - def __init__(self, timeout=None): + def __init__(self): """初始化函数""" self._url = None - self.timeout = timeout if timeout is not None else 10 + self._timeout = 10 self._url_available = None - if not hasattr(self, 'retry_times'): - self.retry_times = 3 - if not hasattr(self, 'retry_interval'): - self.retry_interval = 2 + self.retry_times = 3 + self.retry_interval = 2 self._DownloadKit = None + self._download_path = '' @property def title(self): @@ -399,7 +397,7 @@ class BasePage(BaseParser): @property def download_path(self): """返回默认下载路径""" - return str(Path(self._download_path).absolute()) + return self._download_path @property def download(self): diff --git a/DrissionPage/base.pyi b/DrissionPage/base.pyi index d24d305..6c49719 100644 --- a/DrissionPage/base.pyi +++ b/DrissionPage/base.pyi @@ -157,7 +157,7 @@ class DrissionElement(BaseElement): class BasePage(BaseParser): - def __init__(self, timeout: float = None): + def __init__(self): self._url_available: bool = ... self.retry_times: int = ... self.retry_interval: float = ... diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index 8c18e5e..0264207 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -35,6 +35,7 @@ class ChromiumBase(BasePage): :param tab_id: 要控制的标签页id，不指定默认为激活的 :param timeout: 超时时间 """ + super().__init__() self._is_loading = None self._root_id = None # object id self._debug = False @@ -45,6 +46,7 @@ class ChromiumBase(BasePage): self._listener = None self._wait_download_flag = None self._download_rename = None + self._download_path = '' if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): address = f'127.0.0.1:{address}' @@ -52,8 +54,8 @@ class ChromiumBase(BasePage): self._set_start_options(address, None) self._set_runtime_settings() self._connect_browser(tab_id) - timeout = timeout if timeout is not None else self.timeouts.implicit - super().__init__(timeout) + if timeout is not None: + self.timeout = timeout def _set_start_options(self, address, none): """设置浏览器启动属性 diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/chromium_frame.py index 001d49f..9bd709d 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/chromium_frame.py @@ -3,6 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ +from copy import copy from re import search from threading import Thread from time import sleep, perf_counter @@ -70,10 +71,13 @@ class ChromiumFrame(ChromiumBase): attrs = [f"{attr}='{attrs[attr]}'" for attr in attrs] return f'' - def _runtime_settings(self): + def _set_runtime_settings(self): """重写设置浏览器运行参数方法""" - self._timeouts = self._target_page.timeouts + self._timeouts = copy(self._target_page.timeouts) + self.retry_times = self._target_page.retry_times + self.retry_interval = self._target_page.retry_interval self._page_load_strategy = self._target_page.page_load_strategy + self._download_path = self._target_page.download_path def _driver_init(self, tab_id): """避免出现服务器500错误 @@ -354,7 +358,7 @@ class ChromiumFrame(ChromiumBase): @property def download_path(self): - return self.tab.download_path + return self._download_path def refresh(self): """刷新frame页面""" diff --git a/DrissionPage/chromium_frame.pyi b/DrissionPage/chromium_frame.pyi index 1a798f6..e310dc6 100644 --- a/DrissionPage/chromium_frame.pyi +++ b/DrissionPage/chromium_frame.pyi @@ -39,7 +39,7 @@ class ChromiumFrame(ChromiumBase): def __repr__(self) -> str: ... - def _runtime_settings(self) -> None: ... + def _set_runtime_settings(self) -> None: ... def _driver_init(self, tab_id: str) -> None: ... diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 14bbfbe..db71079 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -26,9 +26,10 @@ class ChromiumPage(ChromiumBase): :param tab_id: 要控制的标签页id，不指定默认为激活的 :param timeout: 超时时间 """ - super().__init__(addr_driver_opts, tab_id, timeout) + super().__init__(addr_driver_opts, tab_id) self._page = self self._dl_mgr = BrowserDownloadManager(self) + self.set.timeouts(implicit=timeout) def _set_start_options(self, addr_driver_opts, none): """设置浏览器启动属性 @@ -62,6 +63,8 @@ class ChromiumPage(ChromiumBase): page_load=self._driver_options.timeouts['pageLoad'], script=self._driver_options.timeouts['script'], implicit=self._driver_options.timeouts['implicit']) + if self._driver_options.timeouts['implicit'] is not None: + self._timeout = self._driver_options.timeouts['implicit'] self._page_load_strategy = self._driver_options.page_load_strategy self._download_path = self._driver_options.download_path @@ -460,9 +463,24 @@ class BrowserDownloadManager(object): def _onDownloadWillBegin(self, **kwargs): """用于获取弹出新标签页触发的下载任务""" - sleep(.1) + sleep(.2) if kwargs['guid'] not in self._missions: - self.add_mission(kwargs['guid'], self._page.download_path, kwargs['suggestedFilename']) + if self._page._wait_download_flag is False: + self._page.run_cdp('Browser.cancelDownload', guid=kwargs['guid']) + + if self._page._download_rename: + tmp = kwargs['suggestedFilename'].rsplit('.', 1) + ext_name = tmp[-1] if len(tmp) > 1 else '' + tmp = self._page._download_rename.rsplit('.', 1) + ext_rename = tmp[-1] if len(tmp) > 1 else '' + n = self._page._download_rename if ext_rename == ext_name else f'{self._page._download_rename}.{ext_name}' + self._download_rename = None + + else: + n = kwargs['suggestedFilename'] + + self._page._dl_mgr.add_mission(kwargs['guid'], self._page.download_path, n) + self._wait_download_flag = {'url': kwargs['url'], 'name': n} def _onDownloadProgress(self, **kwargs): """下载状态变化时执行""" diff --git a/DrissionPage/chromium_tab.py b/DrissionPage/chromium_tab.py index 9f68d5f..a46550b 100644 --- a/DrissionPage/chromium_tab.py +++ b/DrissionPage/chromium_tab.py @@ -25,7 +25,7 @@ class ChromiumTab(ChromiumBase): def _set_runtime_settings(self): """重写设置浏览器运行参数方法""" - self._timeouts = self.page.timeouts + self._timeouts = copy(self.page.timeouts) self.retry_times = self.page.retry_times self.retry_interval = self.page.retry_interval self._page_load_strategy = self.page.page_load_strategy diff --git a/DrissionPage/network_listener.pyi b/DrissionPage/network_listener.pyi index 759f7b2..76c5e72 100644 --- a/DrissionPage/network_listener.pyi +++ b/DrissionPage/network_listener.pyi @@ -28,6 +28,8 @@ class NetworkListener(object): def stop(self) -> None: ... + def wait(self):... + @property def results(self) -> Union[DataPacket, Dict[str, List[DataPacket]], False]: ... diff --git a/DrissionPage/session_page.py b/DrissionPage/session_page.py index 7e07aa7..0931de5 100644 --- a/DrissionPage/session_page.py +++ b/DrissionPage/session_page.py @@ -26,14 +26,15 @@ class SessionPage(BasePage): :param session_or_options: Session对象或SessionOptions对象 :param timeout: 连接超时时间，为None时从ini文件读取 """ + super().__init__() self._response = None self._session = None self._set = None self._set_start_options(session_or_options, None) self._set_runtime_settings() self._create_session() - timeout = timeout if timeout is not None else self.timeout - super().__init__(timeout) + if timeout is not None: + self.timeout = timeout def _set_start_options(self, session_or_options, none): """启动配置 diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index dcfd210..6972713 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -122,7 +122,7 @@ class ChromiumBaseSetter(object): :param path: 下载路径 :return: None """ - self._page._download_path = str(path) + self._page._download_path = str(Path(path).absolute()) if self._page._DownloadKit: self._page._DownloadKit.set.goal_path(path) @@ -193,7 +193,7 @@ class SessionPageSetter(object): :param path: 下载路径 :return: None """ - self._page._download_path = str(path) + self._page._download_path = str(Path(path).absolute()) if self._page._DownloadKit: self._page._DownloadKit.set.goal_path(path) From 0b964121464071d29cd4d880824945d0fffcb6fc Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 30 Aug 2023 17:33:00 +0800 Subject: [PATCH 013/182] =?UTF-8?q?=E5=A2=9E=E5=8A=A0`wait()`=E6=96=B9?= =?UTF-8?q?=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/setter.pyi | 2 ++ DrissionPage/waiter.py | 14 ++++++++++++++ DrissionPage/waiter.pyi | 4 ++++ 3 files changed, 20 insertions(+) diff --git a/DrissionPage/setter.pyi b/DrissionPage/setter.pyi index 7e2748a..043c558 100644 --- a/DrissionPage/setter.pyi +++ b/DrissionPage/setter.pyi @@ -80,6 +80,8 @@ class SessionPageSetter(object): def retry_interval(self, interval: float) -> None: ... + def download_path(self, path: Union[str, Path]) -> None: ... + def timeout(self, second: float) -> None: ... def cookie(self, cookie: Union[Cookie, str, dict]) -> None: ... diff --git a/DrissionPage/waiter.py b/DrissionPage/waiter.py index 93d5d5b..69cba72 100644 --- a/DrissionPage/waiter.py +++ b/DrissionPage/waiter.py @@ -12,6 +12,13 @@ class ChromiumBaseWaiter(object): """ self._driver = page_or_ele + def __call__(self, second): + """等待若干秒 + :param second: 秒数 + :return: None + """ + sleep(second) + def ele_delete(self, loc_or_ele, timeout=None, raise_err=None): """等待元素从DOM中删除 :param loc_or_ele: 要等待的元素，可以是已有元素、定位符 @@ -223,6 +230,13 @@ class ChromiumElementWaiter(object): self._page = page self._ele = ele + def __call__(self, second): + """等待若干秒 + :param second: 秒数 + :return: None + """ + sleep(second) + def delete(self, timeout=None, raise_err=None): """等待元素从dom删除 :param timeout: 超时时间，为None使用元素所在页面timeout属性 diff --git a/DrissionPage/waiter.pyi b/DrissionPage/waiter.pyi index 66bbb43..59f272a 100644 --- a/DrissionPage/waiter.pyi +++ b/DrissionPage/waiter.pyi @@ -15,6 +15,8 @@ class ChromiumBaseWaiter(object): def __init__(self, page: ChromiumBase): self._driver: ChromiumBase = ... + def __call__(self, second: float) -> None: ... + def ele_delete(self, loc_or_ele: Union[str, tuple, ChromiumElement], timeout: float = None, raise_err: bool = None) -> bool: ... @@ -60,6 +62,8 @@ class ChromiumElementWaiter(object): self._ele: ChromiumElement = ... self._page: ChromiumBase = ... + def __call__(self, second: float) -> None: ... + def delete(self, timeout: float = None, raise_err: bool = None) -> bool: ... def display(self, timeout: float = None, raise_err: bool = None) -> bool: ... From f7368c64e54ba66f9ad59338b5d2df1d3ca2cee9 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 30 Aug 2023 20:23:34 +0800 Subject: [PATCH 014/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=85=83=E7=B4=A0?= =?UTF-8?q?=E6=88=AA=E5=9B=BE=E6=97=B6=E7=AA=97=E5=8F=A3=E5=A4=96=E9=83=A8?= =?UTF-8?q?=E5=88=86=E7=A9=BA=E7=99=BD=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_base.py | 2 +- DrissionPage/chromium_element.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index 0264207..c2c9400 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -904,7 +904,7 @@ class ChromiumBase(BasePage): h = right_bottom[1] - y vp = {'x': x, 'y': y, 'width': w, 'height': h, 'scale': 1} png = self.run_cdp_loaded('Page.captureScreenshot', format=pic_type, - captureBeyondViewport=False, clip=vp)['data'] + captureBeyondViewport=True, clip=vp)['data'] else: png = self.run_cdp_loaded('Page.captureScreenshot', format=pic_type)['data'] diff --git a/DrissionPage/chromium_element.py b/DrissionPage/chromium_element.py index afcb6d4..10ac389 100644 --- a/DrissionPage/chromium_element.py +++ b/DrissionPage/chromium_element.py @@ -518,8 +518,8 @@ class ChromiumElement(DrissionElement): while not self.run_js(js) and perf_counter() < end_time: sleep(.1) - self.scroll.to_see(center=True) - sleep(1) + # self.scroll.to_see(center=True) + # sleep(1) left, top = self.location width, height = self.size left_top = (left, top) From 1958ad283eebeced6b342f93e60ab3eeec80635c Mon Sep 17 00:00:00 2001 From: donggoing <374766849@qq.com> Date: Thu, 31 Aug 2023 00:22:45 +0800 Subject: [PATCH 015/182] =?UTF-8?q?=E6=B7=BB=E5=8A=A0wait=5Funtil=E6=96=B9?= =?UTF-8?q?=E6=B3=95=EF=BC=8C=E6=94=AF=E6=8C=81=E8=87=AA=E5=AE=9A=E4=B9=89?= =?UTF-8?q?=E7=BB=84=E5=90=88=E7=AD=89=E5=BE=85=E6=9D=A1=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/commons/tools.py | 37 ++++++++++++++++++++++++++++++++++ DrissionPage/commons/tools.pyi | 3 +++ 2 files changed, 40 insertions(+) diff --git a/DrissionPage/commons/tools.py b/DrissionPage/commons/tools.py index dfdab22..a107c2b 100644 --- a/DrissionPage/commons/tools.py +++ b/DrissionPage/commons/tools.py @@ -7,6 +7,7 @@ from platform import system from pathlib import Path from re import search, sub from shutil import rmtree +from time import perf_counter, sleep def get_usable_path(path): @@ -177,6 +178,42 @@ def get_chrome_hwnds_from_pid(pid, title): EnumWindows(callback, hwnds) return hwnds +def wait_until(page, condition, timeout=10, poll=0.1, raise_err=True): + """等待返回值不为False或空，直到超时 + :param page (DrissionPage): DrissionPage对象 + :param condition (function | str | tuple): 等待条件，返回值不为False则停止等待 + :param timeout (float, optional): 超时时间 + :param poll (float, optional): 轮询间隔 + :param message (str, optional): 超时时的报错信息 + :param ignored_exceptions (bool, optional): 是否忽略异常 + :return: DP Element or bool + """ + end_time = perf_counter() + timeout + if isinstance(condition, str) or isinstance(condition, tuple): + if not callable(getattr(page, 's_ele', None)): + raise AttributeError('page对象缺少s_ele方法') + condition_method = lambda page: page.s_ele(condition) + elif callable(condition): + condition_method = condition + else: + raise ValueError('condition必须是函数或者字符串或者元组') + while perf_counter() < end_time: + try: + value = condition_method(page) + if value: + return value + except Exception as exc: + pass + + sleep(poll) + if perf_counter() > end_time: + break + + if raise_err: + raise TimeoutError('等待超时') + else: + return False + # def get_exe_from_port(port): # """获取端口号第一条进程的可执行文件路径 # :param port: 端口号 diff --git a/DrissionPage/commons/tools.pyi b/DrissionPage/commons/tools.pyi index 54b8197..1781bc6 100644 --- a/DrissionPage/commons/tools.pyi +++ b/DrissionPage/commons/tools.pyi @@ -6,6 +6,7 @@ from os import popen from pathlib import Path from typing import Union +from types import FunctionType from chromium_page import ChromiumPage @@ -38,3 +39,5 @@ def get_browser_progress_id(progress: Union[popen, None], address: str) -> Union def get_chrome_hwnds_from_pid(pid: Union[str, int], title: str) -> list: ... + +def wait_until(page, condition: Union[FunctionType, str, tuple], timeout: float, poll: float, raise_err: bool): ... From 95717981c88a2fc7f3feb89e42fcaa2001f9ca1d Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 6 Sep 2023 17:02:15 +0800 Subject: [PATCH 016/182] =?UTF-8?q?=E6=97=A0=E7=95=8C=E9=9D=A2Linux?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=90=AF=E7=94=A8=E6=97=A0=E5=A4=B4=EF=BC=9B?= =?UTF-8?q?MAC=E5=92=8CLinux=E6=B7=BB=E5=8A=A0=E9=BB=98=E8=AE=A4=E6=B5=8F?= =?UTF-8?q?=E8=A7=88=E5=99=A8=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/commons/browser.py | 12 +++++++++++- DrissionPage/easy_set.py | 16 +++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/DrissionPage/commons/browser.py b/DrissionPage/commons/browser.py index 339091f..c0d42d0 100644 --- a/DrissionPage/commons/browser.py +++ b/DrissionPage/commons/browser.py @@ -8,6 +8,7 @@ from pathlib import Path from subprocess import Popen, DEVNULL from tempfile import gettempdir from time import perf_counter, sleep +from platform import system from requests import get as requests_get @@ -62,6 +63,7 @@ def get_launch_args(opt): result = set() has_user_path = False remote_allow = False + headless = False for i in opt.arguments: if i.startswith(('--load-extension=', '--remote-debugging-port=')): continue @@ -71,6 +73,8 @@ def get_launch_args(opt): continue elif i.startswith('--remote-allow-origins='): remote_allow = True + elif i.startswith('--headless'): + headless = True result.add(i) @@ -83,6 +87,12 @@ def get_launch_args(opt): if not remote_allow: result.add('--remote-allow-origins=*') + if not headless and system().lower() == 'linux': + from os import popen + r = popen('systemctl list-units | grep graphical.target') + if 'graphical.target' not in r.read(): + result.add('--headless=new') + result = list(result) # ----------处理插件extensions------------- @@ -164,7 +174,7 @@ def test_connect(ip, port): def _run_browser(port, path: str, args) -> Popen: """创建chrome进程 :param port: 端口号 - :param path: 浏览器地址 + :param path: 浏览器路径 :param args: 启动参数 :return: 进程对象 """ diff --git a/DrissionPage/easy_set.py b/DrissionPage/easy_set.py index d783c5b..becea0f 100644 --- a/DrissionPage/easy_set.py +++ b/DrissionPage/easy_set.py @@ -197,7 +197,19 @@ def get_chrome_path(ini_path=None, return str(path) from platform import system - if system().lower() != 'windows': + sys = system().lower() + if sys == 'macos': + return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' + + elif sys == 'linux': + paths = ('/usr/bin/google-chrome', '/opt/google/chrome/google-chrome', + '/user/lib/chromium-browser/chromium-browser') + for p in paths: + if Path(p).exists(): + return p + return None + + elif sys != 'windows': return None # -----------从注册表中获取-------------- @@ -207,8 +219,6 @@ def get_chrome_path(ini_path=None, key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\chrome.exe', reserved=0, access=winreg.KEY_READ) - # key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'Software\Google\Chrome\BLBeacon\version', - # reserved=0, access=winreg.KEY_READ) k = winreg.EnumValue(key, 0) winreg.CloseKey(key) From e5055decd8e9ed0ce2683bef23832814f829ebf5 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 7 Sep 2023 18:00:11 +0800 Subject: [PATCH 017/182] =?UTF-8?q?=E5=9F=BA=E6=9C=AC=E5=AE=8C=E6=88=90?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E5=8A=9F=E8=83=BD=E5=AE=8C=E5=96=84=EF=BC=8C?= =?UTF-8?q?=E5=BE=85=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_base.py | 53 ++++++++---- DrissionPage/chromium_base.pyi | 5 ++ DrissionPage/chromium_element.py | 14 ++-- DrissionPage/chromium_frame.py | 1 + DrissionPage/chromium_page.py | 71 ++++++++-------- DrissionPage/chromium_page.pyi | 13 ++- DrissionPage/chromium_tab.py | 3 +- DrissionPage/commons/tools.py | 16 ++-- DrissionPage/commons/tools.pyi | 1 + DrissionPage/setter.py | 5 ++ DrissionPage/setter.pyi | 2 + DrissionPage/waiter.py | 134 +++++++++++++++++++++++++++++-- DrissionPage/waiter.pyi | 31 ++++++- 13 files changed, 270 insertions(+), 79 deletions(-) diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index c2c9400..70b4cf7 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -23,7 +23,7 @@ from .errors import ContextLossError, ElementLossError, AlertExistsError, CDPErr from .network_listener import NetworkListener from .session_element import make_session_ele from .setter import ChromiumBaseSetter -from .waiter import ChromiumBaseWaiter +from .waiter import ChromiumBaseWaiter, DownloadMission class ChromiumBase(BasePage): @@ -44,9 +44,12 @@ class ChromiumBase(BasePage): self._set = None self._screencast = None self._listener = None + self._wait_download_flag = None self._download_rename = None self._download_path = '' + self._when_download_file_exists = 'rename' + self._download_missions = set() if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): address = f'127.0.0.1:{address}' @@ -249,22 +252,7 @@ class ChromiumBase(BasePage): def _onDownloadWillBegin(self, **kwargs): """下载即将开始时执行""" - if self._wait_download_flag is False: - self._page.run_cdp('Browser.cancelDownload', guid=kwargs['guid']) - - if self._download_rename: - tmp = kwargs['suggestedFilename'].rsplit('.', 1) - ext_name = tmp[-1] if len(tmp) > 1 else '' - tmp = self._download_rename.rsplit('.', 1) - ext_rename = tmp[-1] if len(tmp) > 1 else '' - n = self._download_rename if ext_rename == ext_name else f'{self._download_rename}.{ext_name}' - self._download_rename = None - - else: - n = kwargs['suggestedFilename'] - - self._page._dl_mgr.add_mission(kwargs['guid'], self.download_path, n) - self._wait_download_flag = {'url': kwargs['url'], 'name': n} + handle_download(self, kwargs) def __call__(self, loc_or_str, timeout=None): """在内部查找元素 @@ -1141,3 +1129,34 @@ class ScreencastMode(object): def imgs_mode(self): self._screencast._mode = 'imgs' + + +def handle_download(tab, kwargs): + """在下载开始前处理任务 + :param tab: 触发任务的tab对象 + :param kwargs: 浏览器返回的数据 + :return: None + """ + tab._page._dl_mgr._missions[kwargs['guid']] = None + + if tab._download_rename: + tmp = kwargs['suggestedFilename'].rsplit('.', 1) + ext_name = tmp[-1] if len(tmp) > 1 else '' + tmp = tab._download_rename.rsplit('.', 1) + ext_rename = tmp[-1] if len(tmp) > 1 else '' + n = tab._download_rename if ext_rename == ext_name else f'{tab._download_rename}.{ext_name}' + tab._download_rename = None + + else: + n = kwargs['suggestedFilename'] + + m = DownloadMission(tab, kwargs['guid'], tab.download_path, n, kwargs['url']) + tab._page._dl_mgr.add_mission(m) + tab._wait_download_flag = m + tab._download_missions.add(m) + + if tab._wait_download_flag is False: # 取消该任务 + m._set_done('canceled', True) + + if tab._when_download_file_exists == 'skip' and (Path(m.path) / m.name).exists(): + m._set_done('skipped', True) diff --git a/DrissionPage/chromium_base.pyi b/DrissionPage/chromium_base.pyi index 03d102f..ac9ff03 100644 --- a/DrissionPage/chromium_base.pyi +++ b/DrissionPage/chromium_base.pyi @@ -47,6 +47,8 @@ class ChromiumBase(BasePage): self._listener: NetworkListener = ... self._wait_download_flag: bool = ... self._download_rename: str = ... + self._when_download_file_exists: str = ... + self._download_missions: set = ... def _connect_browser(self, tab_id: str = None) -> None: ... @@ -275,3 +277,6 @@ class ScreencastMode(object): def frugal_imgs_mode(self) -> None: ... def imgs_mode(self) -> None: ... + + +def handle_download(tab: ChromiumBase, kwargs: dict) -> None: ... diff --git a/DrissionPage/chromium_element.py b/DrissionPage/chromium_element.py index 10ac389..ca70063 100644 --- a/DrissionPage/chromium_element.py +++ b/DrissionPage/chromium_element.py @@ -12,6 +12,7 @@ from .base import DrissionElement, BaseElement from .commons.constants import FRAME_ELEMENT, NoneElement, Settings from .commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions from .commons.locator import get_loc +from .commons.tools import make_valid_name from .commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, location_in_viewport, offset_scroll from .errors import ContextLossError, ElementLossError, JavaScriptError, NoRectError, ElementNotFoundError, \ CDPError, NoResourceError, CanNotClickError @@ -474,15 +475,11 @@ class ChromiumElement(DrissionElement): if not result: return None - if result['base64Encoded']: - if base64_to_bytes: - from base64 import b64decode - data = b64decode(result['content']) - else: - data = result['content'] + if result['base64Encoded'] and base64_to_bytes: + from base64 import b64decode + return b64decode(result['content']) else: - data = result['content'] - return data + return result['content'] def save(self, path=None, rename=None, timeout=None): """保存图片或其它有src属性的元素的资源 @@ -497,6 +494,7 @@ class ChromiumElement(DrissionElement): path = path or '.' rename = rename or basename(self.prop('currentSrc')) + rename = make_valid_name(rename) write_type = 'wb' if isinstance(data, bytes) else 'w' Path(path).mkdir(parents=True, exist_ok=True) diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/chromium_frame.py index 9bd709d..e4ed12a 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/chromium_frame.py @@ -78,6 +78,7 @@ class ChromiumFrame(ChromiumBase): self.retry_interval = self._target_page.retry_interval self._page_load_strategy = self._target_page.page_load_strategy self._download_path = self._target_page.download_path + self._when_download_file_exists = self._target_page._when_download_file_exists def _driver_init(self, tab_id): """避免出现服务器500错误 diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index db71079..4901f64 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -7,6 +7,7 @@ from shutil import move from time import perf_counter, sleep from .chromium_base import ChromiumBase, Timeout +from .chromium_base import handle_download from .chromium_driver import ChromiumDriver from .chromium_tab import ChromiumTab from .commons.browser import connect_browser @@ -450,50 +451,56 @@ class BrowserDownloadManager(object): page.set.download_path(page.download_path) self._page.browser_driver.set_listener('Browser.downloadProgress', self._onDownloadProgress) self._page.browser_driver.set_listener('Browser.downloadWillBegin', self._onDownloadWillBegin) - self._missions = {} + self._missions = set() - def add_mission(self, guid, path, name): + @property + def missions(self): + return self._missions + + def add_mission(self, mission): """添加下载任务信息 - :param guid: guid - :param path: 保存路径 - :param name: 保存文件名 + :param mission: DownloadMission对象 :return: None """ - self._missions[guid] = {'path': path, 'name': name} + self._missions.add(mission) + + def cancel(self, mission): + """取消一个下载任务 + :param mission: 任务对象 + :return: None + """ + self._page.browser_driver.call_method('Browser.cancelDownload', guid=mission.id) + self._missions.remove(mission) def _onDownloadWillBegin(self, **kwargs): """用于获取弹出新标签页触发的下载任务""" - sleep(.2) + sleep(.3) if kwargs['guid'] not in self._missions: - if self._page._wait_download_flag is False: - self._page.run_cdp('Browser.cancelDownload', guid=kwargs['guid']) - - if self._page._download_rename: - tmp = kwargs['suggestedFilename'].rsplit('.', 1) - ext_name = tmp[-1] if len(tmp) > 1 else '' - tmp = self._page._download_rename.rsplit('.', 1) - ext_rename = tmp[-1] if len(tmp) > 1 else '' - n = self._page._download_rename if ext_rename == ext_name else f'{self._page._download_rename}.{ext_name}' - self._download_rename = None - - else: - n = kwargs['suggestedFilename'] - - self._page._dl_mgr.add_mission(kwargs['guid'], self._page.download_path, n) - self._wait_download_flag = {'url': kwargs['url'], 'name': n} + handle_download(self._page, kwargs) def _onDownloadProgress(self, **kwargs): """下载状态变化时执行""" - if kwargs['state'] in ('completed', 'canceled') and kwargs['guid'] in self._missions: - guid = kwargs['guid'] - if kwargs['state'] == 'completed': - path = self._missions[guid]['path'] - name = self._missions[guid]['name'] - form_path = f'{self._page.download_path}\\{guid}' - to_path = get_usable_path(f'{path}\\{name}') - move(form_path, to_path) + if kwargs['guid'] in self._missions: + mission = self._missions[kwargs['guid']] + # print(mission) + if kwargs['state'] == 'inProgress': + mission.state = 'running' + mission.received_bytes = kwargs['receivedBytes'] + mission.total_bytes = kwargs['totalBytes'] - self._missions.pop(guid) + elif kwargs['state'] == 'completed': + mission.received_bytes = kwargs['receivedBytes'] + mission.total_bytes = kwargs['totalBytes'] + form_path = f'{self._page.download_path}\\{mission.id}' + to_path = get_usable_path(f'{mission.path}\\{mission.name}') + move(form_path, to_path) + mission.final_path = to_path + mission.state = 'completed' + self._missions.pop(mission.id) + + else: + mission.state = 'canceled' + self._missions.pop(mission.id) class Alert(object): diff --git a/DrissionPage/chromium_page.pyi b/DrissionPage/chromium_page.pyi index 3dab20f..52b6110 100644 --- a/DrissionPage/chromium_page.pyi +++ b/DrissionPage/chromium_page.pyi @@ -3,14 +3,14 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union, Tuple, List +from typing import Union, Tuple, List, Dict, Optional, Set from .chromium_base import ChromiumBase from .chromium_driver import ChromiumDriver from .chromium_tab import ChromiumTab from .configs.chromium_options import ChromiumOptions from .setter import ChromiumPageSetter -from .waiter import ChromiumPageWaiter +from .waiter import ChromiumPageWaiter, DownloadMission class ChromiumPage(ChromiumBase): @@ -127,11 +127,16 @@ class ChromiumTabRect(object): class BrowserDownloadManager(object): _page: ChromiumPage = ... - _missions: dict = ... + _missions: Set[DownloadMission] = ... def __init__(self, page: ChromiumPage): ... - def add_mission(self, guid: str, path: str, name: str) -> None: ... + @property + def missions(self) -> Set[DownloadMission]: ... + + def add_mission(self, mission: DownloadMission) -> None: ... + + def cancel(self, mission: DownloadMission) -> None: ... def _onDownloadWillBegin(self, **kwargs) -> None: ... diff --git a/DrissionPage/chromium_tab.py b/DrissionPage/chromium_tab.py index a46550b..c5ca0a4 100644 --- a/DrissionPage/chromium_tab.py +++ b/DrissionPage/chromium_tab.py @@ -30,6 +30,7 @@ class ChromiumTab(ChromiumBase): self.retry_interval = self.page.retry_interval self._page_load_strategy = self.page.page_load_strategy self._download_path = self.page.download_path + self._when_download_file_exists = self.page._when_download_file_exists def close(self): """关闭当前标签页""" @@ -59,7 +60,7 @@ class WebPageTab(SessionPage, ChromiumTab): :param page: WebPage对象 :param tab_id: 要控制的标签页id """ - self.page = page + self._page = page self.address = page.address self._debug = page._debug self._debug_recorder = page._debug_recorder diff --git a/DrissionPage/commons/tools.py b/DrissionPage/commons/tools.py index a107c2b..e360688 100644 --- a/DrissionPage/commons/tools.py +++ b/DrissionPage/commons/tools.py @@ -178,14 +178,14 @@ def get_chrome_hwnds_from_pid(pid, title): EnumWindows(callback, hwnds) return hwnds + def wait_until(page, condition, timeout=10, poll=0.1, raise_err=True): """等待返回值不为False或空，直到超时 - :param page (DrissionPage): DrissionPage对象 - :param condition (function | str | tuple): 等待条件，返回值不为False则停止等待 - :param timeout (float, optional): 超时时间 - :param poll (float, optional): 轮询间隔 - :param message (str, optional): 超时时的报错信息 - :param ignored_exceptions (bool, optional): 是否忽略异常 + :param page: DrissionPage对象 + :param condition: 等待条件，返回值不为False则停止等待 + :param timeout: 超时时间 + :param poll: 轮询间隔 + :param raise_err: 是否抛出异常 :return: DP Element or bool """ end_time = perf_counter() + timeout @@ -204,11 +204,11 @@ def wait_until(page, condition, timeout=10, poll=0.1, raise_err=True): return value except Exception as exc: pass - + sleep(poll) if perf_counter() > end_time: break - + if raise_err: raise TimeoutError('等待超时') else: diff --git a/DrissionPage/commons/tools.pyi b/DrissionPage/commons/tools.pyi index 1781bc6..d7ed7f7 100644 --- a/DrissionPage/commons/tools.pyi +++ b/DrissionPage/commons/tools.pyi @@ -40,4 +40,5 @@ def get_browser_progress_id(progress: Union[popen, None], address: str) -> Union def get_chrome_hwnds_from_pid(pid: Union[str, int], title: str) -> list: ... + def wait_until(page, condition: Union[FunctionType, str, tuple], timeout: float, poll: float, raise_err: bool): ... diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index 6972713..abc6194 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -133,6 +133,11 @@ class ChromiumBaseSetter(object): """ self._page._download_rename = name + def when_download_file_exists(self, mode): + if mode not in ('rename', 'overwrite', 'skip'): + raise ValueError(f"mode参数只能是'rename', 'overwrite', 'skip' 之一，现在是：{mode}") + self._page._when_download_file_exists = mode + class TabSetter(ChromiumBaseSetter): def __init__(self, page): diff --git a/DrissionPage/setter.pyi b/DrissionPage/setter.pyi index 043c558..39981d2 100644 --- a/DrissionPage/setter.pyi +++ b/DrissionPage/setter.pyi @@ -54,6 +54,8 @@ class ChromiumBaseSetter(object): def download_file_name(self, name: str) -> None: ... + def when_download_file_exists(self, mode: str) -> None: ... + class TabSetter(ChromiumBaseSetter): def __init__(self, page): ... diff --git a/DrissionPage/waiter.py b/DrissionPage/waiter.py index 69cba72..1eae5a1 100644 --- a/DrissionPage/waiter.py +++ b/DrissionPage/waiter.py @@ -1,4 +1,5 @@ # -*- coding:utf-8 -*- +from pathlib import Path from time import sleep, perf_counter from .commons.constants import Settings @@ -85,13 +86,13 @@ class ChromiumBaseWaiter(object): while self._driver._upload_list: sleep(.01) - def browser_download_begin(self, timeout=None, cancel=False): + def download_begin(self, timeout=None, cancel_it=False): """等待浏览器下载开始，可将其拦截 :param timeout: 超时时间，None使用页面对象超时时间 - :param cancel: 是否取消该任务 + :param cancel_it: 是否取消该任务 :return: 成功返回任务信息dict，失败返回False """ - self._driver._wait_download_flag = False if cancel else True + self._driver._wait_download_flag = False if cancel_it else True if timeout is None: timeout = self._driver.timeout @@ -105,6 +106,32 @@ class ChromiumBaseWaiter(object): self._driver._wait_download_flag = None return r + def downloads_done(self, timeout=None, cancel_if_timeout=True): + """等待所有浏览器下载任务结束 + :param timeout: 超时时间，为None时无限等待 + :param cancel_if_timeout: 超时时是否取消剩余任务 + :return: 是否等待成功 + """ + if not timeout: + while self._driver._download_missions: + sleep(.5) + return True + + else: + end_time = perf_counter() + timeout + while end_time > perf_counter(): + if not self._driver._download_missions: + return True + sleep(.5) + + if self._driver._download_missions: + if cancel_if_timeout: + for m in self._driver._download_missions: + m.cancel() + return False + else: + return True + def url_change(self, text, exclude=False, timeout=None, raise_err=None): """等待url变成包含或不包含指定文本 :param text: 用于识别的文本 @@ -200,9 +227,10 @@ class ChromiumPageWaiter(ChromiumBaseWaiter): else: return False - def browser_downloads_complete(self, timeout=None): - """等待所有下载任务结束 + def all_downloads_done(self, timeout=None, cancel_if_timeout=True): + """等待所有浏览器下载任务结束 :param timeout: 超时时间，为None时无限等待 + :param cancel_if_timeout: 超时时是否取消剩余任务 :return: 是否等待成功 """ if not timeout: @@ -216,7 +244,14 @@ class ChromiumPageWaiter(ChromiumBaseWaiter): if not self._driver._dl_mgr._missions: return True sleep(.5) - return False if self._driver._dl_mgr._missions else True + + if self._driver._dl_mgr._missions: + if cancel_if_timeout: + for m in self._driver._dl_mgr._missions: + m.cancel() + return False + else: + return True class ChromiumElementWaiter(object): @@ -341,3 +376,90 @@ class FrameWaiter(ChromiumBaseWaiter, ChromiumElementWaiter): """ super().__init__(frame) super(ChromiumBaseWaiter, self).__init__(frame, frame.frame_ele) + + +class DownloadMission(object): + def __init__(self, tab, _id, path, name, url): + self.url = url + self.tab = tab + self.id = _id + self.path = path + self.name = name + self.state = 'waiting' + self.total_bytes = None + self.received_bytes = 0 + self.final_path = None + + def __repr__(self): + # return f'' + return f'' + + @property + def rate(self): + """以百分比形式返回下载进度""" + return round((self.received_bytes / self.total_bytes) * 100, 2) if self.total_bytes else None + + def cancel(self): + """取消该任务，如任务已完成，删除已下载的文件""" + self._set_done('canceled', True) + if self.final_path: + Path(self.final_path).unlink(True) + + def wait(self, show=True, timeout=None, cancel_if_timeout=True): + """等待任务结束 + :param show: 是否显示下载信息 + :param timeout: 超时时间，为None则无限等待 + :param cancel_if_timeout: 超时时是否取消任务 + :return: 等待成功返回完整路径，否则返回False + """ + if show: + print(f'url：{self.url}') + t2 = perf_counter() + while self.name is None and perf_counter() - t2 < 4: + sleep(0.01) + print(f'文件名：{self.name}') + print(f'目标路径：{self.path}') + + if timeout is None: + while self.id in self.tab._page._dl_mgr.missions: + if show: + print(f'\r{self.rate}% ', end='') + sleep(.2) + + else: + running = True + end_time = perf_counter() + timeout + while perf_counter() < end_time: + if show: + print(f'\r{self.rate}% ', end='') + if self.id not in self.tab._page._dl_mgr.missions: + running = False + break + sleep(.2) + + if running and cancel_if_timeout: + self.cancel() + + if show: + if self.state == 'completed': + print(f'下载完成 {self.final_path}') + elif self.state == 'canceled': + print(f'下载取消') + elif self.state == 'skipped': + print(f'已跳过') + print() + + return self.final_path if self.final_path else False + + def _set_done(self, state, cancel=False, final_path=None): + """设置任务结束 + :param state: 任务状态 + :param cancel: 是否取消 + :param final_path: 最终路径 + :return: None + """ + self.state = state + self.final_path = final_path + if cancel: + self.tab._page._dl_mgr.cancel(self) + self.tab._download_missions.remove(self) diff --git a/DrissionPage/waiter.pyi b/DrissionPage/waiter.pyi index 59f272a..7858606 100644 --- a/DrissionPage/waiter.pyi +++ b/DrissionPage/waiter.pyi @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union +from typing import Union, Optional from .chromium_base import ChromiumBase from .chromium_element import ChromiumElement @@ -37,7 +37,9 @@ class ChromiumBaseWaiter(object): def upload_paths_inputted(self) -> None: ... - def browser_download_begin(self, timeout: float = None, cancel: bool = False) -> Union[dict, bool]: ... + def download_begin(self, timeout: float = None, cancel_it: bool = False) -> Union[DownloadMission, bool]: ... + + def downloads_done(self, timeout: float = None, cancel_if_timeout: bool = True) -> bool: ... def url_change(self, text: str, exclude: bool = False, timeout: float = None, raise_err: bool = None) -> bool: ... @@ -52,7 +54,7 @@ class ChromiumPageWaiter(ChromiumBaseWaiter): def new_tab(self, timeout: float = None, raise_err: bool = None) -> bool: ... - def browser_downloads_complete(self, timeout: float = None) -> bool: ... + def all_downloads_done(self, timeout: float = None, cancel_if_timeout: bool = True) -> bool: ... class ChromiumElementWaiter(object): @@ -85,3 +87,26 @@ class ChromiumElementWaiter(object): class FrameWaiter(ChromiumBaseWaiter, ChromiumElementWaiter): def __init__(self, frame: ChromiumFrame): ... + + +class DownloadMission(object): + tab: ChromiumBase = ... + url: str = ... + id: str = ... + path: str = ... + name: str = ... + state: str = ... + total_bytes: Optional[int] = ... + received_bytes: int = ... + final_path: Optional[str] = ... + + def __init__(self, tab: ChromiumBase, _id: str, path: str, name: str, url: str): ... + + @property + def rate(self) -> float: ... + + def cancel(self) -> None: ... + + def wait(self, show: bool = True, timeout=None, cancel_if_timeout=True) -> Union[bool, str]: ... + + def _set_done(self, state: str, cancel: bool = False, final_path: str = None) -> None: ... From 7394ab8059da9de7495ef1f9b4d31eb39eeb354e Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 8 Sep 2023 00:36:04 +0800 Subject: [PATCH 018/182] =?UTF-8?q?=E5=AE=8C=E5=96=84=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_page.py | 16 ++++++---------- DrissionPage/chromium_page.pyi | 6 +++--- DrissionPage/waiter.py | 2 +- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 4901f64..8009d16 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -451,7 +451,7 @@ class BrowserDownloadManager(object): page.set.download_path(page.download_path) self._page.browser_driver.set_listener('Browser.downloadProgress', self._onDownloadProgress) self._page.browser_driver.set_listener('Browser.downloadWillBegin', self._onDownloadWillBegin) - self._missions = set() + self._missions = {} @property def missions(self): @@ -462,7 +462,7 @@ class BrowserDownloadManager(object): :param mission: DownloadMission对象 :return: None """ - self._missions.add(mission) + self._missions[mission.id] = mission def cancel(self, mission): """取消一个下载任务 @@ -470,7 +470,7 @@ class BrowserDownloadManager(object): :return: None """ self._page.browser_driver.call_method('Browser.cancelDownload', guid=mission.id) - self._missions.remove(mission) + self._missions.pop(mission.id) def _onDownloadWillBegin(self, **kwargs): """用于获取弹出新标签页触发的下载任务""" @@ -482,7 +482,6 @@ class BrowserDownloadManager(object): """下载状态变化时执行""" if kwargs['guid'] in self._missions: mission = self._missions[kwargs['guid']] - # print(mission) if kwargs['state'] == 'inProgress': mission.state = 'running' mission.received_bytes = kwargs['receivedBytes'] @@ -492,15 +491,12 @@ class BrowserDownloadManager(object): mission.received_bytes = kwargs['receivedBytes'] mission.total_bytes = kwargs['totalBytes'] form_path = f'{self._page.download_path}\\{mission.id}' - to_path = get_usable_path(f'{mission.path}\\{mission.name}') + to_path = str(get_usable_path(f'{mission.path}\\{mission.name}')) move(form_path, to_path) - mission.final_path = to_path - mission.state = 'completed' - self._missions.pop(mission.id) + mission._set_done('completed', final_path=to_path) else: - mission.state = 'canceled' - self._missions.pop(mission.id) + mission._set_done('canceled') class Alert(object): diff --git a/DrissionPage/chromium_page.pyi b/DrissionPage/chromium_page.pyi index 52b6110..be96790 100644 --- a/DrissionPage/chromium_page.pyi +++ b/DrissionPage/chromium_page.pyi @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union, Tuple, List, Dict, Optional, Set +from typing import Union, Tuple, List, Dict from .chromium_base import ChromiumBase from .chromium_driver import ChromiumDriver @@ -127,12 +127,12 @@ class ChromiumTabRect(object): class BrowserDownloadManager(object): _page: ChromiumPage = ... - _missions: Set[DownloadMission] = ... + _missions: Dict[str, DownloadMission] = ... def __init__(self, page: ChromiumPage): ... @property - def missions(self) -> Set[DownloadMission]: ... + def missions(self) -> Dict[str, DownloadMission]: ... def add_mission(self, mission: DownloadMission) -> None: ... diff --git a/DrissionPage/waiter.py b/DrissionPage/waiter.py index 1eae5a1..c8f73a4 100644 --- a/DrissionPage/waiter.py +++ b/DrissionPage/waiter.py @@ -247,7 +247,7 @@ class ChromiumPageWaiter(ChromiumBaseWaiter): if self._driver._dl_mgr._missions: if cancel_if_timeout: - for m in self._driver._dl_mgr._missions: + for m in self._driver._dl_mgr._missions.values(): m.cancel() return False else: From e9d499bcb8366b07f54f13c2caf1ca752ce6689a Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 8 Sep 2023 16:22:17 +0800 Subject: [PATCH 019/182] =?UTF-8?q?ele.save()=E5=A2=9E=E5=8A=A0name?= =?UTF-8?q?=E5=8F=82=E6=95=B0=EF=BC=8C=E8=BF=94=E5=9B=9E=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=EF=BC=9Bget=5Fscreenshot()=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?name=E5=92=8Cscroll=5Fto=5Fcenter=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_base.py | 31 ++++++++++++----------- DrissionPage/chromium_base.pyi | 4 +-- DrissionPage/chromium_element.py | 42 ++++++++++++++++--------------- DrissionPage/chromium_element.pyi | 6 ++--- DrissionPage/chromium_frame.py | 33 +++++++++++++----------- DrissionPage/chromium_frame.pyi | 4 +-- 6 files changed, 65 insertions(+), 55 deletions(-) diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index 70b4cf7..93b5c8a 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -5,7 +5,7 @@ """ from base64 import b64decode from json import loads, JSONDecodeError -from os import sep +from os.path import sep from pathlib import Path from threading import Thread from time import perf_counter, sleep, time @@ -768,10 +768,11 @@ class ChromiumBase(BasePage): ''' return {i['key']: i['val'] for i in self.run_js_loaded(js)} - def get_screenshot(self, path=None, as_bytes=None, as_base64=None, + def get_screenshot(self, path=None, name=None, as_bytes=None, as_base64=None, full_page=False, left_top=None, right_bottom=None): """对页面进行截图，可对整个网页、可见网页、指定范围截图。对可视范围外截图需要90以上版本浏览器支持 - :param path: 完整路径，后缀可选 'jpg','jpeg','png','webp' + :param path: 保存路径 + :param name: 完整文件名，后缀可选 'jpg','jpeg','png','webp' :param as_bytes: 是否以字节形式返回图片，可选 'jpg','jpeg','png','webp'，生效时path参数和as_base64参数无效 :param as_base64: 是否以base64字符串形式返回图片，可选 'jpg','jpeg','png','webp'，生效时path参数无效 :param full_page: 是否整页截图，为True截取整个网页，为False截取可视窗口 @@ -779,7 +780,7 @@ class ChromiumBase(BasePage): :param right_bottom: 截取范围右下角角坐标 :return: 图片完整路径或字节文本 """ - return self._get_screenshot(path=path, as_bytes=as_bytes, as_base64=as_base64, + return self._get_screenshot(path=path, name=name, as_bytes=as_bytes, as_base64=as_base64, full_page=full_page, left_top=left_top, right_bottom=right_bottom) def clear_cache(self, session_storage=True, local_storage=True, cache=True, cookies=True): @@ -843,10 +844,11 @@ class ChromiumBase(BasePage): return True - def _get_screenshot(self, path=None, as_bytes=None, as_base64=None, + def _get_screenshot(self, path=None, name=None, as_bytes=None, as_base64=None, full_page=False, left_top=None, right_bottom=None, ele=None): """对页面进行截图，可对整个网页、可见网页、指定范围截图。对可视范围外截图需要90以上版本浏览器支持 - :param path: 完整路径，后缀可选 'jpg','jpeg','png','webp' + :param path: 保存路径 + :param name: 完整文件名，后缀可选 'jpg','jpeg','png','webp' :param as_bytes: 是否以字节形式返回图片，可选 'jpg','jpeg','png','webp'，生效时path参数和as_base64参数无效 :param as_base64: 是否以base64字符串形式返回图片，可选 'jpg','jpeg','png','webp'，生效时path参数无效 :param full_page: 是否整页截图，为True截取整个网页，为False截取可视窗口 @@ -860,7 +862,7 @@ class ChromiumBase(BasePage): pic_type = 'png' else: if as_bytes not in ('jpg', 'jpeg', 'png', 'webp'): - raise ValueError("只能接收 'jpg', 'jpeg', 'png', 'webp' 四种格式。") + raise TypeError("只能接收 'jpg', 'jpeg', 'png', 'webp' 四种格式。") pic_type = 'jpeg' if as_bytes == 'jpg' else as_bytes elif as_base64: @@ -868,16 +870,18 @@ class ChromiumBase(BasePage): pic_type = 'png' else: if as_base64 not in ('jpg', 'jpeg', 'png', 'webp'): - raise ValueError("只能接收 'jpg', 'jpeg', 'png', 'webp' 四种格式。") + raise TypeError("只能接收 'jpg', 'jpeg', 'png', 'webp' 四种格式。") pic_type = 'jpeg' if as_base64 == 'jpg' else as_base64 else: + if not name: + name = f'{self.title}.jpg' if not path: - path = f'{self.title}.jpg' - path = get_usable_path(path) + path = '.' + if not name.endswith(('.jpg', '.jpeg', '.png', '.webp')): + name = f'{name}.jpg' + path = get_usable_path(f'{path}{sep}{name}') pic_type = path.suffix.lower() - if pic_type not in ('.jpg', '.jpeg', '.png', '.webp'): - raise TypeError(f'不支持的文件格式：{pic_type}。') pic_type = 'jpeg' if pic_type == '.jpg' else pic_type[1:] width, height = self.size @@ -1099,8 +1103,7 @@ class Screencast(object): """非节俭模式运行方法""" self._running = True while self._enable: - p = self._path / f'{time()}.jpg' - self._page.get_screenshot(path=p) + self._page.get_screenshot(path=self._path, name=f'{time()}.jpg') sleep(.04) self._running = False diff --git a/DrissionPage/chromium_base.pyi b/DrissionPage/chromium_base.pyi index ac9ff03..050880b 100644 --- a/DrissionPage/chromium_base.pyi +++ b/DrissionPage/chromium_base.pyi @@ -198,13 +198,13 @@ class ChromiumBase(BasePage): def get_local_storage(self, item: str = None) -> Union[str, dict, None]: ... - def get_screenshot(self, path: [str, Path] = None, + def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, as_base64: [bool, str] = None, full_page: bool = False, left_top: Tuple[int, int] = None, right_bottom: Tuple[int, int] = None) -> Union[str, bytes]: ... - def _get_screenshot(self, path: [str, Path] = None, + def _get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, as_base64: [bool, str] = None, full_page: bool = False, left_top: Tuple[int, int] = None, diff --git a/DrissionPage/chromium_element.py b/DrissionPage/chromium_element.py index ca70063..376a91d 100644 --- a/DrissionPage/chromium_element.py +++ b/DrissionPage/chromium_element.py @@ -3,8 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from os import sep -from os.path import basename +from os.path import basename, sep from pathlib import Path from time import perf_counter, sleep @@ -12,7 +11,7 @@ from .base import DrissionElement, BaseElement from .commons.constants import FRAME_ELEMENT, NoneElement, Settings from .commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions from .commons.locator import get_loc -from .commons.tools import make_valid_name +from .commons.tools import get_usable_path from .commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, location_in_viewport, offset_scroll from .errors import ContextLossError, ElementLossError, JavaScriptError, NoRectError, ElementNotFoundError, \ CDPError, NoResourceError, CanNotClickError @@ -481,50 +480,53 @@ class ChromiumElement(DrissionElement): else: return result['content'] - def save(self, path=None, rename=None, timeout=None): + def save(self, path=None, name=None, timeout=None): """保存图片或其它有src属性的元素的资源 :param path: 文件保存路径，为None时保存到当前文件夹 - :param rename: 文件名称，为None时从资源url获取 + :param name: 文件名称，为None时从资源url获取 :param timeout: 等待资源加载的超时时间 - :return: None + :return: 返回保存路径 """ data = self.get_src(timeout=timeout) if not data: raise NoResourceError path = path or '.' - rename = rename or basename(self.prop('currentSrc')) - rename = make_valid_name(rename) + name = name or basename(self.prop('currentSrc')) + path = get_usable_path(f'{path}{sep}{name}').absolute() write_type = 'wb' if isinstance(data, bytes) else 'w' - Path(path).mkdir(parents=True, exist_ok=True) - with open(f'{path}{sep}{rename}', write_type) as f: + with open(path, write_type) as f: f.write(data) - def get_screenshot(self, path=None, as_bytes=None, as_base64=None): + return str(path) + + def get_screenshot(self, path=None, name=None, as_bytes=None, as_base64=None, scroll_to_center=False): """对当前元素截图，可保存到文件，或以字节方式返回 - :param path: 完整路径，后缀可选 'jpg','jpeg','png','webp' + :param path: 文件保存路径 + :param name: 完整文件名，后缀可选 'jpg','jpeg','png','webp' :param as_bytes: 是否以字节形式返回图片，可选 'jpg','jpeg','png','webp'，生效时path参数和as_base64参数无效 :param as_base64: 是否以base64字符串形式返回图片，可选 'jpg','jpeg','png','webp'，生效时path参数无效 + :param scroll_to_center: 截图前是否滚动到视口中央 :return: 图片完整路径或字节文本 """ if self.tag == 'img': # 等待图片加载完成 - js = ('return this.complete && typeof this.naturalWidth != "undefined" ' - '&& this.naturalWidth > 0 && typeof this.naturalHeight != "undefined" ' - '&& this.naturalHeight > 0') + js = ('return this.complete && typeof this.naturalWidth != "undefined" && this.naturalWidth > 0 ' + '&& typeof this.naturalHeight != "undefined" && this.naturalHeight > 0') end_time = perf_counter() + self.page.timeout while not self.run_js(js) and perf_counter() < end_time: sleep(.1) - # self.scroll.to_see(center=True) - # sleep(1) + if scroll_to_center: + self.scroll.to_see(center=True) + left, top = self.location width, height = self.size left_top = (left, top) right_bottom = (left + width, top + height) - if not path: - path = f'{self.tag}.jpg' - return self.page._get_screenshot(path, as_bytes=as_bytes, as_base64=as_base64, full_page=False, + if not name: + name = f'{self.tag}.jpg' + return self.page._get_screenshot(path, name, as_bytes=as_bytes, as_base64=as_base64, full_page=False, left_top=left_top, right_bottom=right_bottom, ele=self) def input(self, vals, clear=True, by_js=False): diff --git a/DrissionPage/chromium_element.pyi b/DrissionPage/chromium_element.pyi index da4654f..58fac20 100644 --- a/DrissionPage/chromium_element.pyi +++ b/DrissionPage/chromium_element.pyi @@ -180,10 +180,10 @@ class ChromiumElement(DrissionElement): def get_src(self, timeout: float = None, base64_to_bytes: bool = True) -> Union[bytes, str, None]: ... - def save(self, path: [str, bool] = None, rename: str = None, timeout: float = None) -> None: ... + def save(self, path: [str, bool] = None, name: str = None, timeout: float = None) -> str: ... - def get_screenshot(self, path: [str, Path] = None, as_bytes: [bool, str] = None, - as_base64: [bool, str] = None) -> Union[str, bytes]: ... + def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, + as_base64: [bool, str] = None, scroll_to_center: bool = False) -> Union[str, bytes]: ... def input(self, vals: Any, clear: bool = True, by_js: bool = False) -> None: ... diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/chromium_frame.py index e4ed12a..9d136d3 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/chromium_frame.py @@ -4,6 +4,7 @@ @Contact : g1879@qq.com """ from copy import copy +from os.path import sep from re import search from threading import Thread from time import sleep, perf_counter @@ -492,19 +493,21 @@ class ChromiumFrame(ChromiumBase): self._check_ok() return self.frame_ele.afters(filter_loc, timeout, ele_only=ele_only) - def get_screenshot(self, path=None, as_bytes=None, as_base64=None): + def get_screenshot(self, path=None, name=None, as_bytes=None, as_base64=None): """对页面进行截图，可对整个网页、可见网页、指定范围截图。对可视范围外截图需要90以上版本浏览器支持 - :param path: 完整路径，后缀可选 'jpg','jpeg','png','webp' + :param path: 文件保存路径 + :param name: 完整文件名，后缀可选 'jpg','jpeg','png','webp' :param as_bytes: 是否以字节形式返回图片，可选 'jpg','jpeg','png','webp'，生效时path参数和as_base64参数无效 :param as_base64: 是否以base64字符串形式返回图片，可选 'jpg','jpeg','png','webp'，生效时path参数无效 :return: 图片完整路径或字节文本 """ - return self.frame_ele.get_screenshot(path=path, as_bytes=as_bytes, as_base64=as_base64) + return self.frame_ele.get_screenshot(path=path, name=name, as_bytes=as_bytes, as_base64=as_base64) - def _get_screenshot(self, path=None, as_bytes: [bool, str] = None, as_base64: [bool, str] = None, + def _get_screenshot(self, path=None, name=None, as_bytes: [bool, str] = None, as_base64: [bool, str] = None, full_page=False, left_top=None, right_bottom=None, ele=None): - """实现对元素截图 - :param path: 完整路径，后缀可选 'jpg','jpeg','png','webp' + """实现截图 + :param path: 文件保存路径 + :param name: 完整文件名，后缀可选 'jpg','jpeg','png','webp' :param as_bytes: 是否以字节形式返回图片，可选 'jpg','jpeg','png','webp'，生效时path参数和as_base64参数无效 :param as_base64: 是否以base64字符串形式返回图片，可选 'jpg','jpeg','png','webp'，生效时path参数无效 :param full_page: 是否整页截图，为True截取整个网页，为False截取可视窗口 @@ -514,7 +517,7 @@ class ChromiumFrame(ChromiumBase): :return: 图片完整路径或字节文本 """ if not self._is_diff_domain: - return super().get_screenshot(path=path, as_bytes=as_bytes, as_base64=as_base64, + return super().get_screenshot(path=path, name=name, as_bytes=as_bytes, as_base64=as_base64, full_page=full_page, left_top=left_top, right_bottom=right_bottom) if as_bytes: @@ -522,7 +525,7 @@ class ChromiumFrame(ChromiumBase): pic_type = 'png' else: if as_bytes not in ('jpg', 'jpeg', 'png', 'webp'): - raise ValueError("只能接收 'jpg', 'jpeg', 'png', 'webp' 四种格式。") + raise TypeError("只能接收 'jpg', 'jpeg', 'png', 'webp' 四种格式。") pic_type = 'jpeg' if as_bytes == 'jpg' else as_bytes elif as_base64: @@ -530,16 +533,18 @@ class ChromiumFrame(ChromiumBase): pic_type = 'png' else: if as_base64 not in ('jpg', 'jpeg', 'png', 'webp'): - raise ValueError("只能接收 'jpg', 'jpeg', 'png', 'webp' 四种格式。") + raise TypeError("只能接收 'jpg', 'jpeg', 'png', 'webp' 四种格式。") pic_type = 'jpeg' if as_base64 == 'jpg' else as_base64 else: if not path: - path = f'{self.title}.jpg' - path = get_usable_path(path) + path = '.' + if not name: + name = f'{self.title}.jpg' + if not name.endswith(('.jpg', '.jpeg', '.png', '.webp')): + name = f'{name}.jpg' + path = get_usable_path(f'{path}{sep}{name}') pic_type = path.suffix.lower() - if pic_type not in ('.jpg', '.jpeg', '.png', '.webp'): - raise TypeError(f'不支持的文件格式：{pic_type}。') pic_type = 'jpeg' if pic_type == '.jpg' else pic_type[1:] self.frame_ele.scroll.to_see(center=True) @@ -562,7 +567,7 @@ class ChromiumFrame(ChromiumBase): new_ele.scroll.to_see(True) top = int(self.frame_ele.style('border-top').split('px')[0]) left = int(self.frame_ele.style('border-left').split('px')[0]) - r = self._target_page.get_screenshot(path=path, as_bytes=as_bytes, as_base64=as_base64, + r = self._target_page.get_screenshot(path=path, name=name, as_bytes=as_bytes, as_base64=as_base64, left_top=(cx + left, cy + top), right_bottom=(cx + w + left, cy + h + top)) self._target_page.remove_ele(new_ele) return r diff --git a/DrissionPage/chromium_frame.pyi b/DrissionPage/chromium_frame.pyi index e310dc6..3bf2f12 100644 --- a/DrissionPage/chromium_frame.pyi +++ b/DrissionPage/chromium_frame.pyi @@ -174,11 +174,11 @@ class ChromiumFrame(ChromiumBase): timeout: float = None, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def get_screenshot(self, path: [str, Path] = None, + def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, as_base64: [bool, str] = None) -> Union[str, bytes]: ... - def _get_screenshot(self, path: [str, Path] = None, + def _get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, as_base64: [bool, str] = None, full_page: bool = False, left_top: Tuple[int, int] = None, From a6233aa92366e0b80c219ff79dc8583dd526f507 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 8 Sep 2023 17:36:00 +0800 Subject: [PATCH 020/182] =?UTF-8?q?=E8=8E=B7=E5=8F=96=E5=85=83=E7=B4=A0siz?= =?UTF-8?q?e=E6=9B=B4=E5=87=86=E7=A1=AE=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_element.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DrissionPage/chromium_element.py b/DrissionPage/chromium_element.py index 376a91d..089ba08 100644 --- a/DrissionPage/chromium_element.py +++ b/DrissionPage/chromium_element.py @@ -123,8 +123,8 @@ class ChromiumElement(DrissionElement): @property def size(self): """返回元素宽和高组成的元组""" - model = self.page.run_cdp('DOM.getBoxModel', backendNodeId=self._backend_id)['model'] - return model['width'], model['height'] + border = self.page.run_cdp('DOM.getBoxModel', backendNodeId=self._backend_id)['model']['border'] + return int(border[2] - border[0]), int(border[5] - border[1]) @property def set(self): From ed2883e2b96ca75f1d0774c1263a813a3c38cded Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 11 Sep 2023 07:10:26 +0800 Subject: [PATCH 021/182] =?UTF-8?q?browser=5Fdriver=E5=92=8CBrowserManager?= =?UTF-8?q?=E6=94=B9=E6=88=90=E6=AF=8F=E4=B8=AA=E6=B5=8F=E8=A7=88=E5=99=A8?= =?UTF-8?q?=E5=8F=AA=E4=B8=80=E4=B8=AA=E5=AF=B9=E8=B1=A1=EF=BC=9B=E6=94=B9?= =?UTF-8?q?=E8=BF=9B=E4=B8=8B=E6=B5=8F=E8=A7=88=E5=99=A8=E8=BD=BD=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=EF=BC=8C=E6=9C=AA=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_driver.py | 18 +++++++++ DrissionPage/chromium_driver.pyi | 6 ++- DrissionPage/chromium_page.py | 69 +++++++++++++++++++++++--------- DrissionPage/chromium_page.pyi | 7 +++- DrissionPage/waiter.py | 17 +------- DrissionPage/waiter.pyi | 2 - 6 files changed, 80 insertions(+), 39 deletions(-) diff --git a/DrissionPage/chromium_driver.py b/DrissionPage/chromium_driver.py index fde230f..b638a0e 100644 --- a/DrissionPage/chromium_driver.py +++ b/DrissionPage/chromium_driver.py @@ -233,3 +233,21 @@ class ChromiumDriver(object): return f"" __repr__ = __str__ + + +class BrowserDriver(ChromiumDriver): + BROWSERS = {} + + def __new__(cls, tab_id, tab_type, address): + if tab_id in cls.BROWSERS: + return cls.BROWSERS[tab_id] + return object.__new__(cls) + + def __init__(self, tab_id, tab_type, address): + if tab_id in BrowserDriver.BROWSERS: + return + super().__init__(tab_id, tab_type, address) + BrowserDriver.BROWSERS[tab_id] = self + + def __repr__(self): + return f"" diff --git a/DrissionPage/chromium_driver.pyi b/DrissionPage/chromium_driver.pyi index 9ba7feb..152d4c4 100644 --- a/DrissionPage/chromium_driver.pyi +++ b/DrissionPage/chromium_driver.pyi @@ -5,7 +5,7 @@ """ from queue import Queue from threading import Thread, Event -from typing import Union, Callable +from typing import Union, Callable, Dict class GenericAttr(object): @@ -58,3 +58,7 @@ class ChromiumDriver(object): def get_listener(self, event: str) -> Union[Callable, None]: ... def __str__(self) -> str: ... + + +class BrowserDriver(ChromiumDriver): + BROWSERS: Dict[str, ChromiumDriver] = ... diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 8009d16..742f05c 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -3,12 +3,14 @@ @Author : g1879 @Contact : g1879@qq.com """ +from pathlib import Path from shutil import move +from threading import Lock from time import perf_counter, sleep from .chromium_base import ChromiumBase, Timeout from .chromium_base import handle_download -from .chromium_driver import ChromiumDriver +from .chromium_driver import ChromiumDriver, BrowserDriver from .chromium_tab import ChromiumTab from .commons.browser import connect_browser from .commons.tools import get_usable_path @@ -98,7 +100,7 @@ class ChromiumPage(ChromiumBase): u = f'http://{self.address}/json/version' ws = self._control_session.get(u).json()['webSocketDebuggerUrl'] self._control_session.get(u, headers={'Connection': 'close'}) - self._browser_driver = ChromiumDriver(ws.split('/')[-1], 'browser', self.address) + self._browser_driver = BrowserDriver(ws.split('/')[-1], 'browser', self.address) self._browser_driver.start() self._alert = Alert() @@ -446,13 +448,32 @@ class ChromiumTabRect(object): class BrowserDownloadManager(object): + BROWSERS = {} + + def __new__(cls, page): + """ + :param page: ChromiumPage对象 + """ + if page.browser_driver.id in cls.BROWSERS: + return cls.BROWSERS[page.browser_driver.id] + return object.__new__(cls) + def __init__(self, page): + """ + :param page: ChromiumPage对象 + """ + if page.browser_driver.id in BrowserDownloadManager.BROWSERS: + return + self._page = page + self._lock = Lock() page.set.download_path(page.download_path) self._page.browser_driver.set_listener('Browser.downloadProgress', self._onDownloadProgress) self._page.browser_driver.set_listener('Browser.downloadWillBegin', self._onDownloadWillBegin) self._missions = {} + BrowserDownloadManager.BROWSERS[page.browser_driver.id] = self + @property def missions(self): return self._missions @@ -464,12 +485,20 @@ class BrowserDownloadManager(object): """ self._missions[mission.id] = mission - def cancel(self, mission): - """取消一个下载任务 + def set_done(self, mission, state, cancel=False, final_path=None): + """设置任务结束 :param mission: 任务对象 + :param state: 任务状态 + :param cancel: 是否取消 + :param final_path: 最终路径 :return: None """ - self._page.browser_driver.call_method('Browser.cancelDownload', guid=mission.id) + mission.state = state + mission.final_path = final_path + if cancel: + self._page.browser_driver.call_method('Browser.cancelDownload', guid=mission.id) + if mission.final_path: + Path(mission.final_path).unlink(True) self._missions.pop(mission.id) def _onDownloadWillBegin(self, **kwargs): @@ -481,22 +510,24 @@ class BrowserDownloadManager(object): def _onDownloadProgress(self, **kwargs): """下载状态变化时执行""" if kwargs['guid'] in self._missions: - mission = self._missions[kwargs['guid']] - if kwargs['state'] == 'inProgress': - mission.state = 'running' - mission.received_bytes = kwargs['receivedBytes'] - mission.total_bytes = kwargs['totalBytes'] + with self._lock: + if kwargs['guid'] in self._missions: + mission = self._missions[kwargs['guid']] + if kwargs['state'] == 'inProgress': + mission.state = 'running' + mission.received_bytes = kwargs['receivedBytes'] + mission.total_bytes = kwargs['totalBytes'] - elif kwargs['state'] == 'completed': - mission.received_bytes = kwargs['receivedBytes'] - mission.total_bytes = kwargs['totalBytes'] - form_path = f'{self._page.download_path}\\{mission.id}' - to_path = str(get_usable_path(f'{mission.path}\\{mission.name}')) - move(form_path, to_path) - mission._set_done('completed', final_path=to_path) + elif kwargs['state'] == 'completed': + mission.received_bytes = kwargs['receivedBytes'] + mission.total_bytes = kwargs['totalBytes'] + form_path = f'{self._page.download_path}\\{mission.id}' + to_path = str(get_usable_path(f'{mission.path}\\{mission.name}')) + move(form_path, to_path) + self.set_done(mission, 'completed', final_path=to_path) - else: - mission._set_done('canceled') + else: + self.set_done(mission, 'canceled') class Alert(object): diff --git a/DrissionPage/chromium_page.pyi b/DrissionPage/chromium_page.pyi index be96790..88d62e0 100644 --- a/DrissionPage/chromium_page.pyi +++ b/DrissionPage/chromium_page.pyi @@ -3,6 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ +from threading import Lock from typing import Union, Tuple, List, Dict from .chromium_base import ChromiumBase @@ -128,6 +129,10 @@ class ChromiumTabRect(object): class BrowserDownloadManager(object): _page: ChromiumPage = ... _missions: Dict[str, DownloadMission] = ... + _lock: Lock = ... + BROWSERS: Dict[str, BrowserDownloadManager] = ... + + def __new__(cls, page: ChromiumPage): ... def __init__(self, page: ChromiumPage): ... @@ -136,7 +141,7 @@ class BrowserDownloadManager(object): def add_mission(self, mission: DownloadMission) -> None: ... - def cancel(self, mission: DownloadMission) -> None: ... + def set_done(self, mission: DownloadMission, state: str, cancel: bool = False, final_path: str = None) -> None: ... def _onDownloadWillBegin(self, **kwargs) -> None: ... diff --git a/DrissionPage/waiter.py b/DrissionPage/waiter.py index c8f73a4..e693873 100644 --- a/DrissionPage/waiter.py +++ b/DrissionPage/waiter.py @@ -401,9 +401,7 @@ class DownloadMission(object): def cancel(self): """取消该任务，如任务已完成，删除已下载的文件""" - self._set_done('canceled', True) - if self.final_path: - Path(self.final_path).unlink(True) + self.tab._page._dl_mgr.set_done(self, state='canceled', cancel=True) def wait(self, show=True, timeout=None, cancel_if_timeout=True): """等待任务结束 @@ -450,16 +448,3 @@ class DownloadMission(object): print() return self.final_path if self.final_path else False - - def _set_done(self, state, cancel=False, final_path=None): - """设置任务结束 - :param state: 任务状态 - :param cancel: 是否取消 - :param final_path: 最终路径 - :return: None - """ - self.state = state - self.final_path = final_path - if cancel: - self.tab._page._dl_mgr.cancel(self) - self.tab._download_missions.remove(self) diff --git a/DrissionPage/waiter.pyi b/DrissionPage/waiter.pyi index 7858606..ad73281 100644 --- a/DrissionPage/waiter.pyi +++ b/DrissionPage/waiter.pyi @@ -108,5 +108,3 @@ class DownloadMission(object): def cancel(self) -> None: ... def wait(self, show: bool = True, timeout=None, cancel_if_timeout=True) -> Union[bool, str]: ... - - def _set_done(self, state: str, cancel: bool = False, final_path: str = None) -> None: ... From 3eee7132d5cce6a3371644813a6d65651f105691 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 11 Sep 2023 17:25:38 +0800 Subject: [PATCH 022/182] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E5=86=99=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/action_chains.py | 17 +++++++++-------- DrissionPage/chromium_page.py | 1 + 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/DrissionPage/action_chains.py b/DrissionPage/action_chains.py index b9edcaa..d30bf3a 100644 --- a/DrissionPage/action_chains.py +++ b/DrissionPage/action_chains.py @@ -59,7 +59,7 @@ class ActionChains: cx = x + offset_x cy = y + offset_y - self._dr.Input.dispatchMouseEvent(type='mouseMoved', x=cx, y=cy, modifiers=self.modifier) + self._dr.call_method('Input.dispatchMouseEvent', type='mouseMoved', x=cx, y=cy, modifiers=self.modifier) self.curr_x = cx self.curr_y = cy return self @@ -72,7 +72,8 @@ class ActionChains: """ self.curr_x += offset_x self.curr_y += offset_y - self._dr.Input.dispatchMouseEvent(type='mouseMoved', x=self.curr_x, y=self.curr_y, modifiers=self.modifier) + self._dr.call_method('Input.dispatchMouseEvent', type='mouseMoved', x=self.curr_x, y=self.curr_y, + modifiers=self.modifier) return self def click(self, on_ele=None): @@ -170,8 +171,8 @@ class ActionChains: """ if on_ele: self.move_to(on_ele) - self._dr.Input.dispatchMouseEvent(type='mousePressed', button=button, clickCount=count, - x=self.curr_x, y=self.curr_y, modifiers=self.modifier) + self._dr.call_method('Input.dispatchMouseEvent', type='mousePressed', button=button, clickCount=count, + x=self.curr_x, y=self.curr_y, modifiers=self.modifier) return self def _release(self, button): @@ -179,8 +180,8 @@ class ActionChains: :param button: 要释放的按键 :return: self """ - self._dr.Input.dispatchMouseEvent(type='mouseReleased', button=button, clickCount=1, - x=self.curr_x, y=self.curr_y, modifiers=self.modifier) + self._dr.call_method('Input.dispatchMouseEvent', type='mouseReleased', button=button, clickCount=1, + x=self.curr_x, y=self.curr_y, modifiers=self.modifier) return self def scroll(self, delta_x=0, delta_y=0, on_ele=None): @@ -192,8 +193,8 @@ class ActionChains: """ if on_ele: self.move_to(on_ele) - self._dr.Input.dispatchMouseEvent(type='mouseWheel', x=self.curr_x, y=self.curr_y, - deltaX=delta_x, deltaY=delta_y, modifiers=self.modifier) + self._dr.call_method('Input.dispatchMouseEvent', type='mouseWheel', x=self.curr_x, y=self.curr_y, + deltaX=delta_x, deltaY=delta_y, modifiers=self.modifier) return self def up(self, pixel): diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 742f05c..a9336f0 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -476,6 +476,7 @@ class BrowserDownloadManager(object): @property def missions(self): + """返回所有未完成的下载任务""" return self._missions def add_mission(self, mission): From 1015d6c076af0d6b9d81166a3204d18f2bf84a31 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 22 Sep 2023 19:42:53 +0800 Subject: [PATCH 023/182] =?UTF-8?q?=E5=85=83=E7=B4=A0=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0to=5Fcenter()=EF=BC=8C=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0is=5Fwhole=5Fin=5Fviewport=EF=BC=9B=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E5=AF=B9=E8=B1=A1=E5=A2=9E=E5=8A=A0actions=E5=B1=9E?= =?UTF-8?q?=E6=80=A7=EF=BC=9B=E4=BF=AE=E5=A4=8D=E5=85=83=E7=B4=A0=E6=88=AA?= =?UTF-8?q?=E5=9B=BE=E9=97=AE=E9=A2=98=EF=BC=9Bget=5Fusable=5Fpath()?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0is=5Ffile,=20parents=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/action_chains.py | 2 +- DrissionPage/action_chains.pyi | 2 +- DrissionPage/chromium_base.py | 60 ++++++++++++------------------- DrissionPage/chromium_base.pyi | 16 +++++---- DrissionPage/chromium_element.py | 16 +++++++-- DrissionPage/chromium_element.pyi | 7 +++- DrissionPage/chromium_frame.py | 27 +++++++------- DrissionPage/chromium_frame.pyi | 1 - DrissionPage/commons/tools.py | 48 ++++--------------------- DrissionPage/commons/tools.pyi | 8 +---- DrissionPage/commons/web.py | 2 +- DrissionPage/commons/web.pyi | 2 +- DrissionPage/easy_set.py | 2 +- 13 files changed, 78 insertions(+), 115 deletions(-) diff --git a/DrissionPage/action_chains.py b/DrissionPage/action_chains.py index d30bf3a..0b5724a 100644 --- a/DrissionPage/action_chains.py +++ b/DrissionPage/action_chains.py @@ -14,7 +14,7 @@ class ActionChains: def __init__(self, page): """ - :param page: ChromiumPage对象 + :param page: ChromiumBase对象 """ self.page = page self._dr = page.driver diff --git a/DrissionPage/action_chains.pyi b/DrissionPage/action_chains.pyi index 3b54a49..ca6061a 100644 --- a/DrissionPage/action_chains.pyi +++ b/DrissionPage/action_chains.pyi @@ -14,7 +14,7 @@ from .chromium_page import ChromiumPage class ActionChains: def __init__(self, page: ChromiumBase): - self.page: ChromiumPage = ... + self.page: ChromiumBase = ... self._dr: ChromiumDriver = ... self.modifier: int = ... self.curr_x: int = ... diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index 93b5c8a..8b379de 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -12,18 +12,20 @@ from time import perf_counter, sleep, time from requests import Session +from .action_chains import ActionChains from .base import BasePage from .chromium_driver import ChromiumDriver from .chromium_element import ChromiumScroll, ChromiumElement, run_js, make_chromium_ele from .commons.constants import HANDLE_ALERT_METHOD, ERROR, NoneElement from .commons.locator import get_loc from .commons.tools import get_usable_path, clean_folder +from .commons.web import location_in_viewport from .errors import ContextLossError, ElementLossError, AlertExistsError, CDPError, TabClosedError, \ NoRectError, BrowserConnectError, GetDocumentError from .network_listener import NetworkListener from .session_element import make_session_ele from .setter import ChromiumBaseSetter -from .waiter import ChromiumBaseWaiter, DownloadMission +from .waiter import ChromiumBaseWaiter class ChromiumBase(BasePage): @@ -43,13 +45,10 @@ class ChromiumBase(BasePage): self._tab_obj = None self._set = None self._screencast = None + self._actions = None self._listener = None - self._wait_download_flag = None - self._download_rename = None self._download_path = '' - self._when_download_file_exists = 'rename' - self._download_missions = set() if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): address = f'127.0.0.1:{address}' @@ -252,7 +251,7 @@ class ChromiumBase(BasePage): def _onDownloadWillBegin(self, **kwargs): """下载即将开始时执行""" - handle_download(self, kwargs) + self._page._dl_mgr.set_mission(self.tab_id, kwargs['guid']) def __call__(self, loc_or_str, timeout=None): """在内部查找元素 @@ -398,6 +397,14 @@ class ChromiumBase(BasePage): self._screencast = Screencast(self) return self._screencast + @property + def actions(self): + """返回用于执行动作链的对象""" + if self._actions is None: + self._actions = ActionChains(self) + self.wait.load_complete() + return self._actions + @property def listener(self): """返回用于聆听数据包的对象""" @@ -894,9 +901,17 @@ class ChromiumBase(BasePage): x, y = left_top w = right_bottom[0] - x h = right_bottom[1] - y + + v = not (location_in_viewport(self, x, y) and + location_in_viewport(self, right_bottom[0], right_bottom[1])) + if v and (self.run_js('return document.body.scrollHeight > window.innerHeight;') and + not self.run_js('return document.body.scrollWidth > window.innerWidth;')): + x += 10 + vp = {'x': x, 'y': y, 'width': w, 'height': h, 'scale': 1} png = self.run_cdp_loaded('Page.captureScreenshot', format=pic_type, - captureBeyondViewport=True, clip=vp)['data'] + captureBeyondViewport=v, clip=vp)['data'] + else: png = self.run_cdp_loaded('Page.captureScreenshot', format=pic_type)['data'] @@ -1132,34 +1147,3 @@ class ScreencastMode(object): def imgs_mode(self): self._screencast._mode = 'imgs' - - -def handle_download(tab, kwargs): - """在下载开始前处理任务 - :param tab: 触发任务的tab对象 - :param kwargs: 浏览器返回的数据 - :return: None - """ - tab._page._dl_mgr._missions[kwargs['guid']] = None - - if tab._download_rename: - tmp = kwargs['suggestedFilename'].rsplit('.', 1) - ext_name = tmp[-1] if len(tmp) > 1 else '' - tmp = tab._download_rename.rsplit('.', 1) - ext_rename = tmp[-1] if len(tmp) > 1 else '' - n = tab._download_rename if ext_rename == ext_name else f'{tab._download_rename}.{ext_name}' - tab._download_rename = None - - else: - n = kwargs['suggestedFilename'] - - m = DownloadMission(tab, kwargs['guid'], tab.download_path, n, kwargs['url']) - tab._page._dl_mgr.add_mission(m) - tab._wait_download_flag = m - tab._download_missions.add(m) - - if tab._wait_download_flag is False: # 取消该任务 - m._set_done('canceled', True) - - if tab._when_download_file_exists == 'skip' and (Path(m.path) / m.name).exists(): - m._set_done('skipped', True) diff --git a/DrissionPage/chromium_base.pyi b/DrissionPage/chromium_base.pyi index 050880b..82ed6a2 100644 --- a/DrissionPage/chromium_base.pyi +++ b/DrissionPage/chromium_base.pyi @@ -9,6 +9,7 @@ from typing import Union, Tuple, List, Any from DataRecorder import Recorder from requests import Session +from .action_chains import ActionChains from .base import BasePage from .chromium_driver import ChromiumDriver from .chromium_element import ChromiumElement, ChromiumScroll @@ -44,11 +45,12 @@ class ChromiumBase(BasePage): self._wait: ChromiumBaseWaiter = ... self._set: ChromiumBaseSetter = ... self._screencast: Screencast = ... + self._actions: ActionChains = ... self._listener: NetworkListener = ... - self._wait_download_flag: bool = ... - self._download_rename: str = ... - self._when_download_file_exists: str = ... - self._download_missions: set = ... + # self._wait_download_flag: bool = ... + # self._download_rename: str = ... + # self._when_download_file_exists: str = ... + # self._download_missions: set = ... def _connect_browser(self, tab_id: str = None) -> None: ... @@ -144,6 +146,9 @@ class ChromiumBase(BasePage): @property def screencast(self) -> Screencast: ... + @property + def actions(self) -> ActionChains: ... + @property def listener(self) -> NetworkListener: ... @@ -277,6 +282,3 @@ class ScreencastMode(object): def frugal_imgs_mode(self) -> None: ... def imgs_mode(self) -> None: ... - - -def handle_download(tab: ChromiumBase, kwargs: dict) -> None: ... diff --git a/DrissionPage/chromium_element.py b/DrissionPage/chromium_element.py index 089ba08..5bf21f8 100644 --- a/DrissionPage/chromium_element.py +++ b/DrissionPage/chromium_element.py @@ -1435,6 +1435,14 @@ class ChromiumElementStates(object): x, y = self._ele.locations.click_point return location_in_viewport(self._ele.page, x, y) if x else False + @property + def is_whole_in_viewport(self): + """返回元素是否整个都在视口内""" + x1, y1 = self._ele.location + w, h = self._ele.size + x2, y2 = x1 + w, y1 + h + return location_in_viewport(self._ele.page, x1, y1) and location_in_viewport(self._ele.page, x2, y2) + @property def is_covered(self): """返回元素是否被覆盖，与是否在视口中无关""" @@ -1766,6 +1774,10 @@ class ChromiumElementScroll(ChromiumScroll): """ self._driver.page.scroll.to_see(self._driver, center=center) + def to_center(self): + """元素尽量滚动到视口中间""" + self._driver.page.scroll.to_see(self._driver, center=True) + class ChromiumSelect(object): """ChromiumSelect 类专门用于处理 d 模式下 select 标签""" @@ -1936,10 +1948,10 @@ class ChromiumSelect(object): mode = 'false' if cancel else 'true' timeout = timeout if timeout is not None else self._ele.page.timeout - condition = {condition} if isinstance(condition, (str, int)) else set(condition) + condition = set(condition) if isinstance(condition, (list, tuple)) else {condition} if para_type in ('text', 'value'): - return self._text_value(condition, para_type, mode, timeout) + return self._text_value([str(i) for i in condition], para_type, mode, timeout) elif para_type == 'index': return self._index(condition, mode, timeout) diff --git a/DrissionPage/chromium_element.pyi b/DrissionPage/chromium_element.pyi index 58fac20..68a455a 100644 --- a/DrissionPage/chromium_element.pyi +++ b/DrissionPage/chromium_element.pyi @@ -232,6 +232,9 @@ class ChromiumElementStates(object): @property def is_in_viewport(self) -> bool: ... + @property + def is_whole_in_viewport(self) -> bool: ... + @property def is_covered(self) -> bool: ... @@ -489,6 +492,8 @@ class ChromiumElementScroll(ChromiumScroll): def to_see(self, center: Union[bool, None] = None) -> None: ... + def to_center(self) -> None: ... + class ChromiumSelect(object): def __init__(self, ele: ChromiumElement): @@ -538,7 +543,7 @@ class ChromiumSelect(object): cancel: bool = False, timeout: float = None) -> bool: ... - def _text_value(self, condition: set, para_type: str, mode: str, timeout: float) -> bool: ... + def _text_value(self, condition: Union[list, set], para_type: str, mode: str, timeout: float) -> bool: ... def _index(self, condition: set, mode: str, timeout: float) -> bool: ... diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/chromium_frame.py index 9d136d3..4925910 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/chromium_frame.py @@ -4,14 +4,12 @@ @Contact : g1879@qq.com """ from copy import copy -from os.path import sep from re import search from threading import Thread from time import sleep, perf_counter from .chromium_base import ChromiumBase, ChromiumPageScroll from .chromium_element import ChromiumElement -from .commons.tools import get_usable_path from .errors import ContextLossError from .setter import ChromiumFrameSetter from .waiter import FrameWaiter @@ -24,7 +22,7 @@ class ChromiumFrame(ChromiumBase): :param ele: frame所在元素 """ page_type = str(type(page)) - if 'ChromiumPage' in page_type or 'WebPage' in page: + if 'ChromiumPage' in page_type or 'WebPage' in page_type: self._page = self._target_page = self.tab = page else: # Tab、Frame self._page = page.page @@ -79,7 +77,7 @@ class ChromiumFrame(ChromiumBase): self.retry_interval = self._target_page.retry_interval self._page_load_strategy = self._target_page.page_load_strategy self._download_path = self._target_page.download_path - self._when_download_file_exists = self._target_page._when_download_file_exists + # self._when_download_file_exists = self._target_page._when_download_file_exists def _driver_init(self, tab_id): """避免出现服务器500错误 @@ -543,16 +541,16 @@ class ChromiumFrame(ChromiumBase): name = f'{self.title}.jpg' if not name.endswith(('.jpg', '.jpeg', '.png', '.webp')): name = f'{name}.jpg' - path = get_usable_path(f'{path}{sep}{name}') - pic_type = path.suffix.lower() - pic_type = 'jpeg' if pic_type == '.jpg' else pic_type[1:] + pic_type = name.split('.')[-1] + if pic_type == 'jpg': + pic_type = 'jpeg' self.frame_ele.scroll.to_see(center=True) self.scroll.to_see(ele, center=True) cx, cy = ele.locations.viewport_location w, h = ele.size img_data = f'data:image/{pic_type};base64,{self.frame_ele.get_screenshot(as_base64=True)}' - body = self._target_page('t:body') + body = self.tab('t:body') first_child = body('c::first-child') if not isinstance(first_child, ChromiumElement): first_child = first_child.frame_ele @@ -564,12 +562,17 @@ class ChromiumFrame(ChromiumBase): arguments[0].insertBefore(img, this); return img;''' new_ele = first_child.run_js(js, body) - new_ele.scroll.to_see(True) + new_ele.scroll.to_see(center=True) top = int(self.frame_ele.style('border-top').split('px')[0]) left = int(self.frame_ele.style('border-left').split('px')[0]) - r = self._target_page.get_screenshot(path=path, name=name, as_bytes=as_bytes, as_base64=as_base64, - left_top=(cx + left, cy + top), right_bottom=(cx + w + left, cy + h + top)) - self._target_page.remove_ele(new_ele) + + r = self.tab.run_cdp('Page.getLayoutMetrics')['visualViewport'] + sx = r['pageX'] + sy = r['pageY'] + r = self.tab.get_screenshot(path=path, name=name, as_bytes=as_bytes, as_base64=as_base64, + left_top=(cx + left + sx, cy + top + sy), + right_bottom=(cx + w + left + sx, cy + h + top + sy)) + self.tab.remove_ele(new_ele) return r def _find_elements(self, loc_or_ele, timeout=None, single=True, relative=False, raise_err=None): diff --git a/DrissionPage/chromium_frame.pyi b/DrissionPage/chromium_frame.pyi index 3bf2f12..c125fa4 100644 --- a/DrissionPage/chromium_frame.pyi +++ b/DrissionPage/chromium_frame.pyi @@ -24,7 +24,6 @@ class ChromiumFrame(ChromiumBase): self.frame_id: str = ... self._frame_ele: ChromiumElement = ... self._backend_id: str = ... - self.frame_page: ChromiumBase = ... self._doc_ele: ChromiumElement = ... self._is_diff_domain: bool = ... self.doc_ele: ChromiumElement = ... diff --git a/DrissionPage/commons/tools.py b/DrissionPage/commons/tools.py index e360688..7f48033 100644 --- a/DrissionPage/commons/tools.py +++ b/DrissionPage/commons/tools.py @@ -10,21 +10,24 @@ from shutil import rmtree from time import perf_counter, sleep -def get_usable_path(path): +def get_usable_path(path, is_file=True, parents=True): """检查文件或文件夹是否有重名，并返回可以使用的路径 :param path: 文件或文件夹路径 + :param is_file: 目标是文件还是文件夹 + :param parents: 是否创建目标路径 :return: 可用的路径，Path对象 """ path = Path(path) parent = path.parent - parent.mkdir(parents=True, exist_ok=True) + if parents: + parent.mkdir(parents=True, exist_ok=True) path = parent / make_valid_name(path.name) name = path.stem if path.is_file() else path.name ext = path.suffix if path.is_file() else '' first_time = True - while path.exists(): + while path.exists() and path.is_file() == is_file: r = search(r'(.*)_(\d+)$', name) if not r or (r and first_time): @@ -213,42 +216,3 @@ def wait_until(page, condition, timeout=10, poll=0.1, raise_err=True): raise TimeoutError('等待超时') else: return False - -# def get_exe_from_port(port): -# """获取端口号第一条进程的可执行文件路径 -# :param port: 端口号 -# :return: 可执行文件的绝对路径 -# """ -# from os import popen -# -# pid = get_pid_from_port(port) -# if not pid: -# return -# else: -# file_lst = popen(f'wmic process where processid={pid} get executablepath').read().split('\n') -# return file_lst[2].strip() if len(file_lst) > 2 else None -# -# -# def get_pid_from_port(port): -# """获取端口号第一条进程的pid -# :param port: 端口号 -# :return: 进程id -# """ -# from platform import system -# if system().lower() != 'windows' or port is None: -# return None -# -# from os import popen -# from time import perf_counter -# -# try: # 避免Anaconda中可能产生的报错 -# process = popen(f'netstat -ano |findstr {port}').read().split('\n')[0] -# -# t = perf_counter() -# while not process and perf_counter() - t < 5: -# process = popen(f'netstat -ano |findstr {port}').read().split('\n')[0] -# -# return process.split(' ')[-1] or None -# -# except Exception: -# return None diff --git a/DrissionPage/commons/tools.pyi b/DrissionPage/commons/tools.pyi index d7ed7f7..ba8d6ee 100644 --- a/DrissionPage/commons/tools.pyi +++ b/DrissionPage/commons/tools.pyi @@ -11,13 +11,7 @@ from types import FunctionType from chromium_page import ChromiumPage -# def get_exe_from_port(port: Union[str, int]) -> Union[str, None]: ... - - -# def get_pid_from_port(port: Union[str, int]) -> Union[str, None]: ... - - -def get_usable_path(path: Union[str, Path]) -> Path: ... +def get_usable_path(path: Union[str, Path], is_file: bool = True, parents: bool = True) -> Path: ... def make_valid_name(full_name: str) -> str: ... diff --git a/DrissionPage/commons/web.py b/DrissionPage/commons/web.py index 928b545..bad3529 100644 --- a/DrissionPage/commons/web.py +++ b/DrissionPage/commons/web.py @@ -92,7 +92,7 @@ def location_in_viewport(page, loc_x, loc_y): :param page: ChromePage对象 :param loc_x: 页面绝对坐标x :param loc_y: 页面绝对坐标y - :return: + :return: bool """ js = f'''function(){{var x = {loc_x}; var y = {loc_y}; const scrollLeft = document.documentElement.scrollLeft; diff --git a/DrissionPage/commons/web.pyi b/DrissionPage/commons/web.pyi index b91ba71..f4b4931 100644 --- a/DrissionPage/commons/web.pyi +++ b/DrissionPage/commons/web.pyi @@ -20,7 +20,7 @@ def get_ele_txt(e: DrissionElement) -> str: ... def format_html(text: str) -> str: ... -def location_in_viewport(page, loc_x: int, loc_y: int) -> bool: ... +def location_in_viewport(page: ChromiumBase, loc_x: int, loc_y: int) -> bool: ... def offset_scroll(ele: ChromiumElement, offset_x: int, offset_y: int) -> tuple: ... diff --git a/DrissionPage/easy_set.py b/DrissionPage/easy_set.py index becea0f..7ed75a7 100644 --- a/DrissionPage/easy_set.py +++ b/DrissionPage/easy_set.py @@ -198,7 +198,7 @@ def get_chrome_path(ini_path=None, from platform import system sys = system().lower() - if sys == 'macos': + if sys in ('macos', 'darwin'): return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' elif sys == 'linux': From 9690a575024c4e4392f6c229b1cf9588c297d7b7 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 9 Oct 2023 22:59:41 +0800 Subject: [PATCH 024/182] =?UTF-8?q?get=5Fscreenshot()=E5=85=BC=E5=AE=B9?= =?UTF-8?q?=E6=97=A7=E7=89=88=E6=9C=ACpath=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_base.py | 16 +++++++++------- DrissionPage/chromium_frame.py | 17 ++++++++++------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index 8b379de..80f2d13 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -881,13 +881,15 @@ class ChromiumBase(BasePage): pic_type = 'jpeg' if as_base64 == 'jpg' else as_base64 else: - if not name: - name = f'{self.title}.jpg' - if not path: - path = '.' - if not name.endswith(('.jpg', '.jpeg', '.png', '.webp')): - name = f'{name}.jpg' - path = get_usable_path(f'{path}{sep}{name}') + path = str(path).rstrip('\\/') if path else '.' + if not path.endswith(('.jpg', '.jpeg', '.png', '.webp')): + if not name: + name = f'{self.title}.jpg' + elif not name.endswith(('.jpg', '.jpeg', '.png', '.webp')): + name = f'{name}.jpg' + path = f'{path}{sep}{name}' + + path = get_usable_path(path) pic_type = path.suffix.lower() pic_type = 'jpeg' if pic_type == '.jpg' else pic_type[1:] diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/chromium_frame.py index 4925910..28319ee 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/chromium_frame.py @@ -535,13 +535,16 @@ class ChromiumFrame(ChromiumBase): pic_type = 'jpeg' if as_base64 == 'jpg' else as_base64 else: - if not path: - path = '.' - if not name: - name = f'{self.title}.jpg' - if not name.endswith(('.jpg', '.jpeg', '.png', '.webp')): - name = f'{name}.jpg' - pic_type = name.split('.')[-1] + path = str(path).rstrip('\\/') if path else '.' + if path and path.endswith(('.jpg', '.jpeg', '.png', '.webp')): + pic_type = path.rsplit('.', 1)[-1] + + elif name and name.endswith(('.jpg', '.jpeg', '.png', '.webp')): + pic_type = name.rsplit('.', 1)[-1] + + else: + pic_type = 'jpeg' + if pic_type == 'jpg': pic_type = 'jpeg' From fa6bc08b5b9f6c474a390fc2778c1d1c37cdccd3 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 10 Oct 2023 09:41:26 +0800 Subject: [PATCH 025/182] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=88=AA=E5=9B=BE?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_element.py | 4 ++-- DrissionPage/chromium_element.pyi | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DrissionPage/chromium_element.py b/DrissionPage/chromium_element.py index 5bf21f8..cf12697 100644 --- a/DrissionPage/chromium_element.py +++ b/DrissionPage/chromium_element.py @@ -501,7 +501,7 @@ class ChromiumElement(DrissionElement): return str(path) - def get_screenshot(self, path=None, name=None, as_bytes=None, as_base64=None, scroll_to_center=False): + def get_screenshot(self, path=None, name=None, as_bytes=None, as_base64=None, scroll_to_center=True): """对当前元素截图，可保存到文件，或以字节方式返回 :param path: 文件保存路径 :param name: 完整文件名，后缀可选 'jpg','jpeg','png','webp' @@ -516,7 +516,6 @@ class ChromiumElement(DrissionElement): end_time = perf_counter() + self.page.timeout while not self.run_js(js) and perf_counter() < end_time: sleep(.1) - if scroll_to_center: self.scroll.to_see(center=True) @@ -526,6 +525,7 @@ class ChromiumElement(DrissionElement): right_bottom = (left + width, top + height) if not name: name = f'{self.tag}.jpg' + return self.page._get_screenshot(path, name, as_bytes=as_bytes, as_base64=as_base64, full_page=False, left_top=left_top, right_bottom=right_bottom, ele=self) diff --git a/DrissionPage/chromium_element.pyi b/DrissionPage/chromium_element.pyi index 68a455a..03355d2 100644 --- a/DrissionPage/chromium_element.pyi +++ b/DrissionPage/chromium_element.pyi @@ -183,7 +183,7 @@ class ChromiumElement(DrissionElement): def save(self, path: [str, bool] = None, name: str = None, timeout: float = None) -> str: ... def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, - as_base64: [bool, str] = None, scroll_to_center: bool = False) -> Union[str, bytes]: ... + as_base64: [bool, str] = None, scroll_to_center: bool = True) -> Union[str, bytes]: ... def input(self, vals: Any, clear: bool = True, by_js: bool = False) -> None: ... From ed2cc9a5794832d23d11ec5e6a786d302f6070ea Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 13 Oct 2023 17:44:17 +0800 Subject: [PATCH 026/182] =?UTF-8?q?=E5=88=9D=E6=AD=A5=E5=AE=8C=E6=88=90?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E5=8A=9F=E8=83=BD=E4=BF=AE=E6=94=B9=EF=BC=9B?= =?UTF-8?q?=E8=B0=83=E6=95=B4driver=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/browser_download_manager.py | 276 ++++++++++++++++++++++ DrissionPage/browser_download_manager.pyi | 77 ++++++ DrissionPage/chromium_base.py | 7 +- DrissionPage/chromium_base.pyi | 3 + DrissionPage/chromium_driver.py | 69 ++---- DrissionPage/chromium_driver.pyi | 9 +- DrissionPage/chromium_page.py | 91 +------ DrissionPage/chromium_page.pyi | 28 +-- DrissionPage/chromium_tab.py | 9 +- DrissionPage/chromium_tab.pyi | 4 + DrissionPage/commons/web.py | 4 +- DrissionPage/setter.py | 22 +- DrissionPage/setter.pyi | 10 +- DrissionPage/waiter.py | 141 +++-------- DrissionPage/waiter.pyi | 31 +-- DrissionPage/web_page.py | 2 +- 16 files changed, 464 insertions(+), 319 deletions(-) create mode 100644 DrissionPage/browser_download_manager.py create mode 100644 DrissionPage/browser_download_manager.pyi diff --git a/DrissionPage/browser_download_manager.py b/DrissionPage/browser_download_manager.py new file mode 100644 index 0000000..4614ac2 --- /dev/null +++ b/DrissionPage/browser_download_manager.py @@ -0,0 +1,276 @@ +# -*- coding:utf-8 -*- +from pathlib import Path +from shutil import move +from threading import Lock +from time import sleep, perf_counter + +from .commons.tools import get_usable_path + + +class BrowserDownloadManager(object): + BROWSERS = {} + + def __new__(cls, page): + """ + :param page: ChromiumPage对象 + """ + if page.browser_driver.id in cls.BROWSERS: + return cls.BROWSERS[page.browser_driver.id] + return object.__new__(cls) + + def __init__(self, page): + """ + :param page: ChromiumPage对象 + """ + if page.browser_driver.id in BrowserDownloadManager.BROWSERS: + return + + self._page = page + self._lock = Lock() + self._when_download_file_exists = 'rename' + + t = TabDownloadSettings(page.tab_id) + t.path = page.download_path + self._tabs_settings = {page.tab_id: t} # {tab_id: TabDownloadSettings} + self._missions = {} # {guid: DownloadMission} + self._tab_missions = {} # {tab_id: DownloadMission} + self._guid_and_tab = {} # 记录guid在哪个tab + self._flags = {} # {tab_id: bool, DownloadMission} + + self._page.browser_driver.set_listener('Browser.downloadProgress', self._onDownloadProgress) + self._page.browser_driver.set_listener('Browser.downloadWillBegin', self._onDownloadWillBegin) + + BrowserDownloadManager.BROWSERS[page.browser_driver.id] = self + + @property + def missions(self): + """返回所有未完成的下载任务""" + return self._missions + + def set_path(self, tab_id, path): + """设置某个tab的下载路径 + :param tab_id: tab id + :param path: 下载路径 + :return: None + """ + self._tabs_settings.setdefault(tab_id, TabDownloadSettings(tab_id)).path = str(Path(path).absolute()) + + def set_rename(self, tab_id, rename): + """设置某个tab的重命名文件名 + :param tab_id: tab id + :param rename: 文件名 + :return: None + """ + self._tabs_settings.setdefault(tab_id, TabDownloadSettings(tab_id)).rename = rename + + def set_file_exists(self, tab_id, mode): + """设置某个tab下载文件重名时执行的策略 + :param tab_id: tab id + :param mode: 下载路径 + :return: None + """ + self._tabs_settings.setdefault(tab_id, TabDownloadSettings(tab_id)).when_file_exists = mode + + def set_flag(self, tab_id, flag): + """设置某个tab的重命名文件名 + :param tab_id: tab id + :param flag: 等待标志 + :return: None + """ + self._flags[tab_id] = flag + + def get_flag(self, tab_id): + """获取tab下载等待标记 + :param tab_id: tab id + :return: 任务对象或False + """ + return self._flags.get(tab_id, None) + + def get_tab_missions(self, tab_id): + """获取某个tab正在下载的任务 + :param tab_id: + :return: 下载任务组成的列表 + """ + return self._tab_missions.get(tab_id, []) + + def set_mission(self, tab_id, guid): + """绑定tab和下载任务信息 + :param tab_id: tab id + :param guid: 下载任务id + :return: None + """ + self._guid_and_tab[guid] = tab_id + + def set_done(self, mission, state, cancel=False, final_path=None): + """设置任务结束 + :param mission: 任务对象 + :param state: 任务状态 + :param cancel: 是否取消 + :param final_path: 最终路径 + :return: None + """ + mission.state = state + mission.final_path = final_path + if cancel: + self._page.browser_driver.call_method('Browser.cancelDownload', guid=mission.id) + if mission.final_path: + Path(mission.final_path).unlink(True) + self._missions.pop(mission.id) + + def _onDownloadWillBegin(self, **kwargs): + """用于获取弹出新标签页触发的下载任务""" + guid = kwargs['guid'] + end = perf_counter() + .3 + while perf_counter() < end: + tab_id = self._guid_and_tab.get(guid, None) + if tab_id: + break + sleep(.005) + else: + tab_id = self._page.tab_id + + settings = TabDownloadSettings(tab_id) + if settings.rename: + tmp = kwargs['suggestedFilename'].rsplit('.', 1) + ext_name = tmp[-1] if len(tmp) > 1 else '' + tmp = settings.rename.rsplit('.', 1) + ext_rename = tmp[-1] if len(tmp) > 1 else '' + name = settings.rename if ext_rename == ext_name else f'{settings.rename}.{ext_name}' + settings.rename = None + + else: + name = kwargs['suggestedFilename'] + + skip = False + goal_path = Path(settings.path) / name + if goal_path.exists(): + if settings.when_file_exists == 'skip': + skip = True + elif settings.when_file_exists == 'overwrite': + goal_path.unlink() + + m = DownloadMission(self, tab_id, guid, settings.path, name, kwargs['url']) + self._missions[guid] = m + + if self.get_flag(tab_id) is False: # 取消该任务 + self.set_done(m, 'canceled', True) + elif skip: + self.set_done(m, 'skipped', True) + + self._flags[tab_id] = m + + def _onDownloadProgress(self, **kwargs): + """下载状态变化时执行""" + if kwargs['guid'] in self._missions: + with self._lock: + if kwargs['guid'] in self._missions: + mission = self._missions[kwargs['guid']] + if kwargs['state'] == 'inProgress': + mission.state = 'running' + mission.received_bytes = kwargs['receivedBytes'] + mission.total_bytes = kwargs['totalBytes'] + + elif kwargs['state'] == 'completed': + mission.received_bytes = kwargs['receivedBytes'] + mission.total_bytes = kwargs['totalBytes'] + form_path = f'{self._page.download_path}\\{mission.id}' + to_path = str(get_usable_path(f'{mission.path}\\{mission.name}')) + move(form_path, to_path) + self.set_done(mission, 'completed', final_path=to_path) + + else: + self.set_done(mission, 'canceled') + + +class TabDownloadSettings(object): + TABS = {} + + def __new__(cls, tab_id): + """ + :param tab_id: tab id + """ + if tab_id in cls.TABS: + return cls.TABS[tab_id] + return object.__new__(cls) + + def __init__(self, tab_id): + """ + :param tab_id: tab id + """ + self.tab_id = tab_id + self.rename = None + self.path = '' + self.when_file_exists = 'rename' + + +class DownloadMission(object): + def __init__(self, mgr, tab_id, _id, path, name, url): + self._mgr = mgr + self.url = url + self.tab_id = tab_id + self.id = _id + self.path = path + self.name = name + self.state = 'waiting' + self.total_bytes = None + self.received_bytes = 0 + self.final_path = None + + def __repr__(self): + # return f'' + return f'' + + @property + def rate(self): + """以百分比形式返回下载进度""" + return round((self.received_bytes / self.total_bytes) * 100, 2) if self.total_bytes else None + + def cancel(self): + """取消该任务，如任务已完成，删除已下载的文件""" + self._mgr.set_done(self, state='canceled', cancel=True) + + def wait(self, show=True, timeout=None, cancel_if_timeout=True): + """等待任务结束 + :param show: 是否显示下载信息 + :param timeout: 超时时间，为None则无限等待 + :param cancel_if_timeout: 超时时是否取消任务 + :return: 等待成功返回完整路径，否则返回False + """ + if show: + print(f'url：{self.url}') + t2 = perf_counter() + while self.name is None and perf_counter() - t2 < 4: + sleep(0.01) + print(f'文件名：{self.name}') + print(f'目标路径：{self.path}') + + if timeout is None: + while self.id in self._mgr.missions: + if show: + print(f'\r{self.rate}% ', end='') + sleep(.2) + + else: + running = True + end_time = perf_counter() + timeout + while perf_counter() < end_time: + if show: + print(f'\r{self.rate}% ', end='') + if self.id not in self._mgr.missions: + running = False + break + sleep(.2) + + if running and cancel_if_timeout: + self.cancel() + + if show: + if self.state == 'completed': + print(f'下载完成 {self.final_path}') + elif self.state == 'canceled': + print(f'下载取消') + elif self.state == 'skipped': + print(f'已跳过') + print() + + return self.final_path if self.final_path else False diff --git a/DrissionPage/browser_download_manager.pyi b/DrissionPage/browser_download_manager.pyi new file mode 100644 index 0000000..eb21561 --- /dev/null +++ b/DrissionPage/browser_download_manager.pyi @@ -0,0 +1,77 @@ +from pathlib import Path +from threading import Lock +from typing import Dict, Optional, Union + +from chromium_base import ChromiumBase +from chromium_page import ChromiumPage + + +class BrowserDownloadManager(object): + BROWSERS: Dict[str, BrowserDownloadManager] = ... + _page: ChromiumPage = ... + _lock: Lock = ... + _missions: Dict[str, DownloadMission] = ... + _tab_missions: dict = ... + _tabs_settings: Dict[str, TabDownloadSettings] = ... + _guid_and_tab: Dict[str, str] = ... + _flags: dict = ... + + def __new__(cls, page: ChromiumPage): ... + + def __init__(self, page: ChromiumPage): ... + + @property + def missions(self) -> Dict[str, DownloadMission]: ... + + def set_path(self, tab_id: str, path: Union[Path, str]) -> None: ... + + def set_rename(self, tab_id: str, rename: str) -> None: ... + + def set_file_exists(self, tab_id: str, mode: str) -> None: ... + + def set_flag(self, tab_id: str, flag: Optional[bool, DownloadMission]) -> None: ... + + def get_flag(self, tab_id: str) -> Optional[bool, DownloadMission]: ... + + def get_tab_missions(self, tab_id: str) -> list: ... + + def set_mission(self, tab_id: str, guid: str) -> None: ... + + def set_done(self, mission: DownloadMission, state: str, cancel: bool = False, final_path: str = None) -> None: ... + + def _onDownloadWillBegin(self, **kwargs) -> None: ... + + def _onDownloadProgress(self, **kwargs) -> None: ... + + +class TabDownloadSettings(object): + TABS: dict = ... + tab_id: str = ... + waiting_flag: Optional[bool, dict] = ... + rename: Optional[str] = ... + path: Optional[str] = ... + when_file_exists: str = ... + + def __init__(self, tab_id: str): ... + + +class DownloadMission(object): + tab: ChromiumBase = ... + _mgr: BrowserDownloadManager = ... + url: str = ... + id: str = ... + path: str = ... + name: str = ... + state: str = ... + total_bytes: Optional[int] = ... + received_bytes: int = ... + final_path: Optional[str] = ... + + def __init__(self, mgr: BrowserDownloadManager, tab_id: str, _id: str, path: str, name: str, url: str): ... + + @property + def rate(self) -> float: ... + + def cancel(self) -> None: ... + + def wait(self, show: bool = True, timeout=None, cancel_if_timeout=True) -> Union[bool, str]: ... diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index 80f2d13..1fb3cc2 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -108,7 +108,6 @@ class ChromiumBase(BasePage): self._is_loading = True self._tab_obj = ChromiumDriver(tab_id=tab_id, tab_type='page', address=self.address) - self._tab_obj.start() self._tab_obj.call_method('DOM.enable') self._tab_obj.call_method('Page.enable') @@ -251,7 +250,7 @@ class ChromiumBase(BasePage): def _onDownloadWillBegin(self, **kwargs): """下载即将开始时执行""" - self._page._dl_mgr.set_mission(self.tab_id, kwargs['guid']) + self.browser._dl_mgr.set_mission(self.tab_id, kwargs['guid']) def __call__(self, loc_or_str, timeout=None): """在内部查找元素 @@ -263,7 +262,7 @@ class ChromiumBase(BasePage): return self.ele(loc_or_str, timeout) @property - def page(self): + def browser(self): return self._page @property @@ -324,7 +323,7 @@ class ChromiumBase(BasePage): @property def _target_id(self): """返回当前标签页id""" - return self.driver.id if self.driver.status == 'started' else '' + return self.driver.id if not self.driver._stopped.is_set() else '' @property def ready_state(self): diff --git a/DrissionPage/chromium_base.pyi b/DrissionPage/chromium_base.pyi index 82ed6a2..db2da7e 100644 --- a/DrissionPage/chromium_base.pyi +++ b/DrissionPage/chromium_base.pyi @@ -83,6 +83,9 @@ class ChromiumBase(BasePage): def __call__(self, loc_or_str: Union[Tuple[str, str], str, ChromiumElement], timeout: float = None) -> ChromiumElement: ... + @property + def browser(self) -> ChromiumPage: ... + @property def title(self) -> str: ... diff --git a/DrissionPage/chromium_driver.py b/DrissionPage/chromium_driver.py index b638a0e..611a767 100644 --- a/DrissionPage/chromium_driver.py +++ b/DrissionPage/chromium_driver.py @@ -12,10 +12,6 @@ from websocket import WebSocketTimeoutException, WebSocketException, WebSocketCo class ChromiumDriver(object): - _INITIAL_ = 'initial' - _STARTED_ = 'started' - _STOPPED_ = 'stopped' - def __init__(self, tab_id, tab_type, address): """ :param tab_id: 标签页id @@ -38,13 +34,13 @@ class ChromiumDriver(object): self._handle_event_th.daemon = True self._stopped = Event() - self._started = False - self.status = self._INITIAL_ self.event_handlers = {} self.method_results = {} self.event_queue = Queue() + self.start() + def _send(self, message, timeout=None): """发送信息到浏览器，并返回浏览器返回的信息 :param message: 发送给浏览器的数据 @@ -105,8 +101,8 @@ class ChromiumDriver(object): while not self._stopped.is_set(): try: self._ws.settimeout(1) - message_json = self._ws.recv() - mes = loads(message_json) + msg_json = self._ws.recv() + msg = loads(msg_json) except WebSocketTimeoutException: continue except (WebSocketException, OSError, WebSocketConnectionClosedException): @@ -114,24 +110,23 @@ class ChromiumDriver(object): return if self._debug: - if self._debug is True or 'id' in mes or (isinstance(self._debug, str) - and mes.get('method', '').startswith(self._debug)): - print(f'<收 {message_json}') + if self._debug is True or 'id' in msg or (isinstance(self._debug, str) + and msg.get('method', '').startswith(self._debug)): + print(f'<收 {msg_json}') elif isinstance(self._debug, (list, tuple, set)): for m in self._debug: - if mes.get('method', '').startswith(m): - print(f'<收 {message_json}') + if msg.get('method', '').startswith(m): + print(f'<收 {msg_json}') break - if "method" in mes: - self.event_queue.put(mes) + if "method" in msg: + self.event_queue.put(msg) - elif "id" in mes: - if mes["id"] in self.method_results: - self.method_results[mes['id']].put(mes) + elif msg.get('id') in self.method_results: + self.method_results[msg['id']].put(msg) elif self._debug: - print(f'未知信息：{mes}') + print(f'未知信息：{msg}') def _handle_event_loop(self): """当接收到浏览器信息，执行已绑定的方法""" @@ -157,10 +152,6 @@ class ChromiumDriver(object): :param kwargs: cdp参数 :return: 执行结果 """ - if not self._started: - self.start() - # raise RuntimeError("不能在启动前调用方法。") - if self._stopped.is_set(): return {'error': 'tab closed', 'type': 'tab_closed'} @@ -178,13 +169,6 @@ class ChromiumDriver(object): def start(self): """启动连接""" - if self._started: - return False - if not self._websocket_url: - raise RuntimeError("已存在另一个连接。") - - self._started = True - self.status = self._STARTED_ self._stopped.clear() self._ws = create_connection(self._websocket_url, enable_multithread=True) self._recv_th.start() @@ -195,10 +179,7 @@ class ChromiumDriver(object): """中断连接""" if self._stopped.is_set(): return False - if not self._started: - return True - self.status = self._STOPPED_ self._stopped.set() if self._ws: self._ws.close() @@ -212,22 +193,12 @@ class ChromiumDriver(object): """绑定cdp event和回调方法 :param event: cdp event :param callback: 绑定到cdp event的回调方法 - :return: 回调方法 + :return: None """ - if not callback: - return self.event_handlers.pop(event, None) - if not callable(callback): - raise RuntimeError("方法不能调用。") - - self.event_handlers[event] = callback - return True - - def get_listener(self, event): - """获取cdp event对应的回调方法 - :param event: cdp event - :return: 回调方法 - """ - return self.event_handlers.get(event, None) + if callback: + self.event_handlers[event] = callback + else: + self.event_handlers.pop(event, None) def __str__(self): return f"" @@ -246,8 +217,8 @@ class BrowserDriver(ChromiumDriver): def __init__(self, tab_id, tab_type, address): if tab_id in BrowserDriver.BROWSERS: return - super().__init__(tab_id, tab_type, address) BrowserDriver.BROWSERS[tab_id] = self + super().__init__(tab_id, tab_type, address) def __repr__(self): return f"" diff --git a/DrissionPage/chromium_driver.pyi b/DrissionPage/chromium_driver.pyi index 152d4c4..9e233a0 100644 --- a/DrissionPage/chromium_driver.pyi +++ b/DrissionPage/chromium_driver.pyi @@ -17,9 +17,6 @@ class GenericAttr(object): class ChromiumDriver(object): - _INITIAL_: str - _STARTED_: str - _STOPPED_: str id: str address: str type: str @@ -31,8 +28,6 @@ class ChromiumDriver(object): _recv_th: Thread _handle_event_th: Thread _stopped: Event - _started: bool - status: str event_handlers: dict method_results: dict event_queue: Queue @@ -53,9 +48,7 @@ class ChromiumDriver(object): def stop(self) -> bool: ... - def set_listener(self, event: str, callback: Union[Callable, None]) -> Union[Callable, None, bool]: ... - - def get_listener(self, event: str) -> Union[Callable, None]: ... + def set_listener(self, event: str, callback: Union[Callable, None]) -> None: ... def __str__(self) -> str: ... diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index a9336f0..ea6a459 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -3,17 +3,13 @@ @Author : g1879 @Contact : g1879@qq.com """ -from pathlib import Path -from shutil import move -from threading import Lock from time import perf_counter, sleep +from .browser_download_manager import BrowserDownloadManager from .chromium_base import ChromiumBase, Timeout -from .chromium_base import handle_download from .chromium_driver import ChromiumDriver, BrowserDriver from .chromium_tab import ChromiumTab from .commons.browser import connect_browser -from .commons.tools import get_usable_path from .configs.chromium_options import ChromiumOptions from .errors import BrowserConnectError from .setter import ChromiumPageSetter @@ -101,7 +97,6 @@ class ChromiumPage(ChromiumBase): ws = self._control_session.get(u).json()['webSocketDebuggerUrl'] self._control_session.get(u, headers={'Connection': 'close'}) self._browser_driver = BrowserDriver(ws.split('/')[-1], 'browser', self.address) - self._browser_driver.start() self._alert = Alert() self._tab_obj.set_listener('Page.javascriptDialogOpening', self._on_alert_open) @@ -447,90 +442,6 @@ class ChromiumTabRect(object): return self._page.browser_driver.call_method('Browser.getWindowForTarget', targetId=self._page.tab_id)['bounds'] -class BrowserDownloadManager(object): - BROWSERS = {} - - def __new__(cls, page): - """ - :param page: ChromiumPage对象 - """ - if page.browser_driver.id in cls.BROWSERS: - return cls.BROWSERS[page.browser_driver.id] - return object.__new__(cls) - - def __init__(self, page): - """ - :param page: ChromiumPage对象 - """ - if page.browser_driver.id in BrowserDownloadManager.BROWSERS: - return - - self._page = page - self._lock = Lock() - page.set.download_path(page.download_path) - self._page.browser_driver.set_listener('Browser.downloadProgress', self._onDownloadProgress) - self._page.browser_driver.set_listener('Browser.downloadWillBegin', self._onDownloadWillBegin) - self._missions = {} - - BrowserDownloadManager.BROWSERS[page.browser_driver.id] = self - - @property - def missions(self): - """返回所有未完成的下载任务""" - return self._missions - - def add_mission(self, mission): - """添加下载任务信息 - :param mission: DownloadMission对象 - :return: None - """ - self._missions[mission.id] = mission - - def set_done(self, mission, state, cancel=False, final_path=None): - """设置任务结束 - :param mission: 任务对象 - :param state: 任务状态 - :param cancel: 是否取消 - :param final_path: 最终路径 - :return: None - """ - mission.state = state - mission.final_path = final_path - if cancel: - self._page.browser_driver.call_method('Browser.cancelDownload', guid=mission.id) - if mission.final_path: - Path(mission.final_path).unlink(True) - self._missions.pop(mission.id) - - def _onDownloadWillBegin(self, **kwargs): - """用于获取弹出新标签页触发的下载任务""" - sleep(.3) - if kwargs['guid'] not in self._missions: - handle_download(self._page, kwargs) - - def _onDownloadProgress(self, **kwargs): - """下载状态变化时执行""" - if kwargs['guid'] in self._missions: - with self._lock: - if kwargs['guid'] in self._missions: - mission = self._missions[kwargs['guid']] - if kwargs['state'] == 'inProgress': - mission.state = 'running' - mission.received_bytes = kwargs['receivedBytes'] - mission.total_bytes = kwargs['totalBytes'] - - elif kwargs['state'] == 'completed': - mission.received_bytes = kwargs['receivedBytes'] - mission.total_bytes = kwargs['totalBytes'] - form_path = f'{self._page.download_path}\\{mission.id}' - to_path = str(get_usable_path(f'{mission.path}\\{mission.name}')) - move(form_path, to_path) - self.set_done(mission, 'completed', final_path=to_path) - - else: - self.set_done(mission, 'canceled') - - class Alert(object): """用于保存alert信息的类""" diff --git a/DrissionPage/chromium_page.pyi b/DrissionPage/chromium_page.pyi index 88d62e0..9347094 100644 --- a/DrissionPage/chromium_page.pyi +++ b/DrissionPage/chromium_page.pyi @@ -3,15 +3,15 @@ @Author : g1879 @Contact : g1879@qq.com """ -from threading import Lock -from typing import Union, Tuple, List, Dict +from typing import Union, Tuple, List +from .browser_download_manager import BrowserDownloadManager from .chromium_base import ChromiumBase from .chromium_driver import ChromiumDriver from .chromium_tab import ChromiumTab from .configs.chromium_options import ChromiumOptions from .setter import ChromiumPageSetter -from .waiter import ChromiumPageWaiter, DownloadMission +from .waiter import ChromiumPageWaiter class ChromiumPage(ChromiumBase): @@ -126,28 +126,6 @@ class ChromiumTabRect(object): def _get_browser_rect(self) -> dict: ... -class BrowserDownloadManager(object): - _page: ChromiumPage = ... - _missions: Dict[str, DownloadMission] = ... - _lock: Lock = ... - BROWSERS: Dict[str, BrowserDownloadManager] = ... - - def __new__(cls, page: ChromiumPage): ... - - def __init__(self, page: ChromiumPage): ... - - @property - def missions(self) -> Dict[str, DownloadMission]: ... - - def add_mission(self, mission: DownloadMission) -> None: ... - - def set_done(self, mission: DownloadMission, state: str, cancel: bool = False, final_path: str = None) -> None: ... - - def _onDownloadWillBegin(self, **kwargs) -> None: ... - - def _onDownloadProgress(self, **kwargs) -> None: ... - - class Alert(object): def __init__(self): diff --git a/DrissionPage/chromium_tab.py b/DrissionPage/chromium_tab.py index c5ca0a4..caad458 100644 --- a/DrissionPage/chromium_tab.py +++ b/DrissionPage/chromium_tab.py @@ -5,6 +5,7 @@ """ from copy import copy +from .waiter import ChromiumTabWaiter from .chromium_base import ChromiumBase from .commons.web import set_session_cookies, set_browser_cookies from .session_page import SessionPage @@ -30,7 +31,6 @@ class ChromiumTab(ChromiumBase): self.retry_interval = self.page.retry_interval self._page_load_strategy = self.page.page_load_strategy self._download_path = self.page.download_path - self._when_download_file_exists = self.page._when_download_file_exists def close(self): """关闭当前标签页""" @@ -53,6 +53,13 @@ class ChromiumTab(ChromiumBase): self._set = TabSetter(self) return self._set + @property + def wait(self): + """返回用于等待的对象""" + if self._wait is None: + self._wait = ChromiumTabWaiter(self) + return self._wait + class WebPageTab(SessionPage, ChromiumTab): def __init__(self, page, tab_id): diff --git a/DrissionPage/chromium_tab.pyi b/DrissionPage/chromium_tab.pyi index 850c1f3..f38f8fa 100644 --- a/DrissionPage/chromium_tab.pyi +++ b/DrissionPage/chromium_tab.pyi @@ -7,6 +7,7 @@ from typing import Union, Tuple, Any, List from requests import Session, Response +from waiter import ChromiumTabWaiter from .chromium_base import ChromiumBase from .chromium_element import ChromiumElement from .chromium_frame import ChromiumFrame @@ -36,6 +37,9 @@ class ChromiumTab(ChromiumBase): @property def set(self) -> TabSetter: ... + @property + def wait(self) -> ChromiumTabWaiter: ... + class WebPageTab(SessionPage, ChromiumTab): def __init__(self, page: WebPage, tab_id: str): diff --git a/DrissionPage/commons/web.py b/DrissionPage/commons/web.py index bad3529..5a438f3 100644 --- a/DrissionPage/commons/web.py +++ b/DrissionPage/commons/web.py @@ -164,7 +164,7 @@ def is_js_func(func): def cookie_to_dict(cookie): """把Cookie对象转为dict格式 - :param cookie: Cookie对象 + :param cookie: Cookie对象、字符串或字典 :return: cookie字典 """ if isinstance(cookie, Cookie): @@ -177,7 +177,7 @@ def cookie_to_dict(cookie): cookie_dict = cookie elif isinstance(cookie, str): - cookie = cookie.split(',' if ',' in cookie else ';') + cookie = cookie.rstrip(';,').split(',' if ',' in cookie else ';') cookie_dict = {} for key, attr in enumerate(cookie): diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index abc6194..c7e0b31 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -126,22 +126,30 @@ class ChromiumBaseSetter(object): if self._page._DownloadKit: self._page._DownloadKit.set.goal_path(path) + +class TabSetter(ChromiumBaseSetter): + def __init__(self, page): + super().__init__(page) + + def download_path(self, path): + """设置下载路径 + :param path: 下载路径 + :return: None + """ + super().download_path(path) + self._page.browser._dl_mgr.set_path(self._page.tab_id, path) + def download_file_name(self, name): """设置下一个被下载文件的名称 :param name: 文件名，可不含后缀 :return: None """ - self._page._download_rename = name + self._page.browser._dl_mgr.set_rename(self._page.tab_id, name) def when_download_file_exists(self, mode): if mode not in ('rename', 'overwrite', 'skip'): raise ValueError(f"mode参数只能是'rename', 'overwrite', 'skip' 之一，现在是：{mode}") - self._page._when_download_file_exists = mode - - -class TabSetter(ChromiumBaseSetter): - def __init__(self, page): - super().__init__(page) + self._page.browser._dl_mgr.set_file_exists(self._page.tab_id, mode) class ChromiumPageSetter(ChromiumBaseSetter): diff --git a/DrissionPage/setter.pyi b/DrissionPage/setter.pyi index 39981d2..26130cb 100644 --- a/DrissionPage/setter.pyi +++ b/DrissionPage/setter.pyi @@ -52,14 +52,16 @@ class ChromiumBaseSetter(object): def download_path(self, path: Union[str, Path]) -> None: ... - def download_file_name(self, name: str) -> None: ... - - def when_download_file_exists(self, mode: str) -> None: ... - class TabSetter(ChromiumBaseSetter): def __init__(self, page): ... + def download_path(self, path: Union[str, Path]) -> None: ... + + def download_file_name(self, name: str) -> None: ... + + def when_download_file_exists(self, mode: str) -> None: ... + class ChromiumPageSetter(ChromiumBaseSetter): _page: ChromiumPage = ... diff --git a/DrissionPage/waiter.py b/DrissionPage/waiter.py index e693873..7075ae2 100644 --- a/DrissionPage/waiter.py +++ b/DrissionPage/waiter.py @@ -1,5 +1,4 @@ # -*- coding:utf-8 -*- -from pathlib import Path from time import sleep, perf_counter from .commons.constants import Settings @@ -90,48 +89,23 @@ class ChromiumBaseWaiter(object): """等待浏览器下载开始，可将其拦截 :param timeout: 超时时间，None使用页面对象超时时间 :param cancel_it: 是否取消该任务 - :return: 成功返回任务信息dict，失败返回False + :return: 成功返回任务对象，失败返回False """ - self._driver._wait_download_flag = False if cancel_it else True + self._driver.browser._dl_mgr.set_flag(self._driver.tab_id, False if cancel_it else True) if timeout is None: timeout = self._driver.timeout r = False end_time = perf_counter() + timeout while perf_counter() < end_time: - if not isinstance(self._driver._wait_download_flag, bool): - r = self._driver._wait_download_flag + v = self._driver.browser._dl_mgr.get_flag(self._driver.tab_id) + if not isinstance(v, bool): + r = v break - self._driver._wait_download_flag = None + self._driver.browser._dl_mgr.set_flag(self._driver.tab_id, None) return r - def downloads_done(self, timeout=None, cancel_if_timeout=True): - """等待所有浏览器下载任务结束 - :param timeout: 超时时间，为None时无限等待 - :param cancel_if_timeout: 超时时是否取消剩余任务 - :return: 是否等待成功 - """ - if not timeout: - while self._driver._download_missions: - sleep(.5) - return True - - else: - end_time = perf_counter() + timeout - while end_time > perf_counter(): - if not self._driver._download_missions: - return True - sleep(.5) - - if self._driver._download_missions: - if cancel_if_timeout: - for m in self._driver._download_missions: - m.cancel() - return False - else: - return True - def url_change(self, text, exclude=False, timeout=None, raise_err=None): """等待url变成包含或不包含指定文本 :param text: 用于识别的文本 @@ -204,7 +178,36 @@ class ChromiumBaseWaiter(object): return False -class ChromiumPageWaiter(ChromiumBaseWaiter): +class ChromiumTabWaiter(ChromiumBaseWaiter): + + def downloads_done(self, timeout=None, cancel_if_timeout=True): + """等待所有浏览器下载任务结束 + :param timeout: 超时时间，为None时无限等待 + :param cancel_if_timeout: 超时时是否取消剩余任务 + :return: 是否等待成功 + """ + if not timeout: + while self._driver.browser._dl_mgr.get_tab_missions(self._driver.tab_id): + sleep(.5) + return True + + else: + end_time = perf_counter() + timeout + while end_time > perf_counter(): + if not self._driver.browser._dl_mgr.get_tab_missions(self._driver.tab_id): + return True + sleep(.5) + + if self._driver.browser._dl_mgr.get_tab_missions(self._driver.tab_id): + if cancel_if_timeout: + for m in self._driver.browser._dl_mgr.get_tab_missions(self._driver.tab_id): + m.cancel() + return False + else: + return True + + +class ChromiumPageWaiter(ChromiumTabWaiter): def __init__(self, page): super().__init__(page) # self._listener = None @@ -376,75 +379,3 @@ class FrameWaiter(ChromiumBaseWaiter, ChromiumElementWaiter): """ super().__init__(frame) super(ChromiumBaseWaiter, self).__init__(frame, frame.frame_ele) - - -class DownloadMission(object): - def __init__(self, tab, _id, path, name, url): - self.url = url - self.tab = tab - self.id = _id - self.path = path - self.name = name - self.state = 'waiting' - self.total_bytes = None - self.received_bytes = 0 - self.final_path = None - - def __repr__(self): - # return f'' - return f'' - - @property - def rate(self): - """以百分比形式返回下载进度""" - return round((self.received_bytes / self.total_bytes) * 100, 2) if self.total_bytes else None - - def cancel(self): - """取消该任务，如任务已完成，删除已下载的文件""" - self.tab._page._dl_mgr.set_done(self, state='canceled', cancel=True) - - def wait(self, show=True, timeout=None, cancel_if_timeout=True): - """等待任务结束 - :param show: 是否显示下载信息 - :param timeout: 超时时间，为None则无限等待 - :param cancel_if_timeout: 超时时是否取消任务 - :return: 等待成功返回完整路径，否则返回False - """ - if show: - print(f'url：{self.url}') - t2 = perf_counter() - while self.name is None and perf_counter() - t2 < 4: - sleep(0.01) - print(f'文件名：{self.name}') - print(f'目标路径：{self.path}') - - if timeout is None: - while self.id in self.tab._page._dl_mgr.missions: - if show: - print(f'\r{self.rate}% ', end='') - sleep(.2) - - else: - running = True - end_time = perf_counter() + timeout - while perf_counter() < end_time: - if show: - print(f'\r{self.rate}% ', end='') - if self.id not in self.tab._page._dl_mgr.missions: - running = False - break - sleep(.2) - - if running and cancel_if_timeout: - self.cancel() - - if show: - if self.state == 'completed': - print(f'下载完成 {self.final_path}') - elif self.state == 'canceled': - print(f'下载取消') - elif self.state == 'skipped': - print(f'已跳过') - print() - - return self.final_path if self.final_path else False diff --git a/DrissionPage/waiter.pyi b/DrissionPage/waiter.pyi index ad73281..823ee3b 100644 --- a/DrissionPage/waiter.pyi +++ b/DrissionPage/waiter.pyi @@ -3,8 +3,9 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union, Optional +from typing import Union +from .browser_download_manager import DownloadMission from .chromium_base import ChromiumBase from .chromium_element import ChromiumElement from .chromium_frame import ChromiumFrame @@ -49,7 +50,12 @@ class ChromiumBaseWaiter(object): raise_err: bool = None) -> bool: ... -class ChromiumPageWaiter(ChromiumBaseWaiter): +class ChromiumTabWaiter(ChromiumBaseWaiter): + + def downloads_done(self, timeout: float = None, cancel_if_timeout: bool = True) -> bool: ... + + +class ChromiumPageWaiter(ChromiumTabWaiter): _driver: ChromiumPage = ... def new_tab(self, timeout: float = None, raise_err: bool = None) -> bool: ... @@ -87,24 +93,3 @@ class ChromiumElementWaiter(object): class FrameWaiter(ChromiumBaseWaiter, ChromiumElementWaiter): def __init__(self, frame: ChromiumFrame): ... - - -class DownloadMission(object): - tab: ChromiumBase = ... - url: str = ... - id: str = ... - path: str = ... - name: str = ... - state: str = ... - total_bytes: Optional[int] = ... - received_bytes: int = ... - final_path: Optional[str] = ... - - def __init__(self, tab: ChromiumBase, _id: str, path: str, name: str, url: str): ... - - @property - def rate(self) -> float: ... - - def cancel(self) -> None: ... - - def wait(self, show: bool = True, timeout=None, cancel_if_timeout=True) -> Union[bool, str]: ... diff --git a/DrissionPage/web_page.py b/DrissionPage/web_page.py index ec8a872..b506c85 100644 --- a/DrissionPage/web_page.py +++ b/DrissionPage/web_page.py @@ -27,6 +27,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): :param driver_or_options: ChromiumDriver对象，只使用s模式时应传入False :param session_or_options: Session对象或SessionOptions对象，只使用d模式时应传入False """ + super(ChromiumBase, self).__init__() # 调用Base的__init__() self._mode = mode.lower() if self._mode not in ('s', 'd'): raise ValueError('mode参数只能是s或d。') @@ -51,7 +52,6 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self._create_session() t = timeout if isinstance(timeout, (int, float)) else self.timeouts.implicit - super(ChromiumBase, self).__init__(t) # 调用Base的__init__() def _set_start_options(self, dr_opt, se_opt): """处理两种模式的设置 From a37ea0a50dee872fec3cc6e43ba95fddcaad5eb2 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 16 Oct 2023 00:14:17 +0800 Subject: [PATCH 027/182] =?UTF-8?q?=E6=94=B9=E8=BF=9B=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=EF=BC=8C=E5=BE=85=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/base.py | 5 +++-- DrissionPage/browser_download_manager.py | 19 +++++++++++++++---- DrissionPage/chromium_base.py | 2 +- DrissionPage/chromium_page.py | 3 ++- DrissionPage/setter.py | 18 +----------------- DrissionPage/setter.pyi | 4 ---- 6 files changed, 22 insertions(+), 29 deletions(-) diff --git a/DrissionPage/base.py b/DrissionPage/base.py index c7b89d8..48880a6 100644 --- a/DrissionPage/base.py +++ b/DrissionPage/base.py @@ -4,6 +4,7 @@ @Contact : g1879@qq.com """ from abc import abstractmethod +from pathlib import Path from re import sub from urllib.parse import quote @@ -366,7 +367,7 @@ class BasePage(BaseParser): self.retry_times = 3 self.retry_interval = 2 self._DownloadKit = None - self._download_path = '' + self._download_path = str(Path('.').absolute()) @property def title(self): @@ -403,7 +404,7 @@ class BasePage(BaseParser): def download(self): """返回下载器对象""" if self._DownloadKit is None: - self._DownloadKit = DownloadKit(session=self, goal_path=self.download_path) + self._DownloadKit = DownloadKit(driver=self, goal_path=self.download_path) return self._DownloadKit def _before_connect(self, url, retry, interval): diff --git a/DrissionPage/browser_download_manager.py b/DrissionPage/browser_download_manager.py index 4614ac2..e7df58e 100644 --- a/DrissionPage/browser_download_manager.py +++ b/DrissionPage/browser_download_manager.py @@ -22,8 +22,9 @@ class BrowserDownloadManager(object): """ :param page: ChromiumPage对象 """ - if page.browser_driver.id in BrowserDownloadManager.BROWSERS: + if hasattr(self, '_created'): return + self._created = True self._page = page self._lock = Lock() @@ -39,6 +40,9 @@ class BrowserDownloadManager(object): self._page.browser_driver.set_listener('Browser.downloadProgress', self._onDownloadProgress) self._page.browser_driver.set_listener('Browser.downloadWillBegin', self._onDownloadWillBegin) + self._page.browser_driver.call_method('Browser.setDownloadBehavior', + downloadPath=self._page.download_path, + behavior='allowAndName', eventsEnabled=True) BrowserDownloadManager.BROWSERS[page.browser_driver.id] = self @@ -54,6 +58,9 @@ class BrowserDownloadManager(object): :return: None """ self._tabs_settings.setdefault(tab_id, TabDownloadSettings(tab_id)).path = str(Path(path).absolute()) + self._page.browser_driver.call_method('Browser.setDownloadBehavior', + downloadPath=str(Path(path).absolute()), + behavior='allowAndName', eventsEnabled=True) def set_rename(self, tab_id, rename): """设置某个tab的重命名文件名 @@ -120,7 +127,7 @@ class BrowserDownloadManager(object): def _onDownloadWillBegin(self, **kwargs): """用于获取弹出新标签页触发的下载任务""" guid = kwargs['guid'] - end = perf_counter() + .3 + end = perf_counter() + .5 while perf_counter() < end: tab_id = self._guid_and_tab.get(guid, None) if tab_id: @@ -178,7 +185,7 @@ class BrowserDownloadManager(object): move(form_path, to_path) self.set_done(mission, 'completed', final_path=to_path) - else: + else: # canceled self.set_done(mission, 'canceled') @@ -197,11 +204,16 @@ class TabDownloadSettings(object): """ :param tab_id: tab id """ + if hasattr(self, '_created'): + return + self._created = True self.tab_id = tab_id self.rename = None self.path = '' self.when_file_exists = 'rename' + TabDownloadSettings.TABS[tab_id] = self + class DownloadMission(object): def __init__(self, mgr, tab_id, _id, path, name, url): @@ -217,7 +229,6 @@ class DownloadMission(object): self.final_path = None def __repr__(self): - # return f'' return f'' @property diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index 1fb3cc2..7179964 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -48,7 +48,7 @@ class ChromiumBase(BasePage): self._actions = None self._listener = None - self._download_path = '' + self._download_path = str(Path('.').absolute()) if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): address = f'127.0.0.1:{address}' diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index ea6a459..8e39bca 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -3,6 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ +from pathlib import Path from time import perf_counter, sleep from .browser_download_manager import BrowserDownloadManager @@ -65,7 +66,7 @@ class ChromiumPage(ChromiumBase): if self._driver_options.timeouts['implicit'] is not None: self._timeout = self._driver_options.timeouts['implicit'] self._page_load_strategy = self._driver_options.page_load_strategy - self._download_path = self._driver_options.download_path + self._download_path = str(Path(self._driver_options.download_path).absolute()) def _connect_browser(self, tab_id=None): """连接浏览器，在第一次时运行 diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index c7e0b31..ad17d0a 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -123,6 +123,7 @@ class ChromiumBaseSetter(object): :return: None """ self._page._download_path = str(Path(path).absolute()) + self._page.browser._dl_mgr.set_path(self._page.tab_id, path) if self._page._DownloadKit: self._page._DownloadKit.set.goal_path(path) @@ -131,14 +132,6 @@ class TabSetter(ChromiumBaseSetter): def __init__(self, page): super().__init__(page) - def download_path(self, path): - """设置下载路径 - :param path: 下载路径 - :return: None - """ - super().download_path(path) - self._page.browser._dl_mgr.set_path(self._page.tab_id, path) - def download_file_name(self, name): """设置下一个被下载文件的名称 :param name: 文件名，可不含后缀 @@ -176,15 +169,6 @@ class ChromiumPageSetter(ChromiumBaseSetter): tab_or_id = tab_or_id.tab_id self._page._control_session.get(f'http://{self._page.address}/json/activate/{tab_or_id}') - def download_path(self, path): - """设置下载路径 - :param path: 下载路径 - :return: None - """ - super().download_path(path) - self._page.browser_driver.call_method('Browser.setDownloadBehavior', downloadPath=self._page.download_path, - behavior='allowAndName', eventsEnabled=True) - class SessionPageSetter(object): def __init__(self, page): diff --git a/DrissionPage/setter.pyi b/DrissionPage/setter.pyi index 26130cb..16bc234 100644 --- a/DrissionPage/setter.pyi +++ b/DrissionPage/setter.pyi @@ -56,8 +56,6 @@ class ChromiumBaseSetter(object): class TabSetter(ChromiumBaseSetter): def __init__(self, page): ... - def download_path(self, path: Union[str, Path]) -> None: ... - def download_file_name(self, name: str) -> None: ... def when_download_file_exists(self, mode: str) -> None: ... @@ -73,8 +71,6 @@ class ChromiumPageSetter(ChromiumBaseSetter): def tab_to_front(self, tab_or_id: Union[str, ChromiumTab] = None) -> None: ... - def download_path(self, path: Union[str, Path]) -> None: ... - class SessionPageSetter(object): def __init__(self, page: SessionPage): From 66993c3905ff4404b482766dd9f993c2e160181c Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 16 Oct 2023 06:39:26 +0000 Subject: [PATCH 028/182] add .gitee/ISSUE_TEMPLATE.zh-CN.md. --- .gitee/ISSUE_TEMPLATE.zh-CN.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitee/ISSUE_TEMPLATE.zh-CN.md diff --git a/.gitee/ISSUE_TEMPLATE.zh-CN.md b/.gitee/ISSUE_TEMPLATE.zh-CN.md new file mode 100644 index 0000000..1c9e53c --- /dev/null +++ b/.gitee/ISSUE_TEMPLATE.zh-CN.md @@ -0,0 +1,3 @@ +- 使用上的问题请先查看文档[使用文档](http://g1879.gitee.io/drissionpagedocs) +- 遇到bug请详细描述如何重现，并附上代码 +- 提问前先给本库打个星，谢谢 \ No newline at end of file From 36590206fb568534198a66e2e21fdf94dc70d12d Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 16 Oct 2023 18:03:48 +0800 Subject: [PATCH 029/182] =?UTF-8?q?=E6=94=B9=E8=BF=9B=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/browser_download_manager.py | 16 +++++++++++++--- DrissionPage/browser_download_manager.pyi | 5 ++++- DrissionPage/setter.py | 12 ++++++------ DrissionPage/setter.pyi | 6 +++--- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/DrissionPage/browser_download_manager.py b/DrissionPage/browser_download_manager.py index e7df58e..ce23f34 100644 --- a/DrissionPage/browser_download_manager.py +++ b/DrissionPage/browser_download_manager.py @@ -58,9 +58,10 @@ class BrowserDownloadManager(object): :return: None """ self._tabs_settings.setdefault(tab_id, TabDownloadSettings(tab_id)).path = str(Path(path).absolute()) - self._page.browser_driver.call_method('Browser.setDownloadBehavior', - downloadPath=str(Path(path).absolute()), - behavior='allowAndName', eventsEnabled=True) + if tab_id == self._page.tab_id: + self._page.browser_driver.call_method('Browser.setDownloadBehavior', + downloadPath=str(Path(path).absolute()), + behavior='allowAndName', eventsEnabled=True) def set_rename(self, tab_id, rename): """设置某个tab的重命名文件名 @@ -122,6 +123,8 @@ class BrowserDownloadManager(object): self._page.browser_driver.call_method('Browser.cancelDownload', guid=mission.id) if mission.final_path: Path(mission.final_path).unlink(True) + if mission.tab_id in self._tab_missions: + self._tab_missions[mission.tab_id].remove(mission.id) self._missions.pop(mission.id) def _onDownloadWillBegin(self, **kwargs): @@ -163,6 +166,8 @@ class BrowserDownloadManager(object): self.set_done(m, 'canceled', True) elif skip: self.set_done(m, 'skipped', True) + else: + self._tab_missions.setdefault(tab_id, []).append(guid) self._flags[tab_id] = m @@ -236,6 +241,11 @@ class DownloadMission(object): """以百分比形式返回下载进度""" return round((self.received_bytes / self.total_bytes) * 100, 2) if self.total_bytes else None + @property + def is_done(self): + """返回任务是否在运行中""" + return self.state == 'completed' + def cancel(self): """取消该任务，如任务已完成，删除已下载的文件""" self._mgr.set_done(self, state='canceled', cancel=True) diff --git a/DrissionPage/browser_download_manager.pyi b/DrissionPage/browser_download_manager.pyi index eb21561..bcc1c0e 100644 --- a/DrissionPage/browser_download_manager.pyi +++ b/DrissionPage/browser_download_manager.pyi @@ -56,7 +56,7 @@ class TabDownloadSettings(object): class DownloadMission(object): - tab: ChromiumBase = ... + tab_id: str = ... _mgr: BrowserDownloadManager = ... url: str = ... id: str = ... @@ -72,6 +72,9 @@ class DownloadMission(object): @property def rate(self) -> float: ... + @property + def is_done(self) -> bool: ... + def cancel(self) -> None: ... def wait(self, show: bool = True, timeout=None, cancel_if_timeout=True) -> Union[bool, str]: ... diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index ad17d0a..2298124 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -117,6 +117,11 @@ class ChromiumBaseSetter(object): self._page.run_cdp('Network.enable') self._page.run_cdp('Network.setExtraHTTPHeaders', headers=headers) + +class TabSetter(ChromiumBaseSetter): + def __init__(self, page): + super().__init__(page) + def download_path(self, path): """设置下载路径 :param path: 下载路径 @@ -127,11 +132,6 @@ class ChromiumBaseSetter(object): if self._page._DownloadKit: self._page._DownloadKit.set.goal_path(path) - -class TabSetter(ChromiumBaseSetter): - def __init__(self, page): - super().__init__(page) - def download_file_name(self, name): """设置下一个被下载文件的名称 :param name: 文件名，可不含后缀 @@ -145,7 +145,7 @@ class TabSetter(ChromiumBaseSetter): self._page.browser._dl_mgr.set_file_exists(self._page.tab_id, mode) -class ChromiumPageSetter(ChromiumBaseSetter): +class ChromiumPageSetter(TabSetter): def main_tab(self, tab_id=None): """设置主tab :param tab_id: 标签页id，不传入则设置当前tab diff --git a/DrissionPage/setter.pyi b/DrissionPage/setter.pyi index 16bc234..8e1aac4 100644 --- a/DrissionPage/setter.pyi +++ b/DrissionPage/setter.pyi @@ -50,18 +50,18 @@ class ChromiumBaseSetter(object): def upload_files(self, files: Union[str, list, tuple]) -> None: ... - def download_path(self, path: Union[str, Path]) -> None: ... - class TabSetter(ChromiumBaseSetter): def __init__(self, page): ... + def download_path(self, path: Union[str, Path]) -> None: ... + def download_file_name(self, name: str) -> None: ... def when_download_file_exists(self, mode: str) -> None: ... -class ChromiumPageSetter(ChromiumBaseSetter): +class ChromiumPageSetter(TabSetter): _page: ChromiumPage = ... def main_tab(self, tab_id: str = None) -> None: ... From 30f022fe71907f83bca79f33dab18b4a32bcfcc2 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 16 Oct 2023 22:50:12 +0800 Subject: [PATCH 030/182] =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E5=90=8C=E5=90=8D?= =?UTF-8?q?=E5=A4=84=E7=90=86=E7=AD=96=E7=95=A5=E6=94=AF=E6=8C=81=E7=AE=80?= =?UTF-8?q?=E5=86=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitee/ISSUE_TEMPLATE.zh-CN.md | 6 +++--- DrissionPage/setter.py | 12 ++++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.gitee/ISSUE_TEMPLATE.zh-CN.md b/.gitee/ISSUE_TEMPLATE.zh-CN.md index 1c9e53c..fc92e66 100644 --- a/.gitee/ISSUE_TEMPLATE.zh-CN.md +++ b/.gitee/ISSUE_TEMPLATE.zh-CN.md @@ -1,3 +1,3 @@ -- 使用上的问题请先查看文档[使用文档](http://g1879.gitee.io/drissionpagedocs) -- 遇到bug请详细描述如何重现，并附上代码 -- 提问前先给本库打个星，谢谢 \ No newline at end of file +1. 使用上的问题请先查看文档[使用文档](http://g1879.gitee.io/drissionpagedocs) +2. 遇到bug请详细描述如何重现，并附上代码 +3. 提问前先给本库打个星，谢谢 \ No newline at end of file diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index 2298124..5677012 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -140,8 +140,16 @@ class TabSetter(ChromiumBaseSetter): self._page.browser._dl_mgr.set_rename(self._page.tab_id, name) def when_download_file_exists(self, mode): - if mode not in ('rename', 'overwrite', 'skip'): - raise ValueError(f"mode参数只能是'rename', 'overwrite', 'skip' 之一，现在是：{mode}") + """设置当存在同名文件时的处理方式 + :param mode: 可在 'rename', 'overwrite', 'skip', 'r', 'o', 's'中选择 + :return: None + """ + types = {'rename': 'rename', 'overwrite': 'overwrite', 'skip': 'skip', 'r': 'rename', 'o': 'overwrite', + 's': 'skip'} + mode = types.get(mode, None) + if not mode: + raise ValueError(f"mode参数只能是'rename', 'overwrite', 'skip', 'r', 'o', 's' 之一，现在是：{mode}") + self._page.browser._dl_mgr.set_file_exists(self._page.tab_id, mode) From ba1c3fb3cceadb922616fbb7668e824e8aac7d60 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 17 Oct 2023 10:16:09 +0800 Subject: [PATCH 031/182] =?UTF-8?q?new=5Ftab()=E6=94=B9=E4=B8=BA=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=E6=A0=87=E7=AD=BE=E9=A1=B5=E5=AF=B9=E8=B1=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_page.py | 13 ++++++++++--- DrissionPage/chromium_page.pyi | 6 ++++-- DrissionPage/web_page.py | 13 +++++++++---- DrissionPage/web_page.pyi | 6 +++--- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 8e39bca..58215d3 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -172,8 +172,7 @@ class ChromiumPage(ChromiumBase): :param tab_id: 要获取的标签页id，为None时获取当前tab :return: 标签页对象 """ - tab_id = tab_id or self.tab_id - return ChromiumTab(self, tab_id) + return tab_id if isinstance(tab_id, ChromiumTab) else ChromiumTab(self, tab_id or self.tab_id) def find_tabs(self, title=None, url=None, tab_type=None, single=True): """查找符合条件的tab，返回它们的id组成的列表 @@ -197,7 +196,7 @@ class ChromiumPage(ChromiumBase): and (tab_type is None or i['type'] in tab_type))] return r[0]['id'] if r and single else r - def new_tab(self, url=None, switch_to=False): + def _new_tab(self, url=None, switch_to=False): """新建一个标签页,该标签页在最后面 :param url: 新标签页跳转到的网址 :param switch_to: 新建标签页后是否把焦点移过去 @@ -226,6 +225,14 @@ class ChromiumPage(ChromiumBase): return tid + def new_tab(self, url=None, switch_to=False): + """新建一个标签页,该标签页在最后面 + :param url: 新标签页跳转到的网址 + :param switch_to: 新建标签页后是否把焦点移过去 + :return: 新标签页对象 + """ + return ChromiumTab(self, self._new_tab(url, switch_to)) + def to_main_tab(self): """跳转到主标签页""" self.to_tab(self._main_tab) diff --git a/DrissionPage/chromium_page.pyi b/DrissionPage/chromium_page.pyi index 9347094..497ea44 100644 --- a/DrissionPage/chromium_page.pyi +++ b/DrissionPage/chromium_page.pyi @@ -63,12 +63,14 @@ class ChromiumPage(ChromiumBase): @property def set(self) -> ChromiumPageSetter: ... - def get_tab(self, tab_id: str = None) -> ChromiumTab: ... + def get_tab(self, tab_id: Union[str, ChromiumTab] = None) -> ChromiumTab: ... def find_tabs(self, title: str = None, url: str = None, tab_type: Union[str, list, tuple, set] = None, single: bool = True) -> Union[str, List[str]]: ... - def new_tab(self, url: str = None, switch_to: bool = False) -> str: ... + def _new_tab(self, url: str = None, switch_to: bool = False) -> str: ... + + def new_tab(self, url: str = None, switch_to: bool = False) -> ChromiumTab: ... def to_main_tab(self) -> None: ... diff --git a/DrissionPage/web_page.py b/DrissionPage/web_page.py index b506c85..2d97b0c 100644 --- a/DrissionPage/web_page.py +++ b/DrissionPage/web_page.py @@ -374,8 +374,15 @@ class WebPage(SessionPage, ChromiumPage, BasePage): :param tab_id: 要获取的标签页id，为None时获取当前tab :return: 标签页对象 """ - tab_id = tab_id or self.tab_id - return WebPageTab(self, tab_id) + return tab_id if isinstance(tab_id, WebPageTab) else WebPageTab(self, tab_id or self.tab_id) + + def new_tab(self, url=None, switch_to=False): + """新建一个标签页,该标签页在最后面 + :param url: 新标签页跳转到的网址 + :param switch_to: 新建标签页后是否把焦点移过去 + :return: 新标签页对象 + """ + return WebPageTab(self, self._new_tab(url, switch_to)) def close_driver(self): """关闭driver及浏览器""" @@ -424,5 +431,3 @@ class WebPage(SessionPage, ChromiumPage, BasePage): super(SessionPage, self).quit() self._tab_obj = None self._has_driver = None - - diff --git a/DrissionPage/web_page.pyi b/DrissionPage/web_page.pyi index 1621fd9..5734631 100644 --- a/DrissionPage/web_page.pyi +++ b/DrissionPage/web_page.pyi @@ -124,7 +124,9 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def get_cookies(self, as_dict: bool = False, all_domains: bool = False, all_info: bool = False) -> Union[dict, list]: ... - def get_tab(self, tab_id: str = None) -> WebPageTab: ... + def get_tab(self, tab_id: Union[str, WebPageTab] = None) -> WebPageTab: ... + + def new_tab(self, url: str = None, switch_to: bool = False) -> WebPageTab: ... def close_driver(self) -> None: ... @@ -165,5 +167,3 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def quit(self) -> None: ... def _on_download_begin(self, **kwargs): ... - - From 8dd184e8489a58068f64c17fdc8c8cfb0a2919cc Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 17 Oct 2023 15:48:28 +0800 Subject: [PATCH 032/182] =?UTF-8?q?=E5=BE=AE=E8=B0=83=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_driver.py | 3 ++- DrissionPage/setter.py | 6 +++--- DrissionPage/setter.pyi | 6 ++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/DrissionPage/chromium_driver.py b/DrissionPage/chromium_driver.py index 611a767..05aab26 100644 --- a/DrissionPage/chromium_driver.py +++ b/DrissionPage/chromium_driver.py @@ -215,8 +215,9 @@ class BrowserDriver(ChromiumDriver): return object.__new__(cls) def __init__(self, tab_id, tab_type, address): - if tab_id in BrowserDriver.BROWSERS: + if hasattr(self, '_created'): return + self._created = True BrowserDriver.BROWSERS[tab_id] = self super().__init__(tab_id, tab_type, address) diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index 5677012..54cefe8 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -146,9 +146,9 @@ class TabSetter(ChromiumBaseSetter): """ types = {'rename': 'rename', 'overwrite': 'overwrite', 'skip': 'skip', 'r': 'rename', 'o': 'overwrite', 's': 'skip'} - mode = types.get(mode, None) - if not mode: - raise ValueError(f"mode参数只能是'rename', 'overwrite', 'skip', 'r', 'o', 's' 之一，现在是：{mode}") + mode = types.get(mode, mode) + if mode not in types: + raise ValueError(f'''mode参数只能是 '{"', '".join(types.keys())}' 之一，现在是：{mode}''') self._page.browser._dl_mgr.set_file_exists(self._page.tab_id, mode) diff --git a/DrissionPage/setter.pyi b/DrissionPage/setter.pyi index 8e1aac4..8803ed8 100644 --- a/DrissionPage/setter.pyi +++ b/DrissionPage/setter.pyi @@ -5,7 +5,7 @@ """ from http.cookiejar import Cookie from pathlib import Path -from typing import Union, Tuple +from typing import Union, Tuple, Literal from requests.adapters import HTTPAdapter from requests.auth import HTTPBasicAuth @@ -19,6 +19,8 @@ from .chromium_tab import ChromiumTab from .session_page import SessionPage from .web_page import WebPage +FILE_EXISTS = Literal['skip', 'rename', 'overwrite', 's', 'r', 'o'] + class ChromiumBaseSetter(object): def __init__(self, page): @@ -58,7 +60,7 @@ class TabSetter(ChromiumBaseSetter): def download_file_name(self, name: str) -> None: ... - def when_download_file_exists(self, mode: str) -> None: ... + def when_download_file_exists(self, mode: FILE_EXISTS) -> None: ... class ChromiumPageSetter(TabSetter): From 345c051aa11d5397cb2dafeed9d7341b2f2ef814 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 17 Oct 2023 17:45:36 +0800 Subject: [PATCH 033/182] =?UTF-8?q?=E6=94=B9=E8=BF=9B=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E9=80=BB=E8=BE=91=EF=BC=8C=E5=BE=85=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/browser_download_manager.py | 63 ++++++++++++----------- DrissionPage/browser_download_manager.pyi | 7 ++- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/DrissionPage/browser_download_manager.py b/DrissionPage/browser_download_manager.py index ce23f34..a226822 100644 --- a/DrissionPage/browser_download_manager.py +++ b/DrissionPage/browser_download_manager.py @@ -1,7 +1,6 @@ # -*- coding:utf-8 -*- from pathlib import Path from shutil import move -from threading import Lock from time import sleep, perf_counter from .commons.tools import get_usable_path @@ -27,7 +26,6 @@ class BrowserDownloadManager(object): self._created = True self._page = page - self._lock = Lock() self._when_download_file_exists = 'rename' t = TabDownloadSettings(page.tab_id) @@ -36,7 +34,7 @@ class BrowserDownloadManager(object): self._missions = {} # {guid: DownloadMission} self._tab_missions = {} # {tab_id: DownloadMission} self._guid_and_tab = {} # 记录guid在哪个tab - self._flags = {} # {tab_id: bool, DownloadMission} + self._flags = {} # {tab_id: [bool, DownloadMission]} self._page.browser_driver.set_listener('Browser.downloadProgress', self._onDownloadProgress) self._page.browser_driver.set_listener('Browser.downloadWillBegin', self._onDownloadWillBegin) @@ -109,28 +107,35 @@ class BrowserDownloadManager(object): """ self._guid_and_tab[guid] = tab_id - def set_done(self, mission, state, cancel=False, final_path=None): + def set_done(self, mission, state, final_path=None): """设置任务结束 :param mission: 任务对象 :param state: 任务状态 - :param cancel: 是否取消 :param final_path: 最终路径 :return: None """ - mission.state = state + if mission.state not in ('canceled', 'skipped'): + mission.state = state mission.final_path = final_path - if cancel: - self._page.browser_driver.call_method('Browser.cancelDownload', guid=mission.id) - if mission.final_path: - Path(mission.final_path).unlink(True) if mission.tab_id in self._tab_missions: self._tab_missions[mission.tab_id].remove(mission.id) self._missions.pop(mission.id) + def cancel(self, mission, state): + """取消任务 + :param mission: 任务对象 + :param state: 任务状态 + :return: None + """ + mission.state = state + self._page.browser_driver.call_method('Browser.cancelDownload', guid=mission.id) + if mission.final_path: + Path(mission.final_path).unlink(True) + def _onDownloadWillBegin(self, **kwargs): """用于获取弹出新标签页触发的下载任务""" guid = kwargs['guid'] - end = perf_counter() + .5 + end = perf_counter() + .3 while perf_counter() < end: tab_id = self._guid_and_tab.get(guid, None) if tab_id: @@ -163,9 +168,9 @@ class BrowserDownloadManager(object): self._missions[guid] = m if self.get_flag(tab_id) is False: # 取消该任务 - self.set_done(m, 'canceled', True) + self.cancel(m, 'canceled') elif skip: - self.set_done(m, 'skipped', True) + self.cancel(m, 'skipped') else: self._tab_missions.setdefault(tab_id, []).append(guid) @@ -174,24 +179,22 @@ class BrowserDownloadManager(object): def _onDownloadProgress(self, **kwargs): """下载状态变化时执行""" if kwargs['guid'] in self._missions: - with self._lock: - if kwargs['guid'] in self._missions: - mission = self._missions[kwargs['guid']] - if kwargs['state'] == 'inProgress': - mission.state = 'running' - mission.received_bytes = kwargs['receivedBytes'] - mission.total_bytes = kwargs['totalBytes'] + mission = self._missions[kwargs['guid']] + if kwargs['state'] == 'inProgress': + mission.state = 'running' + mission.received_bytes = kwargs['receivedBytes'] + mission.total_bytes = kwargs['totalBytes'] - elif kwargs['state'] == 'completed': - mission.received_bytes = kwargs['receivedBytes'] - mission.total_bytes = kwargs['totalBytes'] - form_path = f'{self._page.download_path}\\{mission.id}' - to_path = str(get_usable_path(f'{mission.path}\\{mission.name}')) - move(form_path, to_path) - self.set_done(mission, 'completed', final_path=to_path) + elif kwargs['state'] == 'completed': + mission.received_bytes = kwargs['receivedBytes'] + mission.total_bytes = kwargs['totalBytes'] + form_path = f'{self._page.download_path}\\{mission.id}' + to_path = str(get_usable_path(f'{mission.path}\\{mission.name}')) + move(form_path, to_path) + self.set_done(mission, 'completed', final_path=to_path) - else: # canceled - self.set_done(mission, 'canceled') + else: # 'canceled' + self.set_done(mission, 'canceled') class TabDownloadSettings(object): @@ -248,7 +251,7 @@ class DownloadMission(object): def cancel(self): """取消该任务，如任务已完成，删除已下载的文件""" - self._mgr.set_done(self, state='canceled', cancel=True) + self._mgr.cancel(self, state='canceled') def wait(self, show=True, timeout=None, cancel_if_timeout=True): """等待任务结束 diff --git a/DrissionPage/browser_download_manager.pyi b/DrissionPage/browser_download_manager.pyi index bcc1c0e..3176457 100644 --- a/DrissionPage/browser_download_manager.pyi +++ b/DrissionPage/browser_download_manager.pyi @@ -1,15 +1,12 @@ from pathlib import Path -from threading import Lock from typing import Dict, Optional, Union -from chromium_base import ChromiumBase from chromium_page import ChromiumPage class BrowserDownloadManager(object): BROWSERS: Dict[str, BrowserDownloadManager] = ... _page: ChromiumPage = ... - _lock: Lock = ... _missions: Dict[str, DownloadMission] = ... _tab_missions: dict = ... _tabs_settings: Dict[str, TabDownloadSettings] = ... @@ -37,7 +34,9 @@ class BrowserDownloadManager(object): def set_mission(self, tab_id: str, guid: str) -> None: ... - def set_done(self, mission: DownloadMission, state: str, cancel: bool = False, final_path: str = None) -> None: ... + def set_done(self, mission: DownloadMission, state: str, final_path: str = None) -> None: ... + + def cancel(self, mission: DownloadMission, state: str) -> None: ... def _onDownloadWillBegin(self, **kwargs) -> None: ... From c450a0c4527f36a2f959e31e58d266e74d21b8ed Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 18 Oct 2023 00:19:29 +0800 Subject: [PATCH 034/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=8C=E5=BE=85?= =?UTF-8?q?=E5=AE=8C=E5=96=84tab=E8=A7=A6=E5=8F=91=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/browser_download_manager.py | 62 +++++++++++++++-------- DrissionPage/browser_download_manager.pyi | 8 ++- DrissionPage/chromium_base.py | 1 + DrissionPage/chromium_page.pyi | 2 +- DrissionPage/waiter.py | 2 +- 5 files changed, 50 insertions(+), 25 deletions(-) diff --git a/DrissionPage/browser_download_manager.py b/DrissionPage/browser_download_manager.py index a226822..bf8e7a2 100644 --- a/DrissionPage/browser_download_manager.py +++ b/DrissionPage/browser_download_manager.py @@ -1,4 +1,5 @@ # -*- coding:utf-8 -*- +from os.path import sep from pathlib import Path from shutil import move from time import sleep, perf_counter @@ -117,31 +118,41 @@ class BrowserDownloadManager(object): if mission.state not in ('canceled', 'skipped'): mission.state = state mission.final_path = final_path - if mission.tab_id in self._tab_missions: + if mission.tab_id in self._tab_missions and mission.id in self._tab_missions[mission.tab_id]: self._tab_missions[mission.tab_id].remove(mission.id) self._missions.pop(mission.id) - def cancel(self, mission, state): + def cancel(self, mission): """取消任务 :param mission: 任务对象 - :param state: 任务状态 :return: None """ - mission.state = state + mission.state = 'canceled' self._page.browser_driver.call_method('Browser.cancelDownload', guid=mission.id) if mission.final_path: Path(mission.final_path).unlink(True) + def skip(self, mission): + """跳过任务 + :param mission: 任务对象 + :return: None + """ + mission.state = 'skipped' + self._page.browser_driver.call_method('Browser.cancelDownload', guid=mission.id) + def _onDownloadWillBegin(self, **kwargs): """用于获取弹出新标签页触发的下载任务""" guid = kwargs['guid'] - end = perf_counter() + .3 + + end = perf_counter() + 2 while perf_counter() < end: tab_id = self._guid_and_tab.get(guid, None) if tab_id: + # print('拿到') break sleep(.005) else: + # print('没拿到') tab_id = self._page.tab_id settings = TabDownloadSettings(tab_id) @@ -164,13 +175,13 @@ class BrowserDownloadManager(object): elif settings.when_file_exists == 'overwrite': goal_path.unlink() - m = DownloadMission(self, tab_id, guid, settings.path, name, kwargs['url']) + m = DownloadMission(self, tab_id, guid, settings.path, name, kwargs['url'], self._page.download_path) self._missions[guid] = m if self.get_flag(tab_id) is False: # 取消该任务 - self.cancel(m, 'canceled') + self.cancel(m) elif skip: - self.cancel(m, 'skipped') + self.skip(m) else: self._tab_missions.setdefault(tab_id, []).append(guid) @@ -181,15 +192,18 @@ class BrowserDownloadManager(object): if kwargs['guid'] in self._missions: mission = self._missions[kwargs['guid']] if kwargs['state'] == 'inProgress': - mission.state = 'running' mission.received_bytes = kwargs['receivedBytes'] mission.total_bytes = kwargs['totalBytes'] elif kwargs['state'] == 'completed': + if mission.state == 'skipped': + Path(f'{mission.save_path}{sep}{mission.id}').unlink(True) + self.set_done(mission, 'skipped') + return mission.received_bytes = kwargs['receivedBytes'] mission.total_bytes = kwargs['totalBytes'] - form_path = f'{self._page.download_path}\\{mission.id}' - to_path = str(get_usable_path(f'{mission.path}\\{mission.name}')) + form_path = f'{mission.save_path}{sep}{mission.id}' + to_path = str(get_usable_path(f'{mission.path}{sep}{mission.name}')) move(form_path, to_path) self.set_done(mission, 'completed', final_path=to_path) @@ -224,17 +238,27 @@ class TabDownloadSettings(object): class DownloadMission(object): - def __init__(self, mgr, tab_id, _id, path, name, url): + def __init__(self, mgr, tab_id, _id, path, name, url, save_path): + """ + :param mgr: BrowserDownloadManager对象 + :param tab_id: 标签页id + :param _id: 任务id + :param path: 保存路径 + :param name: 文件名 + :param url: url + :param save_path: 下载路径 + """ self._mgr = mgr self.url = url self.tab_id = tab_id self.id = _id self.path = path self.name = name - self.state = 'waiting' + self.state = 'running' self.total_bytes = None self.received_bytes = 0 self.final_path = None + self.save_path = save_path def __repr__(self): return f'' @@ -247,11 +271,11 @@ class DownloadMission(object): @property def is_done(self): """返回任务是否在运行中""" - return self.state == 'completed' + return self.state in ('completed', 'skipped', 'canceled') def cancel(self): """取消该任务，如任务已完成，删除已下载的文件""" - self._mgr.cancel(self, state='canceled') + self._mgr.cancel(self) def wait(self, show=True, timeout=None, cancel_if_timeout=True): """等待任务结束 @@ -269,23 +293,19 @@ class DownloadMission(object): print(f'目标路径：{self.path}') if timeout is None: - while self.id in self._mgr.missions: + while not self.is_done: if show: print(f'\r{self.rate}% ', end='') sleep(.2) else: - running = True end_time = perf_counter() + timeout while perf_counter() < end_time: if show: print(f'\r{self.rate}% ', end='') - if self.id not in self._mgr.missions: - running = False - break sleep(.2) - if running and cancel_if_timeout: + if not self.is_done and cancel_if_timeout: self.cancel() if show: diff --git a/DrissionPage/browser_download_manager.pyi b/DrissionPage/browser_download_manager.pyi index 3176457..fde5479 100644 --- a/DrissionPage/browser_download_manager.pyi +++ b/DrissionPage/browser_download_manager.pyi @@ -36,7 +36,9 @@ class BrowserDownloadManager(object): def set_done(self, mission: DownloadMission, state: str, final_path: str = None) -> None: ... - def cancel(self, mission: DownloadMission, state: str) -> None: ... + def cancel(self, mission: DownloadMission) -> None: ... + + def skip(self, mission: DownloadMission) -> None: ... def _onDownloadWillBegin(self, **kwargs) -> None: ... @@ -65,8 +67,10 @@ class DownloadMission(object): total_bytes: Optional[int] = ... received_bytes: int = ... final_path: Optional[str] = ... + save_path: str = ... - def __init__(self, mgr: BrowserDownloadManager, tab_id: str, _id: str, path: str, name: str, url: str): ... + def __init__(self, mgr: BrowserDownloadManager, tab_id: str, _id: str, path: str, name: str, url: str, + save_path: str): ... @property def rate(self) -> float: ... diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index 7179964..d212eab 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -250,6 +250,7 @@ class ChromiumBase(BasePage): def _onDownloadWillBegin(self, **kwargs): """下载即将开始时执行""" + print('aaa') self.browser._dl_mgr.set_mission(self.tab_id, kwargs['guid']) def __call__(self, loc_or_str, timeout=None): diff --git a/DrissionPage/chromium_page.pyi b/DrissionPage/chromium_page.pyi index 497ea44..0122d1d 100644 --- a/DrissionPage/chromium_page.pyi +++ b/DrissionPage/chromium_page.pyi @@ -66,7 +66,7 @@ class ChromiumPage(ChromiumBase): def get_tab(self, tab_id: Union[str, ChromiumTab] = None) -> ChromiumTab: ... def find_tabs(self, title: str = None, url: str = None, - tab_type: Union[str, list, tuple, set] = None, single: bool = True) -> Union[str, List[str]]: ... + tab_type: Union[str, list, tuple] = None, single: bool = True) -> Union[str, List[str]]: ... def _new_tab(self, url: str = None, switch_to: bool = False) -> str: ... diff --git a/DrissionPage/waiter.py b/DrissionPage/waiter.py index 7075ae2..1d80385 100644 --- a/DrissionPage/waiter.py +++ b/DrissionPage/waiter.py @@ -250,7 +250,7 @@ class ChromiumPageWaiter(ChromiumTabWaiter): if self._driver._dl_mgr._missions: if cancel_if_timeout: - for m in self._driver._dl_mgr._missions.values(): + for m in list(self._driver._dl_mgr._missions.values()): m.cancel() return False else: From 0ff5b47a4cbeea29ff577a2471a0f564539a5545 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 18 Oct 2023 14:14:08 +0800 Subject: [PATCH 035/182] =?UTF-8?q?=E7=BB=A7=E7=BB=AD=E6=94=B9=E8=BF=9B?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/browser_download_manager.py | 12 +-- DrissionPage/chromium_base.py | 94 ++++++++++++++---------- DrissionPage/chromium_base.pyi | 6 +- DrissionPage/chromium_frame.py | 48 ++++++------ DrissionPage/chromium_page.py | 5 +- DrissionPage/chromium_page.pyi | 1 + DrissionPage/setter.py | 2 +- DrissionPage/setter.pyi | 2 +- DrissionPage/web_page.py | 7 +- DrissionPage/web_page.pyi | 1 + 10 files changed, 97 insertions(+), 81 deletions(-) diff --git a/DrissionPage/browser_download_manager.py b/DrissionPage/browser_download_manager.py index bf8e7a2..9bcfd3e 100644 --- a/DrissionPage/browser_download_manager.py +++ b/DrissionPage/browser_download_manager.py @@ -143,17 +143,7 @@ class BrowserDownloadManager(object): def _onDownloadWillBegin(self, **kwargs): """用于获取弹出新标签页触发的下载任务""" guid = kwargs['guid'] - - end = perf_counter() + 2 - while perf_counter() < end: - tab_id = self._guid_and_tab.get(guid, None) - if tab_id: - # print('拿到') - break - sleep(.005) - else: - # print('没拿到') - tab_id = self._page.tab_id + tab_id = self._page._frames.get(kwargs['frameId'], self._page.tab_id) settings = TabDownloadSettings(tab_id) if settings.rename: diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index d212eab..0d652d0 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -7,6 +7,7 @@ from base64 import b64decode from json import loads, JSONDecodeError from os.path import sep from pathlib import Path +from re import findall from threading import Thread from time import perf_counter, sleep, time @@ -116,53 +117,60 @@ class ChromiumBase(BasePage): self._tab_obj.set_listener('DOM.documentUpdated', self._onDocumentUpdated) self._tab_obj.set_listener('Page.loadEventFired', self._onLoadEventFired) self._tab_obj.set_listener('Page.frameNavigated', self._onFrameNavigated) - self._tab_obj.set_listener('Page.downloadWillBegin', self._onDownloadWillBegin) + self._tab_obj.set_listener('Page.frameAttached', self._onFrameAttached) + self._tab_obj.set_listener('Page.frameDetached', self._onFrameDetached) def _get_document(self): """刷新cdp使用的document数据""" - if not self._is_reading: - self._is_reading = True + if self._is_reading: + return - if self._debug: - print('获取document') + self._is_reading = True + + if self._debug: + print('获取document') + if self._debug_recorder: + self._debug_recorder.add_data((perf_counter(), '获取document', '开始')) + + try: # 遇到过网站在标签页关闭时触发读取文档导致错误，屏蔽掉 + self._wait_loaded() + except TabClosedError: + return + + end_time = perf_counter() + 10 + while perf_counter() < end_time: + try: + b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] + self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id)['object']['objectId'] if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '获取document', '开始')) + self._debug_recorder.add_data((perf_counter(), '信息', f'root_id：{self._root_id}')) + break - try: # 遇到过网站在标签页关闭时触发读取文档导致错误，屏蔽掉 - self._wait_loaded() - except TabClosedError: - return - - end_time = perf_counter() + 10 - while perf_counter() < end_time: - try: - b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] - self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id)['object']['objectId'] + except CDPError as e: + err = e + if self._debug: + print('重试获取document') if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '信息', f'root_id：{self._root_id}')) - break + self._debug_recorder.add_data((perf_counter(), 'err', '读取root_id出错')) - except CDPError as e: - err = e - if self._debug: - print('重试获取document') - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), 'err', '读取root_id出错')) + sleep(.1) - sleep(.1) + else: + txt = f'请检查是否创建了过多页面对象同时操作浏览器。\n如无法解决，请把以下信息报告作者。\n{err._info}\n' \ + f'报告网址：https://gitee.com/g1879/DrissionPage/issues' + raise GetDocumentError(txt) - else: - txt = f'请检查是否创建了过多页面对象同时操作浏览器。\n如无法解决，请把以下信息报告作者。\n{err._info}\n' \ - f'报告网址：https://gitee.com/g1879/DrissionPage/issues' - raise GetDocumentError(txt) + if self._debug: + print('获取document结束') + if self._debug_recorder: + self._debug_recorder.add_data((perf_counter(), '获取document', '结束')) - if self._debug: - print('获取document结束') - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '获取document', '结束')) + r = self.run_cdp('Page.getFrameTree') + for i in findall(r"'id': '(.*?)'", str(r)): + self.browser._frames[i] = self.tab_id - self._is_loading = False - self._is_reading = False + self._is_loading = False + self._is_reading = False def _wait_loaded(self, timeout=None): """等待页面加载完成，超时触发停止加载 @@ -193,8 +201,18 @@ class ChromiumBase(BasePage): self.stop_loading() return False + def _onFrameDetached(self, **kwargs): + try: + self.browser._frames.pop(kwargs['frameId']) + except KeyError: + pass + + def _onFrameAttached(self, **kwargs): + self.browser._frames[kwargs['frameId']] = self.tab_id + def _onFrameStartedLoading(self, **kwargs): """页面开始加载时执行""" + self.browser._frames[kwargs['frameId']] = self.tab_id if kwargs['frameId'] == self._target_id: self._is_loading = True @@ -205,6 +223,7 @@ class ChromiumBase(BasePage): def _onFrameStoppedLoading(self, **kwargs): """页面加载完成后执行""" + self.browser._frames[kwargs['frameId']] = self.tab_id if kwargs['frameId'] == self._target_id and self._first_run is False and self._is_loading: if self._debug: print('页面停止加载 FrameStoppedLoading') @@ -248,11 +267,6 @@ class ChromiumBase(BasePage): self.run_cdp('Page.setInterceptFileChooserDialog', enabled=False) self._upload_list = None - def _onDownloadWillBegin(self, **kwargs): - """下载即将开始时执行""" - print('aaa') - self.browser._dl_mgr.set_mission(self.tab_id, kwargs['guid']) - def __call__(self, loc_or_str, timeout=None): """在内部查找元素 例：ele = page('@id=ele_id') diff --git a/DrissionPage/chromium_base.pyi b/DrissionPage/chromium_base.pyi index db2da7e..358c52c 100644 --- a/DrissionPage/chromium_base.pyi +++ b/DrissionPage/chromium_base.pyi @@ -62,6 +62,10 @@ class ChromiumBase(BasePage): def _wait_loaded(self, timeout: float = None) -> bool: ... + def _onFrameDetached(self, **kwargs) -> None: ... + + def _onFrameAttached(self, **kwargs) -> None: ... + def _onFrameStartedLoading(self, **kwargs): ... def _onFrameStoppedLoading(self, **kwargs): ... @@ -74,7 +78,7 @@ class ChromiumBase(BasePage): def _onFileChooserOpened(self, **kwargs): ... - def _onDownloadWillBegin(self, **kwargs): ... + # def _onDownloadWillBegin(self, **kwargs): ... def _set_start_options(self, address, none) -> None: ... diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/chromium_frame.py index 28319ee..21cd5c7 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/chromium_frame.py @@ -127,37 +127,39 @@ class ChromiumFrame(ChromiumBase): def _get_new_document(self): """刷新cdp使用的document数据""" - if not self._is_reading: - self._is_reading = True + if self._is_reading: + return - if self._debug: - print('---获取document') + self._is_reading = True - end_time = perf_counter() + 3 - while self.is_alive and perf_counter() < end_time: - try: - if self._is_diff_domain is False: - node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self.ids.backend_id)['node'] - self.doc_ele = ChromiumElement(self._target_page, - backend_id=node['contentDocument']['backendNodeId']) + if self._debug: + print('---获取document') - else: - b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] - self.doc_ele = ChromiumElement(self, backend_id=b_id) + end_time = perf_counter() + 3 + while self.is_alive and perf_counter() < end_time: + try: + if self._is_diff_domain is False: + node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self.ids.backend_id)['node'] + self.doc_ele = ChromiumElement(self._target_page, + backend_id=node['contentDocument']['backendNodeId']) - break + else: + b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] + self.doc_ele = ChromiumElement(self, backend_id=b_id) - except Exception: - sleep(.1) + break - # else: - # raise RuntimeError('获取document失败。') + except Exception: + sleep(.1) - if self._debug: - print('---获取document结束') + # else: + # raise RuntimeError('获取document失败。') - self._is_loading = False - self._is_reading = False + if self._debug: + print('---获取document结束') + + self._is_loading = False + self._is_reading = False def _onFrameNavigated(self, **kwargs): """页面跳转时触发""" diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 58215d3..24fb94f 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -26,8 +26,9 @@ class ChromiumPage(ChromiumBase): :param tab_id: 要控制的标签页id，不指定默认为激活的 :param timeout: 超时时间 """ - super().__init__(addr_driver_opts, tab_id) self._page = self + self._frames = {} + super().__init__(addr_driver_opts, tab_id) self._dl_mgr = BrowserDownloadManager(self) self.set.timeouts(implicit=timeout) @@ -93,7 +94,7 @@ class ChromiumPage(ChromiumBase): self._first_run = False def _page_init(self): - """页面相关设置""" + """浏览器相关设置""" u = f'http://{self.address}/json/version' ws = self._control_session.get(u).json()['webSocketDebuggerUrl'] self._control_session.get(u, headers={'Connection': 'close'}) diff --git a/DrissionPage/chromium_page.pyi b/DrissionPage/chromium_page.pyi index 0122d1d..a6d81ec 100644 --- a/DrissionPage/chromium_page.pyi +++ b/DrissionPage/chromium_page.pyi @@ -27,6 +27,7 @@ class ChromiumPage(ChromiumBase): self._alert: Alert = ... self._browser_driver: ChromiumDriver = ... self._rect: ChromiumTabRect = ... + self._frames: dict = ... def _connect_browser(self, addr_driver_opts: Union[str, ChromiumDriver] = None, diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index 54cefe8..fefe638 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -355,7 +355,7 @@ class WebPageSetter(ChromiumPageSetter): self._chromium_setter.user_agent(ua, platform) -class WebPageTabSetter(ChromiumBaseSetter): +class WebPageTabSetter(TabSetter): def __init__(self, page): super().__init__(page) self._session_setter = SessionPageSetter(self._page) diff --git a/DrissionPage/setter.pyi b/DrissionPage/setter.pyi index 8803ed8..0ed4134 100644 --- a/DrissionPage/setter.pyi +++ b/DrissionPage/setter.pyi @@ -129,7 +129,7 @@ class WebPageSetter(ChromiumPageSetter): def cookies(self, cookies) -> None: ... -class WebPageTabSetter(ChromiumBaseSetter): +class WebPageTabSetter(TabSetter): _page: WebPage = ... _session_setter: SessionPageSetter = ... _chromium_setter: ChromiumBaseSetter = ... diff --git a/DrissionPage/web_page.py b/DrissionPage/web_page.py index 2d97b0c..3282a54 100644 --- a/DrissionPage/web_page.py +++ b/DrissionPage/web_page.py @@ -6,6 +6,7 @@ from requests import Session from .base import BasePage +from .browser_download_manager import BrowserDownloadManager from .chromium_base import ChromiumBase, Timeout from .chromium_driver import ChromiumDriver from .chromium_page import ChromiumPage @@ -45,13 +46,15 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self._response = None self._set = None self._screencast = None + self._frames = {} + self._page = self self._set_start_options(driver_or_options, session_or_options) self._set_runtime_settings() self._connect_browser() self._create_session() - - t = timeout if isinstance(timeout, (int, float)) else self.timeouts.implicit + self._dl_mgr = BrowserDownloadManager(self) + self.set.timeouts(implicit=timeout) def _set_start_options(self, dr_opt, se_opt): """处理两种模式的设置 diff --git a/DrissionPage/web_page.pyi b/DrissionPage/web_page.pyi index 5734631..15f30f8 100644 --- a/DrissionPage/web_page.pyi +++ b/DrissionPage/web_page.pyi @@ -37,6 +37,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self._DownloadKit: DownloadKit = ... self._download_path: str = ... self._tab_obj: ChromiumDriver = ... + self._frames: dict = ... def __call__(self, loc_or_str: Union[Tuple[str, str], str, ChromiumElement, SessionElement], From de6691adc22ab40387f554ee306db32b48a063a2 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 18 Oct 2023 17:52:27 +0800 Subject: [PATCH 036/182] =?UTF-8?q?=E6=B7=BB=E5=8A=A0Browser=E7=B1=BB?= =?UTF-8?q?=EF=BC=8C=E8=B0=83=E6=95=B4=E6=9E=B6=E6=9E=84=EF=BC=8C=E6=9C=AA?= =?UTF-8?q?=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/browser.py | 137 ++++++++++++++++++++++ DrissionPage/browser.pyi | 48 ++++++++ DrissionPage/browser_download_manager.py | 71 +++++------ DrissionPage/browser_download_manager.pyi | 15 ++- DrissionPage/chromium_base.py | 16 +-- DrissionPage/chromium_base.pyi | 13 +- DrissionPage/chromium_driver.py | 4 + DrissionPage/chromium_frame.py | 8 +- DrissionPage/chromium_page.py | 74 +++--------- DrissionPage/chromium_page.pyi | 13 +- DrissionPage/chromium_tab.py | 2 + DrissionPage/chromium_tab.pyi | 5 +- DrissionPage/commons/browser.py | 4 +- DrissionPage/setter.py | 2 +- DrissionPage/waiter.py | 8 +- DrissionPage/web_page.py | 2 - 16 files changed, 278 insertions(+), 144 deletions(-) create mode 100644 DrissionPage/browser.py create mode 100644 DrissionPage/browser.pyi diff --git a/DrissionPage/browser.py b/DrissionPage/browser.py new file mode 100644 index 0000000..36e522e --- /dev/null +++ b/DrissionPage/browser.py @@ -0,0 +1,137 @@ +# -*- coding:utf-8 -*- +from time import sleep + +from .browser_download_manager import BrowserDownloadManager +from .chromium_driver import BrowserDriver + + +class Browser(object): + BROWSERS = {} + + def __new__(cls, browser_id, page): + """ + :param browser_id: BrowserDriver对象 + :param page: ChromiumPage对象 + """ + if browser_id in cls.BROWSERS: + return cls.BROWSERS[browser_id] + return object.__new__(cls) + + def __init__(self, browser_id, page): + """ + :param page: BrowserDriver对象 + :param page: ChromiumPage对象 + """ + if hasattr(self, '_created'): + return + self._created = True + Browser.BROWSERS[browser_id] = self + + self.page = page + self.address = page.address + self._driver = BrowserDriver(browser_id, 'browser', page.address) + self.id = browser_id + self._frames = {} + + self._process_id = None + r = self.run_cdp('SystemInfo.getProcessInfo') + for i in r.get('processInfo', []): + if i['type'] == 'browser': + self._process_id = i['id'] + break + + self._dl_mgr = BrowserDownloadManager(self) + + self.run_cdp('Target.setDiscoverTargets') + self._driver.set_listener('Target.targetDestroyed', self._onTargetDestroyed) + + def _onTargetDestroyed(self, **kwargs): + """标签页关闭时执行""" + tab_id = kwargs['targetId'] + self._dl_mgr.clear_tab_info(tab_id) + for k, i in self._frames.items(): + if i == tab_id: + self._frames.pop(k) + + def run_cdp(self, cmd, **cmd_args): + """执行Chrome DevTools Protocol语句 + :param cmd: 协议项目 + :param cmd_args: 参数 + :return: 执行的结果 + """ + return self._driver.call_method(cmd, **cmd_args) + + @property + def driver(self): + return self._driver + + @property + def tabs_count(self): + """返回标签页数量""" + return len(self.tabs) + + @property + def tabs(self): + """返回所有标签页id组成的列表""" + j = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp + return [i['id'] for i in j if i['type'] == 'page'] + + @property + def process_id(self): + """返回浏览器进程id""" + return self._process_id + + def find_tabs(self, title=None, url=None, tab_type=None, single=True): + """查找符合条件的tab，返回它们的id组成的列表 + :param title: 要匹配title的文本 + :param url: 要匹配url的文本 + :param tab_type: tab类型，可用列表输入多个 + :param single: 是否返回首个结果的id，为False返回所有信息 + :return: tab id或tab dict + """ + tabs = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp + + if isinstance(tab_type, str): + tab_type = {tab_type} + elif isinstance(tab_type, (list, tuple, set)): + tab_type = set(tab_type) + elif tab_type is not None: + raise TypeError('tab_type只能是set、list、tuple、str、None。') + + r = [i for i in tabs if ((title is None or title in i['title']) and (url is None or url in i['url']) + and (tab_type is None or i['type'] in tab_type))] + return r[0]['id'] if r and single else r + + def close_tab(self, tab_id): + """关闭标签页 + :param tab_id: 标签页id + :return: None + """ + self._driver.get(f'http://{self.address}/json/close/{tab_id}') + + def activate_tab(self, tab_id): + """使标签页变为活动状态 + :param tab_id: 标签页id + :return: None + """ + self._driver.get(f'http://{self.address}/json/activate/{tab_id}') + + def get_window_bounds(self): + """返回浏览器窗口位置和大小信息""" + return self.run_cdp('Browser.getWindowForTarget', targetId=self.id)['bounds'] + + def quit(self): + """关闭浏览器""" + self.run_cdp('Browser.close') + self.driver.stop() + + if self.process_id: + from os import popen + from platform import system + txt = f'tasklist | findstr {self.process_id}' if system().lower() == 'windows' \ + else f'ps -ef | grep {self.process_id}' + while True: + p = popen(txt) + if f' {self.process_id} ' not in p.read(): + break + sleep(.2) diff --git a/DrissionPage/browser.pyi b/DrissionPage/browser.pyi new file mode 100644 index 0000000..91e9fdb --- /dev/null +++ b/DrissionPage/browser.pyi @@ -0,0 +1,48 @@ +# -*- coding:utf-8 -*- +from typing import List, Optional, Union + +from .browser_download_manager import BrowserDownloadManager +from .chromium_page import ChromiumPage +from .chromium_driver import BrowserDriver + + +class Browser(object): + BROWSERS: dict = ... + page: ChromiumPage = ... + _driver: BrowserDriver = ... + id: str = ... + address: str = ... + _frames: dict = ... + _process_id: Optional[int] = ... + _dl_mgr: BrowserDownloadManager = ... + + def __new__(cls, browser_id: str, page: ChromiumPage): ... + + def __init__(self, browser_id: str, page: ChromiumPage): ... + + def run_cdp(self, cmd, **cmd_args) -> dict: ... + + @property + def driver(self) -> BrowserDriver: ... + + @property + def tabs_count(self) -> int: ... + + @property + def tabs(self) -> List[str]: ... + + @property + def process_id(self) -> Optional[int]: ... + + def find_tabs(self, title: str = None, url: str = None, + tab_type: Union[str, list, tuple] = None, single: bool = True) -> Union[str, List[str]]: ... + + def close_tab(self, tab_id: str) -> None: ... + + def activate_tab(self, tab_id: str) -> None: ... + + def get_window_bounds(self) -> dict: ... + + def _onTargetDestroyed(self, **kwargs): ... + + def quit(self) -> None: ... diff --git a/DrissionPage/browser_download_manager.py b/DrissionPage/browser_download_manager.py index 9bcfd3e..2d6cba5 100644 --- a/DrissionPage/browser_download_manager.py +++ b/DrissionPage/browser_download_manager.py @@ -8,42 +8,26 @@ from .commons.tools import get_usable_path class BrowserDownloadManager(object): - BROWSERS = {} - def __new__(cls, page): + def __init__(self, browser): """ - :param page: ChromiumPage对象 + :param browser: Browser对象 """ - if page.browser_driver.id in cls.BROWSERS: - return cls.BROWSERS[page.browser_driver.id] - return object.__new__(cls) - - def __init__(self, page): - """ - :param page: ChromiumPage对象 - """ - if hasattr(self, '_created'): - return - self._created = True - - self._page = page + self._browser = browser + self._page = browser.page self._when_download_file_exists = 'rename' - t = TabDownloadSettings(page.tab_id) - t.path = page.download_path - self._tabs_settings = {page.tab_id: t} # {tab_id: TabDownloadSettings} + t = TabDownloadSettings(self._page.tab_id) + t.path = self._page.download_path self._missions = {} # {guid: DownloadMission} + self._tabs_settings = {self._page.tab_id: t} # {tab_id: TabDownloadSettings} self._tab_missions = {} # {tab_id: DownloadMission} - self._guid_and_tab = {} # 记录guid在哪个tab self._flags = {} # {tab_id: [bool, DownloadMission]} - self._page.browser_driver.set_listener('Browser.downloadProgress', self._onDownloadProgress) - self._page.browser_driver.set_listener('Browser.downloadWillBegin', self._onDownloadWillBegin) - self._page.browser_driver.call_method('Browser.setDownloadBehavior', - downloadPath=self._page.download_path, - behavior='allowAndName', eventsEnabled=True) - - BrowserDownloadManager.BROWSERS[page.browser_driver.id] = self + self._browser.driver.set_listener('Browser.downloadProgress', self._onDownloadProgress) + self._browser.driver.set_listener('Browser.downloadWillBegin', self._onDownloadWillBegin) + self._browser.run_cdp('Browser.setDownloadBehavior', downloadPath=self._page.download_path, + behavior='allowAndName', eventsEnabled=True) @property def missions(self): @@ -58,9 +42,8 @@ class BrowserDownloadManager(object): """ self._tabs_settings.setdefault(tab_id, TabDownloadSettings(tab_id)).path = str(Path(path).absolute()) if tab_id == self._page.tab_id: - self._page.browser_driver.call_method('Browser.setDownloadBehavior', - downloadPath=str(Path(path).absolute()), - behavior='allowAndName', eventsEnabled=True) + self._browser.run_cdp('Browser.setDownloadBehavior', downloadPath=str(Path(path).absolute()), + behavior='allowAndName', eventsEnabled=True) def set_rename(self, tab_id, rename): """设置某个tab的重命名文件名 @@ -100,14 +83,6 @@ class BrowserDownloadManager(object): """ return self._tab_missions.get(tab_id, []) - def set_mission(self, tab_id, guid): - """绑定tab和下载任务信息 - :param tab_id: tab id - :param guid: 下载任务id - :return: None - """ - self._guid_and_tab[guid] = tab_id - def set_done(self, mission, state, final_path=None): """设置任务结束 :param mission: 任务对象 @@ -121,6 +96,7 @@ class BrowserDownloadManager(object): if mission.tab_id in self._tab_missions and mission.id in self._tab_missions[mission.tab_id]: self._tab_missions[mission.tab_id].remove(mission.id) self._missions.pop(mission.id) + mission._is_done = True def cancel(self, mission): """取消任务 @@ -128,7 +104,7 @@ class BrowserDownloadManager(object): :return: None """ mission.state = 'canceled' - self._page.browser_driver.call_method('Browser.cancelDownload', guid=mission.id) + self._browser.run_cdp('Browser.cancelDownload', guid=mission.id) if mission.final_path: Path(mission.final_path).unlink(True) @@ -138,12 +114,22 @@ class BrowserDownloadManager(object): :return: None """ mission.state = 'skipped' - self._page.browser_driver.call_method('Browser.cancelDownload', guid=mission.id) + self._browser.run_cdp('Browser.cancelDownload', guid=mission.id) + + def clear_tab_info(self, tab_id): + """当tab关闭时清除有关信息 + :param tab_id: 标签页id + :return: None + """ + self._tabs_settings.pop(tab_id) + self._tab_missions.pop(tab_id) + self._flags.pop(tab_id) + TabDownloadSettings.TABS.pop(tab_id) def _onDownloadWillBegin(self, **kwargs): """用于获取弹出新标签页触发的下载任务""" guid = kwargs['guid'] - tab_id = self._page._frames.get(kwargs['frameId'], self._page.tab_id) + tab_id = self._browser._frames.get(kwargs['frameId'], self._page.tab_id) settings = TabDownloadSettings(tab_id) if settings.rename: @@ -249,6 +235,7 @@ class DownloadMission(object): self.received_bytes = 0 self.final_path = None self.save_path = save_path + self._is_done = False def __repr__(self): return f'' @@ -261,7 +248,7 @@ class DownloadMission(object): @property def is_done(self): """返回任务是否在运行中""" - return self.state in ('completed', 'skipped', 'canceled') + return self._is_done def cancel(self): """取消该任务，如任务已完成，删除已下载的文件""" diff --git a/DrissionPage/browser_download_manager.pyi b/DrissionPage/browser_download_manager.pyi index fde5479..6a9a434 100644 --- a/DrissionPage/browser_download_manager.pyi +++ b/DrissionPage/browser_download_manager.pyi @@ -1,21 +1,19 @@ from pathlib import Path from typing import Dict, Optional, Union -from chromium_page import ChromiumPage +from .browser import Browser +from .chromium_page import ChromiumPage class BrowserDownloadManager(object): - BROWSERS: Dict[str, BrowserDownloadManager] = ... + _browser: Browser = ... _page: ChromiumPage = ... _missions: Dict[str, DownloadMission] = ... _tab_missions: dict = ... _tabs_settings: Dict[str, TabDownloadSettings] = ... - _guid_and_tab: Dict[str, str] = ... _flags: dict = ... - def __new__(cls, page: ChromiumPage): ... - - def __init__(self, page: ChromiumPage): ... + def __init__(self, browser: Browser): ... @property def missions(self) -> Dict[str, DownloadMission]: ... @@ -32,14 +30,14 @@ class BrowserDownloadManager(object): def get_tab_missions(self, tab_id: str) -> list: ... - def set_mission(self, tab_id: str, guid: str) -> None: ... - def set_done(self, mission: DownloadMission, state: str, final_path: str = None) -> None: ... def cancel(self, mission: DownloadMission) -> None: ... def skip(self, mission: DownloadMission) -> None: ... + def clear_tab_info(self, tab_id: str) -> None: ... + def _onDownloadWillBegin(self, **kwargs) -> None: ... def _onDownloadProgress(self, **kwargs) -> None: ... @@ -68,6 +66,7 @@ class DownloadMission(object): received_bytes: int = ... final_path: Optional[str] = ... save_path: str = ... + _is_done: bool = ... def __init__(self, mgr: BrowserDownloadManager, tab_id: str, _id: str, path: str, name: str, url: str, save_path: str): ... diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index 0d652d0..8c10d16 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -11,7 +11,7 @@ from re import findall from threading import Thread from time import perf_counter, sleep, time -from requests import Session +from requests import get from .action_chains import ActionChains from .base import BasePage @@ -79,9 +79,8 @@ class ChromiumBase(BasePage): """ self._chromium_init() if not tab_id: - u = f'http://{self.address}/json' - json = self._control_session.get(u).json() - self._control_session.get(u, headers={'Connection': 'close'}) + json = get(f'http://{self.address}/json', headers={'Connection': 'close'}).json() + tab_id = [i['id'] for i in json if i['type'] == 'page'] if not tab_id: raise BrowserConnectError('浏览器连接失败，可能是浏览器版本原因。') @@ -92,9 +91,6 @@ class ChromiumBase(BasePage): def _chromium_init(self): """浏览器初始设置""" - self._control_session = Session() - self._control_session.keep_alive = False - self._control_session.proxies = {'http': None, 'https': None} self._first_run = True self._is_reading = False self._upload_list = None @@ -277,9 +273,13 @@ class ChromiumBase(BasePage): return self.ele(loc_or_str, timeout) @property - def browser(self): + def main(self): return self._page + @property + def browser(self): + return self._browser + @property def driver(self): """返回用于控制浏览器的ChromiumDriver对象""" diff --git a/DrissionPage/chromium_base.pyi b/DrissionPage/chromium_base.pyi index 358c52c..22ac5df 100644 --- a/DrissionPage/chromium_base.pyi +++ b/DrissionPage/chromium_base.pyi @@ -7,8 +7,8 @@ from pathlib import Path from typing import Union, Tuple, List, Any from DataRecorder import Recorder -from requests import Session +from .browser import Browser from .action_chains import ActionChains from .base import BasePage from .chromium_driver import ChromiumDriver @@ -27,8 +27,8 @@ class ChromiumBase(BasePage): address: Union[str, int], tab_id: str = None, timeout: float = None): + self._browser: Browser = ... self._page: ChromiumPage = ... - self._control_session: Session = ... self.address: str = ... self._tab_obj: ChromiumDriver = ... self._is_reading: bool = ... @@ -47,10 +47,6 @@ class ChromiumBase(BasePage): self._screencast: Screencast = ... self._actions: ActionChains = ... self._listener: NetworkListener = ... - # self._wait_download_flag: bool = ... - # self._download_rename: str = ... - # self._when_download_file_exists: str = ... - # self._download_missions: set = ... def _connect_browser(self, tab_id: str = None) -> None: ... @@ -88,7 +84,10 @@ class ChromiumBase(BasePage): timeout: float = None) -> ChromiumElement: ... @property - def browser(self) -> ChromiumPage: ... + def main(self) -> ChromiumPage: ... + + @property + def browser(self) -> Browser: ... @property def title(self) -> str: ... diff --git a/DrissionPage/chromium_driver.py b/DrissionPage/chromium_driver.py index 05aab26..cd89be0 100644 --- a/DrissionPage/chromium_driver.py +++ b/DrissionPage/chromium_driver.py @@ -7,6 +7,7 @@ from json import dumps, loads from queue import Queue, Empty from threading import Thread, Event +from requests import get from websocket import WebSocketTimeoutException, WebSocketException, WebSocketConnectionClosedException, \ create_connection @@ -223,3 +224,6 @@ class BrowserDriver(ChromiumDriver): def __repr__(self): return f"" + + def get(self, url): + return get(url, headers={'Connection': 'close'}) diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/chromium_frame.py index 21cd5c7..6ad88f8 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/chromium_frame.py @@ -8,6 +8,8 @@ from re import search from threading import Thread from time import sleep, perf_counter +from requests import get + from .chromium_base import ChromiumBase, ChromiumPageScroll from .chromium_element import ChromiumElement from .errors import ContextLossError @@ -24,8 +26,10 @@ class ChromiumFrame(ChromiumBase): page_type = str(type(page)) if 'ChromiumPage' in page_type or 'WebPage' in page_type: self._page = self._target_page = self.tab = page + self._browser = page.browser else: # Tab、Frame self._page = page.page + self._browser = self._page.browser self._target_page = page self.tab = page.tab if 'ChromiumFrame' in page_type else page @@ -87,9 +91,7 @@ class ChromiumFrame(ChromiumBase): try: super()._driver_init(tab_id) except: - u = f'http://{self.address}/json' - self._control_session.get(u) - self._control_session.get(u, headers={'Connection': 'close'}) + get(f'http://{self.address}/json', headers={'Connection': 'close'}) super()._driver_init(tab_id) def _reload(self): diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 24fb94f..ab1ece7 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -6,9 +6,11 @@ from pathlib import Path from time import perf_counter, sleep -from .browser_download_manager import BrowserDownloadManager +from requests import get + +from .browser import Browser from .chromium_base import ChromiumBase, Timeout -from .chromium_driver import ChromiumDriver, BrowserDriver +from .chromium_driver import ChromiumDriver from .chromium_tab import ChromiumTab from .commons.browser import connect_browser from .configs.chromium_options import ChromiumOptions @@ -27,9 +29,7 @@ class ChromiumPage(ChromiumBase): :param timeout: 超时时间 """ self._page = self - self._frames = {} super().__init__(addr_driver_opts, tab_id) - self._dl_mgr = BrowserDownloadManager(self) self.set.timeouts(implicit=timeout) def _set_start_options(self, addr_driver_opts, none): @@ -79,9 +79,7 @@ class ChromiumPage(ChromiumBase): if not self._tab_obj: # 不是传入driver的情况 connect_browser(self._driver_options) if not tab_id: - u = f'http://{self.address}/json' - json = self._control_session.get(u).json() - self._control_session.get(u, headers={'Connection': 'close'}) + json = get(f'http://{self.address}/json', headers={'Connection': 'close'}).json() tab_id = [i['id'] for i in json if i['type'] == 'page'] if not tab_id: raise BrowserConnectError('浏览器连接失败，可能是浏览器版本原因。') @@ -95,10 +93,8 @@ class ChromiumPage(ChromiumBase): def _page_init(self): """浏览器相关设置""" - u = f'http://{self.address}/json/version' - ws = self._control_session.get(u).json()['webSocketDebuggerUrl'] - self._control_session.get(u, headers={'Connection': 'close'}) - self._browser_driver = BrowserDriver(ws.split('/')[-1], 'browser', self.address) + ws = get(f'http://{self.address}/json/version', headers={'Connection': 'close'}).json()['webSocketDebuggerUrl'] + self._browser = Browser(ws.split('/')[-1], self) self._alert = Alert() self._tab_obj.set_listener('Page.javascriptDialogOpening', self._on_alert_open) @@ -107,32 +103,20 @@ class ChromiumPage(ChromiumBase): self._rect = None self._main_tab = self.tab_id - self._process_id = None - r = self.browser_driver.call_method('SystemInfo.getProcessInfo') - if 'processInfo' not in r: - return None - for i in r['processInfo']: - if i['type'] == 'browser': - self._process_id = i['id'] - break - @property - def browser_driver(self): + def browser(self): """返回用于控制浏览器cdp的driver""" - return self._browser_driver + return self._browser @property def tabs_count(self): """返回标签页数量""" - return len(self.tabs) + return self.browser.tabs_count @property def tabs(self): """返回所有标签页id组成的列表""" - u = f'http://{self.address}/json' - j = self._control_session.get(u).json() # 不要改用cdp - self._control_session.get(u, headers={'Connection': 'close'}) - return [i['id'] for i in j if i['type'] == 'page'] + return self.browser.tabs @property def main_tab(self): @@ -146,7 +130,7 @@ class ChromiumPage(ChromiumBase): @property def process_id(self): """返回浏览器进程id""" - return self._process_id + return self.browser.process_id @property def set(self): @@ -183,19 +167,7 @@ class ChromiumPage(ChromiumBase): :param single: 是否返回首个结果的id，为False返回所有信息 :return: tab id或tab dict """ - u = f'http://{self.address}/json' - tabs = self._control_session.get(u).json() # 不要改用cdp - self._control_session.get(u, headers={'Connection': 'close'}) - if isinstance(tab_type, str): - tab_type = {tab_type} - elif isinstance(tab_type, (list, tuple, set)): - tab_type = set(tab_type) - elif tab_type is not None: - raise TypeError('tab_type只能是set、list、tuple、str、None。') - - r = [i for i in tabs if ((title is None or title in i['title']) and (url is None or url in i['url']) - and (tab_type is None or i['type'] in tab_type))] - return r[0]['id'] if r and single else r + return self._browser.find_tabs(title, url, tab_type, single) def _new_tab(self, url=None, switch_to=False): """新建一个标签页,该标签页在最后面 @@ -265,7 +237,7 @@ class ChromiumPage(ChromiumBase): tab_id = self.latest_tab if activate: - self._control_session.get(f'http://{self.address}/json/activate/{tab_id}') + self.browser.activate_tab(tab_id) if tab_id == self.tab_id: return @@ -305,7 +277,7 @@ class ChromiumPage(ChromiumBase): self.driver.stop() for tab in tabs: - self._control_session.get(f'http://{self.address}/json/close/{tab}') + self.browser.close_tab(tab) while len(self.tabs) != end_len: sleep(.1) @@ -345,19 +317,7 @@ class ChromiumPage(ChromiumBase): def quit(self): """关闭浏览器""" - self._tab_obj.call_method('Browser.close') - self._tab_obj.stop() - - if self.process_id: - from os import popen - from platform import system - txt = f'tasklist | findstr {self.process_id}' if system().lower() == 'windows' \ - else f'ps -ef | grep {self.process_id}' - while True: - p = popen(txt) - if f' {self.process_id} ' not in p.read(): - break - sleep(.2) + self.browser.quit() def _on_alert_close(self, **kwargs): """alert关闭时触发的方法""" @@ -448,7 +408,7 @@ class ChromiumTabRect(object): def _get_browser_rect(self): """获取浏览器范围信息""" - return self._page.browser_driver.call_method('Browser.getWindowForTarget', targetId=self._page.tab_id)['bounds'] + return self._page.browser.get_window_bounds() class Alert(object): diff --git a/DrissionPage/chromium_page.pyi b/DrissionPage/chromium_page.pyi index a6d81ec..90f9f09 100644 --- a/DrissionPage/chromium_page.pyi +++ b/DrissionPage/chromium_page.pyi @@ -3,9 +3,9 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union, Tuple, List +from typing import Union, Tuple, List, Optional -from .browser_download_manager import BrowserDownloadManager +from .browser import Browser from .chromium_base import ChromiumBase from .chromium_driver import ChromiumDriver from .chromium_tab import ChromiumTab @@ -21,13 +21,10 @@ class ChromiumPage(ChromiumBase): tab_id: str = None, timeout: float = None): self._driver_options: ChromiumOptions = ... - self._process_id: str = ... - self._dl_mgr: BrowserDownloadManager = ... self._main_tab: str = ... self._alert: Alert = ... - self._browser_driver: ChromiumDriver = ... + self._browser: Browser = ... self._rect: ChromiumTabRect = ... - self._frames: dict = ... def _connect_browser(self, addr_driver_opts: Union[str, ChromiumDriver] = None, @@ -38,7 +35,7 @@ class ChromiumPage(ChromiumBase): def _page_init(self) -> None: ... @property - def browser_driver(self) -> ChromiumDriver: ... + def browser(self) -> Browser: ... @property def tabs_count(self) -> int: ... @@ -59,7 +56,7 @@ class ChromiumPage(ChromiumBase): def latest_tab(self) -> str: ... @property - def process_id(self) -> Union[None, int]: ... + def process_id(self) -> Optional[int]: ... @property def set(self) -> ChromiumPageSetter: ... diff --git a/DrissionPage/chromium_tab.py b/DrissionPage/chromium_tab.py index caad458..449a0a7 100644 --- a/DrissionPage/chromium_tab.py +++ b/DrissionPage/chromium_tab.py @@ -22,6 +22,7 @@ class ChromiumTab(ChromiumBase): :param tab_id: 要控制的标签页id，不指定默认为激活的 """ self._page = page + self._browser = page.browser super().__init__(page.address, tab_id, page.timeout) def _set_runtime_settings(self): @@ -68,6 +69,7 @@ class WebPageTab(SessionPage, ChromiumTab): :param tab_id: 要控制的标签页id """ self._page = page + self._browser = page.browser self.address = page.address self._debug = page._debug self._debug_recorder = page._debug_recorder diff --git a/DrissionPage/chromium_tab.pyi b/DrissionPage/chromium_tab.pyi index f38f8fa..15d04b4 100644 --- a/DrissionPage/chromium_tab.pyi +++ b/DrissionPage/chromium_tab.pyi @@ -7,7 +7,7 @@ from typing import Union, Tuple, Any, List from requests import Session, Response -from waiter import ChromiumTabWaiter +from .browser import Browser from .chromium_base import ChromiumBase from .chromium_element import ChromiumElement from .chromium_frame import ChromiumFrame @@ -16,6 +16,7 @@ from .session_element import SessionElement from .session_page import SessionPage from .setter import TabSetter from .setter import WebPageTabSetter +from .waiter import ChromiumTabWaiter from .web_page import WebPage @@ -23,6 +24,7 @@ class ChromiumTab(ChromiumBase): def __init__(self, page: ChromiumPage, tab_id: str = None): self._page: ChromiumPage = ... + self._browser: Browser = ... def _set_runtime_settings(self) -> None: ... @@ -44,6 +46,7 @@ class ChromiumTab(ChromiumBase): class WebPageTab(SessionPage, ChromiumTab): def __init__(self, page: WebPage, tab_id: str): self._page: WebPage = ... + self._browser: Browser = ... self._mode: str = ... self._has_driver = ... self._has_session = ... diff --git a/DrissionPage/commons/browser.py b/DrissionPage/commons/browser.py index c0d42d0..326bc7c 100644 --- a/DrissionPage/commons/browser.py +++ b/DrissionPage/commons/browser.py @@ -155,9 +155,7 @@ def test_connect(ip, port): end_time = perf_counter() + 30 while perf_counter() < end_time: try: - u = f'http://{ip}:{port}/json' - tabs = requests_get(u, timeout=10, proxies={'http': None, 'https': None}).json() - requests_get(u, headers={'Connection': 'close'}, proxies={'http': None, 'https': None}) + tabs = requests_get(f'http://{ip}:{port}/json', timeout=10, headers={'Connection': 'close'}, proxies={'http': None, 'https': None}).json() for tab in tabs: if tab['type'] == 'page': return diff --git a/DrissionPage/setter.py b/DrissionPage/setter.py index fefe638..3decfca 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/setter.py @@ -175,7 +175,7 @@ class ChromiumPageSetter(TabSetter): tab_or_id = self._page.tab_id elif not isinstance(tab_or_id, str): # 传入Tab对象 tab_or_id = tab_or_id.tab_id - self._page._control_session.get(f'http://{self._page.address}/json/activate/{tab_or_id}') + self._page.browser.activate_tab(tab_or_id) class SessionPageSetter(object): diff --git a/DrissionPage/waiter.py b/DrissionPage/waiter.py index 1d80385..71dfc46 100644 --- a/DrissionPage/waiter.py +++ b/DrissionPage/waiter.py @@ -237,20 +237,20 @@ class ChromiumPageWaiter(ChromiumTabWaiter): :return: 是否等待成功 """ if not timeout: - while self._driver._dl_mgr._missions: + while self._driver.browser._dl_mgr._missions: sleep(.5) return True else: end_time = perf_counter() + timeout while end_time > perf_counter(): - if not self._driver._dl_mgr._missions: + if not self._driver.browser._dl_mgr._missions: return True sleep(.5) - if self._driver._dl_mgr._missions: + if self._driver.browser._dl_mgr._missions: if cancel_if_timeout: - for m in list(self._driver._dl_mgr._missions.values()): + for m in list(self._driver.browser._dl_mgr._missions.values()): m.cancel() return False else: diff --git a/DrissionPage/web_page.py b/DrissionPage/web_page.py index 3282a54..15e179c 100644 --- a/DrissionPage/web_page.py +++ b/DrissionPage/web_page.py @@ -6,7 +6,6 @@ from requests import Session from .base import BasePage -from .browser_download_manager import BrowserDownloadManager from .chromium_base import ChromiumBase, Timeout from .chromium_driver import ChromiumDriver from .chromium_page import ChromiumPage @@ -53,7 +52,6 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self._set_runtime_settings() self._connect_browser() self._create_session() - self._dl_mgr = BrowserDownloadManager(self) self.set.timeouts(implicit=timeout) def _set_start_options(self, dr_opt, se_opt): From 955f8c27ae0f4ffad08e52a9709e71917a92bfed Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 19 Oct 2023 16:20:44 +0800 Subject: [PATCH 037/182] =?UTF-8?q?=E9=87=8D=E6=9E=84ChromiumPage=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/browser.py | 23 ++++++---- DrissionPage/browser.pyi | 7 ++- DrissionPage/chromium_base.py | 28 ++++++------ DrissionPage/chromium_base.pyi | 2 +- DrissionPage/chromium_frame.py | 4 +- DrissionPage/chromium_page.py | 84 +++++++++++++--------------------- DrissionPage/chromium_page.pyi | 9 ++-- DrissionPage/chromium_tab.py | 4 +- DrissionPage/web_page.py | 14 +++--- DrissionPage/web_page.pyi | 2 +- 10 files changed, 83 insertions(+), 94 deletions(-) diff --git a/DrissionPage/browser.py b/DrissionPage/browser.py index 36e522e..387ba2d 100644 --- a/DrissionPage/browser.py +++ b/DrissionPage/browser.py @@ -8,18 +8,20 @@ from .chromium_driver import BrowserDriver class Browser(object): BROWSERS = {} - def __new__(cls, browser_id, page): + def __new__(cls, address, browser_id, page): """ - :param browser_id: BrowserDriver对象 + :param address: 浏览器地址 + :param browser_id: 浏览器id :param page: ChromiumPage对象 """ if browser_id in cls.BROWSERS: return cls.BROWSERS[browser_id] return object.__new__(cls) - def __init__(self, browser_id, page): + def __init__(self, address, browser_id, page): """ - :param page: BrowserDriver对象 + :param address: 浏览器地址 + :param browser_id: 浏览器id :param page: ChromiumPage对象 """ if hasattr(self, '_created'): @@ -28,10 +30,11 @@ class Browser(object): Browser.BROWSERS[browser_id] = self self.page = page - self.address = page.address - self._driver = BrowserDriver(browser_id, 'browser', page.address) + self.address = address + self._driver = BrowserDriver(browser_id, 'browser', address) self.id = browser_id self._frames = {} + self._connected = False self._process_id = None r = self.run_cdp('SystemInfo.getProcessInfo') @@ -40,8 +43,6 @@ class Browser(object): self._process_id = i['id'] break - self._dl_mgr = BrowserDownloadManager(self) - self.run_cdp('Target.setDiscoverTargets') self._driver.set_listener('Target.targetDestroyed', self._onTargetDestroyed) @@ -53,6 +54,12 @@ class Browser(object): if i == tab_id: self._frames.pop(k) + def connect_to_page(self): + """执行与page相关的逻辑""" + if not self._connected: + self._dl_mgr = BrowserDownloadManager(self) + self._connected = True + def run_cdp(self, cmd, **cmd_args): """执行Chrome DevTools Protocol语句 :param cmd: 协议项目 diff --git a/DrissionPage/browser.pyi b/DrissionPage/browser.pyi index 91e9fdb..e4672ef 100644 --- a/DrissionPage/browser.pyi +++ b/DrissionPage/browser.pyi @@ -15,10 +15,11 @@ class Browser(object): _frames: dict = ... _process_id: Optional[int] = ... _dl_mgr: BrowserDownloadManager = ... + _connected: bool = ... - def __new__(cls, browser_id: str, page: ChromiumPage): ... + def __new__(cls, address: str, browser_id: str, page: ChromiumPage): ... - def __init__(self, browser_id: str, page: ChromiumPage): ... + def __init__(self, address: str, browser_id: str, page: ChromiumPage): ... def run_cdp(self, cmd, **cmd_args) -> dict: ... @@ -43,6 +44,8 @@ class Browser(object): def get_window_bounds(self) -> dict: ... + def connect_to_page(self) -> None: ... + def _onTargetDestroyed(self, **kwargs): ... def quit(self) -> None: ... diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index 8c10d16..a0c5c2b 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -43,7 +43,7 @@ class ChromiumBase(BasePage): self._root_id = None # object id self._debug = False self._debug_recorder = None - self._tab_obj = None + self._driver = None self._set = None self._screencast = None self._actions = None @@ -103,18 +103,18 @@ class ChromiumBase(BasePage): :return: None """ self._is_loading = True - self._tab_obj = ChromiumDriver(tab_id=tab_id, tab_type='page', address=self.address) + self._driver = ChromiumDriver(tab_id=tab_id, tab_type='page', address=self.address) - self._tab_obj.call_method('DOM.enable') - self._tab_obj.call_method('Page.enable') + self._driver.call_method('DOM.enable') + self._driver.call_method('Page.enable') - self._tab_obj.set_listener('Page.frameStoppedLoading', self._onFrameStoppedLoading) - self._tab_obj.set_listener('Page.frameStartedLoading', self._onFrameStartedLoading) - self._tab_obj.set_listener('DOM.documentUpdated', self._onDocumentUpdated) - self._tab_obj.set_listener('Page.loadEventFired', self._onLoadEventFired) - self._tab_obj.set_listener('Page.frameNavigated', self._onFrameNavigated) - self._tab_obj.set_listener('Page.frameAttached', self._onFrameAttached) - self._tab_obj.set_listener('Page.frameDetached', self._onFrameDetached) + self._driver.set_listener('Page.frameStoppedLoading', self._onFrameStoppedLoading) + self._driver.set_listener('Page.frameStartedLoading', self._onFrameStartedLoading) + self._driver.set_listener('DOM.documentUpdated', self._onDocumentUpdated) + self._driver.set_listener('Page.loadEventFired', self._onLoadEventFired) + self._driver.set_listener('Page.frameNavigated', self._onFrameNavigated) + self._driver.set_listener('Page.frameAttached', self._onFrameAttached) + self._driver.set_listener('Page.frameDetached', self._onFrameDetached) def _get_document(self): """刷新cdp使用的document数据""" @@ -283,9 +283,9 @@ class ChromiumBase(BasePage): @property def driver(self): """返回用于控制浏览器的ChromiumDriver对象""" - if self._tab_obj is None: + if self._driver is None: raise RuntimeError('浏览器已关闭或链接已断开。') - return self._tab_obj + return self._driver @property def is_loading(self): @@ -738,7 +738,7 @@ class ChromiumBase(BasePage): raise TypeError('必须传入定位符、iframe序号、id、name、ChromiumFrame对象其中之一。') def get_frames(self, loc=None, timeout=None): - """获取所有符号条件的frame对象 + """获取所有符合条件的frame对象 :param loc: 定位符，为None时返回所有 :param timeout: 查找超时时间 :return: ChromiumFrame对象组成的列表 diff --git a/DrissionPage/chromium_base.pyi b/DrissionPage/chromium_base.pyi index 22ac5df..9f6d569 100644 --- a/DrissionPage/chromium_base.pyi +++ b/DrissionPage/chromium_base.pyi @@ -30,7 +30,7 @@ class ChromiumBase(BasePage): self._browser: Browser = ... self._page: ChromiumPage = ... self.address: str = ... - self._tab_obj: ChromiumDriver = ... + self._driver: ChromiumDriver = ... self._is_reading: bool = ... self._timeouts: Timeout = ... self._first_run: bool = ... diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/chromium_frame.py index 6ad88f8..ca42c25 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/chromium_frame.py @@ -110,7 +110,7 @@ class ChromiumFrame(ChromiumBase): self._debug = debug else: self._is_diff_domain = True - self._tab_obj.stop() + self._driver.stop() super().__init__(self.address, self.frame_id, self._target_page.timeout) obj_id = super().run_js('document;', as_expr=True)['objectId'] self.doc_ele = ChromiumElement(self, obj_id=obj_id) @@ -118,7 +118,7 @@ class ChromiumFrame(ChromiumBase): def _check_ok(self): """用于应付同域异域之间跳转导致元素丢失问题""" - if self._tab_obj._stopped.is_set(): + if self._driver._stopped.is_set(): self._reload() try: diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index ab1ece7..9358f99 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -10,11 +10,9 @@ from requests import get from .browser import Browser from .chromium_base import ChromiumBase, Timeout -from .chromium_driver import ChromiumDriver from .chromium_tab import ChromiumTab from .commons.browser import connect_browser from .configs.chromium_options import ChromiumOptions -from .errors import BrowserConnectError from .setter import ChromiumPageSetter from .waiter import ChromiumPageWaiter @@ -22,41 +20,48 @@ from .waiter import ChromiumPageWaiter class ChromiumPage(ChromiumBase): """用于管理浏览器的类""" - def __init__(self, addr_driver_opts=None, tab_id=None, timeout=None): + def __init__(self, addr_or_opts=None, tab_id=None, timeout=None, addr_driver_opts=None): """ - :param addr_driver_opts: 浏览器地址:端口、ChromiumDriver对象或ChromiumOptions对象 + :param addr_or_opts: 浏览器地址:端口或ChromiumOptions对象 :param tab_id: 要控制的标签页id，不指定默认为激活的 :param timeout: 超时时间 """ + if addr_driver_opts: + addr_or_opts = addr_driver_opts self._page = self - super().__init__(addr_driver_opts, tab_id) + address = self._handle_options(addr_or_opts) + self._run_browser() + super().__init__(address, tab_id) self.set.timeouts(implicit=timeout) + self._page_init() - def _set_start_options(self, addr_driver_opts, none): + def _handle_options(self, addr_or_opts): """设置浏览器启动属性 - :param addr_driver_opts: 'ip:port'、ChromiumOptions - :param none: 用于后代继承 - :return: None + :param addr_or_opts: 'ip:port'、ChromiumOptions + :return: 返回浏览器地址 """ - if not addr_driver_opts or isinstance(addr_driver_opts, ChromiumOptions): - self._driver_options = addr_driver_opts or ChromiumOptions(addr_driver_opts) + if not addr_or_opts: + self._driver_options = ChromiumOptions(addr_or_opts) + + elif isinstance(addr_or_opts, ChromiumOptions): + self._driver_options = addr_or_opts # 接收浏览器地址和端口 - elif isinstance(addr_driver_opts, str): + elif isinstance(addr_or_opts, str): self._driver_options = ChromiumOptions() - self._driver_options.debugger_address = addr_driver_opts - - # 接收传递过来的ChromiumDriver，浏览器 - elif isinstance(addr_driver_opts, ChromiumDriver): - self._driver_options = ChromiumOptions(read_file=False) - self._driver_options.debugger_address = addr_driver_opts.address - self._tab_obj = addr_driver_opts + self._driver_options.debugger_address = addr_or_opts else: - raise TypeError('只能接收ChromiumDriver或ChromiumOptions类型参数。') + raise TypeError('只能接收ip:port格式或ChromiumOptions类型参数。') - self.address = self._driver_options.debugger_address.replace('localhost', - '127.0.0.1').lstrip('http://').lstrip('https://') + return self._driver_options.debugger_address + + def _run_browser(self): + """连接浏览器""" + connect_browser(self._driver_options) + ws = get(f'http://{self._driver_options.debugger_address}/json/version', + headers={'Connection': 'close'}).json()['webSocketDebuggerUrl'] + self._browser = Browser(self._driver_options.debugger_address, ws.split('/')[-1], self) def _set_runtime_settings(self): """设置运行时用到的属性""" @@ -69,40 +74,17 @@ class ChromiumPage(ChromiumBase): self._page_load_strategy = self._driver_options.page_load_strategy self._download_path = str(Path(self._driver_options.download_path).absolute()) - def _connect_browser(self, tab_id=None): - """连接浏览器，在第一次时运行 - :param tab_id: 要控制的标签页id，不指定默认为激活的 - :return: None - """ - self._chromium_init() - - if not self._tab_obj: # 不是传入driver的情况 - connect_browser(self._driver_options) - if not tab_id: - json = get(f'http://{self.address}/json', headers={'Connection': 'close'}).json() - tab_id = [i['id'] for i in json if i['type'] == 'page'] - if not tab_id: - raise BrowserConnectError('浏览器连接失败，可能是浏览器版本原因。') - tab_id = tab_id[0] - - self._driver_init(tab_id) - - self._page_init() - self._get_document() - self._first_run = False - def _page_init(self): """浏览器相关设置""" - ws = get(f'http://{self.address}/json/version', headers={'Connection': 'close'}).json()['webSocketDebuggerUrl'] - self._browser = Browser(ws.split('/')[-1], self) - self._alert = Alert() - self._tab_obj.set_listener('Page.javascriptDialogOpening', self._on_alert_open) - self._tab_obj.set_listener('Page.javascriptDialogClosed', self._on_alert_close) + self._driver.set_listener('Page.javascriptDialogOpening', self._on_alert_open) + self._driver.set_listener('Page.javascriptDialogClosed', self._on_alert_close) self._rect = None self._main_tab = self.tab_id + self._browser.connect_to_page() + @property def browser(self): """返回用于控制浏览器cdp的driver""" @@ -327,7 +309,7 @@ class ChromiumPage(ChromiumBase): self._alert.defaultPrompt = None self._alert.response_accept = kwargs.get('result') self._alert.response_text = kwargs['userInput'] - self._tab_obj.has_alert = False + self._driver.has_alert = False def _on_alert_open(self, **kwargs): """alert出现时触发的方法""" @@ -337,7 +319,7 @@ class ChromiumPage(ChromiumBase): self._alert.defaultPrompt = kwargs.get('defaultPrompt', None) self._alert.response_accept = None self._alert.response_text = None - self._tab_obj.has_alert = True + self._driver.has_alert = True class ChromiumTabRect(object): diff --git a/DrissionPage/chromium_page.pyi b/DrissionPage/chromium_page.pyi index 90f9f09..5b05ae5 100644 --- a/DrissionPage/chromium_page.pyi +++ b/DrissionPage/chromium_page.pyi @@ -7,7 +7,6 @@ from typing import Union, Tuple, List, Optional from .browser import Browser from .chromium_base import ChromiumBase -from .chromium_driver import ChromiumDriver from .chromium_tab import ChromiumTab from .configs.chromium_options import ChromiumOptions from .setter import ChromiumPageSetter @@ -17,7 +16,7 @@ from .waiter import ChromiumPageWaiter class ChromiumPage(ChromiumBase): def __init__(self, - addr_driver_opts: Union[str, int, ChromiumOptions, ChromiumDriver] = None, + addr_or_opts: Union[str, int, ChromiumOptions] = None, tab_id: str = None, timeout: float = None): self._driver_options: ChromiumOptions = ... @@ -26,11 +25,9 @@ class ChromiumPage(ChromiumBase): self._browser: Browser = ... self._rect: ChromiumTabRect = ... - def _connect_browser(self, - addr_driver_opts: Union[str, ChromiumDriver] = None, - tab_id: str = None) -> None: ... + def _handle_options(self, addr_or_opts) -> str: ... - def _set_start_options(self, addr_driver_opts: Union[str, ChromiumDriver], none) -> None: ... + def _run_browser(self) -> None: ... def _page_init(self) -> None: ... diff --git a/DrissionPage/chromium_tab.py b/DrissionPage/chromium_tab.py index 449a0a7..9b1d60d 100644 --- a/DrissionPage/chromium_tab.py +++ b/DrissionPage/chromium_tab.py @@ -109,7 +109,7 @@ class WebPageTab(SessionPage, ChromiumTab): @property def _browser_url(self): """返回浏览器当前url""" - return super(SessionPage, self).url if self._tab_obj else None + return super(SessionPage, self).url if self._driver else None @property def title(self): @@ -279,7 +279,7 @@ class WebPageTab(SessionPage, ChromiumTab): # s模式转d模式 if self._mode == 'd': - if self._tab_obj is None: + if self._driver is None: self._connect_browser(self.page._driver_options) self._url = None if not self._has_driver else super(SessionPage, self).url diff --git a/DrissionPage/web_page.py b/DrissionPage/web_page.py index 15e179c..426eae5 100644 --- a/DrissionPage/web_page.py +++ b/DrissionPage/web_page.py @@ -39,7 +39,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self.address = None self._session = None - self._tab_obj = None + self._driver = None self._driver_options = None self._session_options = None self._response = None @@ -62,7 +62,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): """ # 浏览器配置 if isinstance(dr_opt, ChromiumDriver): - self._tab_obj = dr_opt + self._driver = dr_opt self._driver_options = ChromiumOptions() self._driver_options.debugger_address = dr_opt.address dr_opt = False @@ -141,7 +141,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): @property def _browser_url(self): """返回浏览器当前url""" - return super(SessionPage, self).url if self._tab_obj else None + return super(SessionPage, self).url if self._driver else None @property def title(self): @@ -311,7 +311,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): # s模式转d模式 if self._mode == 'd': - if self._tab_obj is None: + if self._driver is None: self._connect_browser(self._driver_options) self._url = None if not self._has_driver else super(SessionPage, self).url @@ -393,8 +393,8 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self.driver.call_method('Browser.close') except Exception: pass - self._tab_obj.stop() - self._tab_obj = None + self._driver.stop() + self._driver = None self._has_driver = None def close_session(self): @@ -430,5 +430,5 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self._has_session = None if self._has_driver: super(SessionPage, self).quit() - self._tab_obj = None + self._driver = None self._has_driver = None diff --git a/DrissionPage/web_page.pyi b/DrissionPage/web_page.pyi index 15f30f8..460f2cf 100644 --- a/DrissionPage/web_page.pyi +++ b/DrissionPage/web_page.pyi @@ -36,7 +36,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self._driver_options: Union[ChromiumOptions, None] = ... self._DownloadKit: DownloadKit = ... self._download_path: str = ... - self._tab_obj: ChromiumDriver = ... + self._driver: ChromiumDriver = ... self._frames: dict = ... def __call__(self, From b822784fdc29273cc88e86d13f76990e979284f9 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 19 Oct 2023 18:02:13 +0800 Subject: [PATCH 038/182] =?UTF-8?q?=E9=87=8D=E6=9E=84WebPage=E5=92=8CWebPa?= =?UTF-8?q?geTab=E9=80=BB=E8=BE=91=EF=BC=8C=E6=9C=AA=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_base.py | 23 ++++----- DrissionPage/chromium_base.pyi | 6 +-- DrissionPage/chromium_frame.py | 2 +- DrissionPage/chromium_frame.pyi | 2 +- DrissionPage/chromium_page.py | 2 +- DrissionPage/chromium_tab.py | 25 +++------ DrissionPage/chromium_tab.pyi | 2 +- DrissionPage/session_page.py | 8 +-- DrissionPage/session_page.pyi | 6 +-- DrissionPage/web_page.py | 92 ++------------------------------- DrissionPage/web_page.pyi | 5 -- 11 files changed, 33 insertions(+), 140 deletions(-) diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index a0c5c2b..8b6d0c3 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -54,13 +54,13 @@ class ChromiumBase(BasePage): if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): address = f'127.0.0.1:{address}' - self._set_start_options(address, None) - self._set_runtime_settings() + self._d_set_start_options(address, None) + self._d_set_runtime_settings() self._connect_browser(tab_id) if timeout is not None: self.timeout = timeout - def _set_start_options(self, address, none): + def _d_set_start_options(self, address, none): """设置浏览器启动属性 :param address: 'ip:port' :param none: 用于后代继承 @@ -68,7 +68,7 @@ class ChromiumBase(BasePage): """ self.address = address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') - def _set_runtime_settings(self): + def _d_set_runtime_settings(self): self._timeouts = Timeout(self) self._page_load_strategy = 'normal' @@ -77,7 +77,12 @@ class ChromiumBase(BasePage): :param tab_id: 要控制的标签页id，不指定默认为激活的 :return: None """ - self._chromium_init() + self._first_run = True + self._is_reading = False + self._upload_list = None + self._wait = None + self._scroll = None + if not tab_id: json = get(f'http://{self.address}/json', headers={'Connection': 'close'}).json() @@ -89,14 +94,6 @@ class ChromiumBase(BasePage): self._get_document() self._first_run = False - def _chromium_init(self): - """浏览器初始设置""" - self._first_run = True - self._is_reading = False - self._upload_list = None - self._wait = None - self._scroll = None - def _driver_init(self, tab_id): """新建页面、页面刷新、切换标签页后要进行的cdp参数初始化 :param tab_id: 要跳转到的标签页id diff --git a/DrissionPage/chromium_base.pyi b/DrissionPage/chromium_base.pyi index 9f6d569..3a90a70 100644 --- a/DrissionPage/chromium_base.pyi +++ b/DrissionPage/chromium_base.pyi @@ -50,8 +50,6 @@ class ChromiumBase(BasePage): def _connect_browser(self, tab_id: str = None) -> None: ... - def _chromium_init(self): ... - def _driver_init(self, tab_id: str) -> None: ... def _get_document(self) -> None: ... @@ -76,9 +74,9 @@ class ChromiumBase(BasePage): # def _onDownloadWillBegin(self, **kwargs): ... - def _set_start_options(self, address, none) -> None: ... + def _d_set_start_options(self, address, none) -> None: ... - def _set_runtime_settings(self) -> None: ... + def _d_set_runtime_settings(self) -> None: ... def __call__(self, loc_or_str: Union[Tuple[str, str], str, ChromiumElement], timeout: float = None) -> ChromiumElement: ... diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/chromium_frame.py index ca42c25..71d3b0d 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/chromium_frame.py @@ -74,7 +74,7 @@ class ChromiumFrame(ChromiumBase): attrs = [f"{attr}='{attrs[attr]}'" for attr in attrs] return f'' - def _set_runtime_settings(self): + def _d_set_runtime_settings(self): """重写设置浏览器运行参数方法""" self._timeouts = copy(self._target_page.timeouts) self.retry_times = self._target_page.retry_times diff --git a/DrissionPage/chromium_frame.pyi b/DrissionPage/chromium_frame.pyi index c125fa4..086565a 100644 --- a/DrissionPage/chromium_frame.pyi +++ b/DrissionPage/chromium_frame.pyi @@ -38,7 +38,7 @@ class ChromiumFrame(ChromiumBase): def __repr__(self) -> str: ... - def _set_runtime_settings(self) -> None: ... + def _d_set_runtime_settings(self) -> None: ... def _driver_init(self, tab_id: str) -> None: ... diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 9358f99..3ff1b95 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -63,7 +63,7 @@ class ChromiumPage(ChromiumBase): headers={'Connection': 'close'}).json()['webSocketDebuggerUrl'] self._browser = Browser(self._driver_options.debugger_address, ws.split('/')[-1], self) - def _set_runtime_settings(self): + def _d_set_runtime_settings(self): """设置运行时用到的属性""" self._timeouts = Timeout(self, page_load=self._driver_options.timeouts['pageLoad'], diff --git a/DrissionPage/chromium_tab.py b/DrissionPage/chromium_tab.py index 9b1d60d..556fff4 100644 --- a/DrissionPage/chromium_tab.py +++ b/DrissionPage/chromium_tab.py @@ -5,12 +5,13 @@ """ from copy import copy -from .waiter import ChromiumTabWaiter +from .base import BasePage from .chromium_base import ChromiumBase from .commons.web import set_session_cookies, set_browser_cookies from .session_page import SessionPage from .setter import TabSetter from .setter import WebPageTabSetter +from .waiter import ChromiumTabWaiter class ChromiumTab(ChromiumBase): @@ -25,7 +26,7 @@ class ChromiumTab(ChromiumBase): self._browser = page.browser super().__init__(page.address, tab_id, page.timeout) - def _set_runtime_settings(self): + def _d_set_runtime_settings(self): """重写设置浏览器运行参数方法""" self._timeouts = copy(self.page.timeouts) self.retry_times = self.page.retry_times @@ -62,29 +63,17 @@ class ChromiumTab(ChromiumBase): return self._wait -class WebPageTab(SessionPage, ChromiumTab): +class WebPageTab(SessionPage, ChromiumTab, BasePage): def __init__(self, page, tab_id): """ :param page: WebPage对象 :param tab_id: 要控制的标签页id """ - self._page = page - self._browser = page.browser - self.address = page.address - self._debug = page._debug - self._debug_recorder = page._debug_recorder self._mode = 'd' self._has_driver = True self._has_session = True - self._session = copy(page.session) - self._response = None - self._set = None - - self._download_set = None - self._download_path = page.download_path - self._DownloadKit = None - super(SessionPage, self)._set_runtime_settings() - self._connect_browser(tab_id) + super().__init__(session_or_options=copy(page.session)) + super(SessionPage, self).__init__(page=page, tab_id=tab_id) def __call__(self, loc_or_str, timeout=None): """在内部查找元素 @@ -351,4 +340,4 @@ class WebPageTab(SessionPage, ChromiumTab): return super()._find_elements(loc_or_ele, single=single) elif self._mode == 'd': return super(SessionPage, self)._find_elements(loc_or_ele, timeout=timeout, single=single, - relative=relative) + relative=relative) \ No newline at end of file diff --git a/DrissionPage/chromium_tab.pyi b/DrissionPage/chromium_tab.pyi index 15d04b4..8a6db8c 100644 --- a/DrissionPage/chromium_tab.pyi +++ b/DrissionPage/chromium_tab.pyi @@ -26,7 +26,7 @@ class ChromiumTab(ChromiumBase): self._page: ChromiumPage = ... self._browser: Browser = ... - def _set_runtime_settings(self) -> None: ... + def _d_set_runtime_settings(self) -> None: ... def close(self) -> None: ... diff --git a/DrissionPage/session_page.py b/DrissionPage/session_page.py index 0931de5..fdad403 100644 --- a/DrissionPage/session_page.py +++ b/DrissionPage/session_page.py @@ -30,13 +30,13 @@ class SessionPage(BasePage): self._response = None self._session = None self._set = None - self._set_start_options(session_or_options, None) - self._set_runtime_settings() + self._s_set_start_options(session_or_options, None) + self._s_set_runtime_settings() self._create_session() if timeout is not None: self.timeout = timeout - def _set_start_options(self, session_or_options, none): + def _s_set_start_options(self, session_or_options, none): """启动配置 :param session_or_options: Session、SessionOptions :param none: 用于后代继承 @@ -49,7 +49,7 @@ class SessionPage(BasePage): self._session_options = SessionOptions() self._session = session_or_options - def _set_runtime_settings(self): + def _s_set_runtime_settings(self): """设置运行时用到的属性""" self._timeout = self._session_options.timeout self._download_path = self._session_options.download_path diff --git a/DrissionPage/session_page.pyi b/DrissionPage/session_page.pyi index 5391a4a..76bba05 100644 --- a/DrissionPage/session_page.pyi +++ b/DrissionPage/session_page.pyi @@ -31,12 +31,12 @@ class SessionPage(BasePage): self.retry_interval: float = ... self._set: SessionPageSetter = ... - def _set_start_options(self, session_or_options, none) -> None: ... + def _s_set_start_options(self, session_or_options, none) -> None: ... + + def _s_set_runtime_settings(self) -> None: ... def _create_session(self) -> None: ... - def _set_runtime_settings(self) -> None: ... - def __call__(self, loc_or_str: Union[Tuple[str, str], str, SessionElement], timeout: float = None) -> Union[SessionElement, str, NoneElement]: ... diff --git a/DrissionPage/web_page.py b/DrissionPage/web_page.py index 426eae5..ce1aabf 100644 --- a/DrissionPage/web_page.py +++ b/DrissionPage/web_page.py @@ -3,16 +3,11 @@ @Author : g1879 @Contact : g1879@qq.com """ -from requests import Session from .base import BasePage -from .chromium_base import ChromiumBase, Timeout -from .chromium_driver import ChromiumDriver from .chromium_page import ChromiumPage from .chromium_tab import WebPageTab from .commons.web import set_session_cookies, set_browser_cookies -from .configs.chromium_options import ChromiumOptions -from .configs.session_options import SessionOptions from .session_page import SessionPage from .setter import WebPageSetter @@ -27,96 +22,15 @@ class WebPage(SessionPage, ChromiumPage, BasePage): :param driver_or_options: ChromiumDriver对象，只使用s模式时应传入False :param session_or_options: Session对象或SessionOptions对象，只使用d模式时应传入False """ - super(ChromiumBase, self).__init__() # 调用Base的__init__() self._mode = mode.lower() if self._mode not in ('s', 'd'): raise ValueError('mode参数只能是s或d。') self._has_driver = True self._has_session = True - self._debug = False - self._debug_recorder = None - self.address = None - - self._session = None - self._driver = None - self._driver_options = None - self._session_options = None - self._response = None - self._set = None - self._screencast = None - self._frames = {} - self._page = self - - self._set_start_options(driver_or_options, session_or_options) - self._set_runtime_settings() - self._connect_browser() - self._create_session() - self.set.timeouts(implicit=timeout) - - def _set_start_options(self, dr_opt, se_opt): - """处理两种模式的设置 - :param dr_opt: ChromiumDriver或ChromiumOptions对象，为None则从ini读取，为False用默认信息创建 - :param se_opt: Session、SessionOptions对象或配置信息，为None则从ini读取，为False用默认信息创建 - :return: None - """ - # 浏览器配置 - if isinstance(dr_opt, ChromiumDriver): - self._driver = dr_opt - self._driver_options = ChromiumOptions() - self._driver_options.debugger_address = dr_opt.address - dr_opt = False - - else: - if dr_opt is None: - self._driver_options = ChromiumOptions() - - elif dr_opt is False: - self._driver_options = ChromiumOptions(read_file=False) - - elif isinstance(dr_opt, ChromiumOptions): - self._driver_options = dr_opt - - else: - raise TypeError('driver_or_options参数只能接收ChromiumDriver, ChromiumOptions、None或False。') - - self.address = self._driver_options.debugger_address.replace('localhost', - '127.0.0.1').lstrip('http://').lstrip('https://') - - # Session配置 - if isinstance(se_opt, Session): - self._session = se_opt - self._session_options = SessionOptions() - se_opt = False - - else: - if se_opt is None: - self._session_options = SessionOptions() - - elif se_opt is False: - self._session_options = SessionOptions(read_file=False) - - elif isinstance(se_opt, SessionOptions): - self._session_options = se_opt - - else: - raise TypeError('session_or_options参数只能接收Session, SessionOptions、None或False。') - - self._timeouts = Timeout(self) - self._page_load_strategy = self._driver_options.page_load_strategy - - if se_opt is not False: - self.set.timeouts(implicit=self._session_options.timeout) - self._download_path = self._session_options.download_path - - if dr_opt is not False: - t = self._driver_options.timeouts - self.set.timeouts(t['implicit'], t['pageLoad'], t['script']) - self._download_path = self._driver_options.download_path - - def _set_runtime_settings(self): - """设置运行时用到的属性""" - pass + super().__init__(session_or_options=session_or_options, timeout=timeout) + super(SessionPage, self).__init__(addr_or_opts=driver_or_options, timeout=timeout) + self.change_mode(self._mode, go=False, copy_cookies=False) def __call__(self, loc_or_str, timeout=None): """在内部查找元素 diff --git a/DrissionPage/web_page.pyi b/DrissionPage/web_page.pyi index 460f2cf..a70dc0e 100644 --- a/DrissionPage/web_page.pyi +++ b/DrissionPage/web_page.pyi @@ -31,13 +31,8 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self._mode: str = ... self._has_driver: bool = ... self._has_session: bool = ... - self.address: str = ... self._session_options: Union[SessionOptions, None] = ... self._driver_options: Union[ChromiumOptions, None] = ... - self._DownloadKit: DownloadKit = ... - self._download_path: str = ... - self._driver: ChromiumDriver = ... - self._frames: dict = ... def __call__(self, loc_or_str: Union[Tuple[str, str], str, ChromiumElement, SessionElement], From 19f99b4d62302e7d9c6c682837b94df658738c4d Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 20 Oct 2023 00:13:00 +0800 Subject: [PATCH 039/182] =?UTF-8?q?=E5=AE=8C=E5=96=84=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E5=AF=B9=E8=B1=A1=E5=88=9D=E5=A7=8B=E5=8C=96=E9=87=8D=E6=9E=84?= =?UTF-8?q?=EF=BC=9BPage=E6=81=A2=E5=A4=8D=E6=8E=A5=E6=94=B6Driver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/chromium_base.py | 3 ++- DrissionPage/chromium_page.py | 32 ++++++++++++++++++-------------- DrissionPage/chromium_page.pyi | 5 +++-- DrissionPage/session_page.py | 2 +- DrissionPage/web_page.py | 9 ++++++--- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/DrissionPage/chromium_base.py b/DrissionPage/chromium_base.py index 8b6d0c3..d820918 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/chromium_base.py @@ -43,7 +43,6 @@ class ChromiumBase(BasePage): self._root_id = None # object id self._debug = False self._debug_recorder = None - self._driver = None self._set = None self._screencast = None self._actions = None @@ -100,6 +99,8 @@ class ChromiumBase(BasePage): :return: None """ self._is_loading = True + if hasattr(self, '_driver'): + return self._driver = ChromiumDriver(tab_id=tab_id, tab_type='page', address=self.address) self._driver.call_method('DOM.enable') diff --git a/DrissionPage/chromium_page.py b/DrissionPage/chromium_page.py index 3ff1b95..bbee74b 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/chromium_page.py @@ -10,6 +10,7 @@ from requests import get from .browser import Browser from .chromium_base import ChromiumBase, Timeout +from .chromium_driver import ChromiumDriver from .chromium_tab import ChromiumTab from .commons.browser import connect_browser from .configs.chromium_options import ChromiumOptions @@ -20,39 +21,42 @@ from .waiter import ChromiumPageWaiter class ChromiumPage(ChromiumBase): """用于管理浏览器的类""" - def __init__(self, addr_or_opts=None, tab_id=None, timeout=None, addr_driver_opts=None): + def __init__(self, addr_driver_opts=None, tab_id=None, timeout=None): """ - :param addr_or_opts: 浏览器地址:端口或ChromiumOptions对象 + :param addr_driver_opts: 浏览器地址:端口或ChromiumOptions对象 :param tab_id: 要控制的标签页id，不指定默认为激活的 :param timeout: 超时时间 """ - if addr_driver_opts: - addr_or_opts = addr_driver_opts self._page = self - address = self._handle_options(addr_or_opts) + address = self._handle_options(addr_driver_opts) self._run_browser() super().__init__(address, tab_id) self.set.timeouts(implicit=timeout) self._page_init() - def _handle_options(self, addr_or_opts): + def _handle_options(self, addr_driver_opts): """设置浏览器启动属性 - :param addr_or_opts: 'ip:port'、ChromiumOptions + :param addr_driver_opts: 'ip:port'、ChromiumOptions、ChromiumDriver :return: 返回浏览器地址 """ - if not addr_or_opts: - self._driver_options = ChromiumOptions(addr_or_opts) + if not addr_driver_opts: + self._driver_options = ChromiumOptions(addr_driver_opts) - elif isinstance(addr_or_opts, ChromiumOptions): - self._driver_options = addr_or_opts + elif isinstance(addr_driver_opts, ChromiumOptions): + self._driver_options = addr_driver_opts # 接收浏览器地址和端口 - elif isinstance(addr_or_opts, str): + elif isinstance(addr_driver_opts, str): self._driver_options = ChromiumOptions() - self._driver_options.debugger_address = addr_or_opts + self._driver_options.debugger_address = addr_driver_opts + + elif isinstance(addr_driver_opts, ChromiumDriver): + self._driver_options = ChromiumOptions(False) + self._driver_options.debugger_address = addr_driver_opts.address + self._driver = addr_driver_opts else: - raise TypeError('只能接收ip:port格式或ChromiumOptions类型参数。') + raise TypeError('只能接收ip:port格式、ChromiumOptions或ChromiumDriver类型参数。') return self._driver_options.debugger_address diff --git a/DrissionPage/chromium_page.pyi b/DrissionPage/chromium_page.pyi index 5b05ae5..a152caa 100644 --- a/DrissionPage/chromium_page.pyi +++ b/DrissionPage/chromium_page.pyi @@ -7,6 +7,7 @@ from typing import Union, Tuple, List, Optional from .browser import Browser from .chromium_base import ChromiumBase +from .chromium_driver import ChromiumDriver from .chromium_tab import ChromiumTab from .configs.chromium_options import ChromiumOptions from .setter import ChromiumPageSetter @@ -16,7 +17,7 @@ from .waiter import ChromiumPageWaiter class ChromiumPage(ChromiumBase): def __init__(self, - addr_or_opts: Union[str, int, ChromiumOptions] = None, + addr_driver_opts: Union[str, int, ChromiumOptions, ChromiumDriver] = None, tab_id: str = None, timeout: float = None): self._driver_options: ChromiumOptions = ... @@ -25,7 +26,7 @@ class ChromiumPage(ChromiumBase): self._browser: Browser = ... self._rect: ChromiumTabRect = ... - def _handle_options(self, addr_or_opts) -> str: ... + def _handle_options(self, addr_driver_opts: Union[str, ChromiumDriver, ChromiumOptions]) -> str: ... def _run_browser(self) -> None: ... diff --git a/DrissionPage/session_page.py b/DrissionPage/session_page.py index fdad403..9bf1e1a 100644 --- a/DrissionPage/session_page.py +++ b/DrissionPage/session_page.py @@ -26,7 +26,7 @@ class SessionPage(BasePage): :param session_or_options: Session对象或SessionOptions对象 :param timeout: 连接超时时间，为None时从ini文件读取 """ - super().__init__() + super(SessionPage, SessionPage).__init__(self) self._response = None self._session = None self._set = None diff --git a/DrissionPage/web_page.py b/DrissionPage/web_page.py index ce1aabf..05dd9f5 100644 --- a/DrissionPage/web_page.py +++ b/DrissionPage/web_page.py @@ -3,11 +3,11 @@ @Author : g1879 @Contact : g1879@qq.com """ - from .base import BasePage from .chromium_page import ChromiumPage from .chromium_tab import WebPageTab from .commons.web import set_session_cookies, set_browser_cookies +from .configs.chromium_options import ChromiumOptions from .session_page import SessionPage from .setter import WebPageSetter @@ -28,8 +28,11 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self._has_driver = True self._has_session = True - super().__init__(session_or_options=session_or_options, timeout=timeout) - super(SessionPage, self).__init__(addr_or_opts=driver_or_options, timeout=timeout) + super().__init__(session_or_options=session_or_options) + if not driver_or_options: + driver_or_options = ChromiumOptions(read_file=driver_or_options) + driver_or_options.set_timeouts(implicit=self._timeout).set_paths(download_path=self.download_path) + super(SessionPage, self).__init__(addr_driver_opts=driver_or_options, timeout=timeout) self.change_mode(self._mode, go=False, copy_cookies=False) def __call__(self, loc_or_str, timeout=None): From aafbc7a839be9041b781a6bd759043b4fbcf9f27 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 20 Oct 2023 14:35:44 +0800 Subject: [PATCH 040/182] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 10 +- DrissionPage/{ => _base}/base.py | 10 +- DrissionPage/{ => _base}/base.pyi | 2 +- DrissionPage/{ => _base}/browser.py | 6 +- DrissionPage/{ => _base}/browser.pyi | 8 +- DrissionPage/{ => _base}/chromium_driver.py | 0 DrissionPage/{ => _base}/chromium_driver.pyi | 0 DrissionPage/{commons => _commons}/browser.py | 0 .../{commons => _commons}/browser.pyi | 2 +- DrissionPage/{commons => _commons}/by.py | 0 DrissionPage/{commons => _commons}/cli.py | 7 +- .../{commons => _commons}/constants.py | 0 DrissionPage/{commons => _commons}/keys.py | 0 DrissionPage/{commons => _commons}/locator.py | 0 .../{commons => _commons}/locator.pyi | 0 DrissionPage/{commons => _commons}/tools.py | 0 DrissionPage/{commons => _commons}/tools.pyi | 2 +- DrissionPage/{commons => _commons}/web.py | 0 DrissionPage/{commons => _commons}/web.pyi | 4 +- .../{configs => _configs}/chromium_options.py | 2 +- .../chromium_options.pyi | 0 .../{configs => _configs}/configs.ini | 0 .../{configs => _configs}/options_manage.py | 0 .../{configs => _configs}/options_manage.pyi | 0 .../{configs => _configs}/session_options.py | 2 +- .../{configs => _configs}/session_options.pyi | 0 .../{ => _elements}/chromium_element.py | 20 +- .../{ => _elements}/chromium_element.pyi | 18 +- .../{ => _elements}/session_element.py | 8 +- .../{ => _elements}/session_element.pyi | 12 +- DrissionPage/{ => _pages}/chromium_base.py | 28 +- DrissionPage/{ => _pages}/chromium_base.pyi | 24 +- DrissionPage/{ => _pages}/chromium_frame.py | 10 +- DrissionPage/{ => _pages}/chromium_frame.pyi | 13 +- DrissionPage/{ => _pages}/chromium_page.py | 16 +- DrissionPage/{ => _pages}/chromium_page.pyi | 14 +- DrissionPage/{ => _pages}/chromium_tab.py | 13 +- DrissionPage/{ => _pages}/chromium_tab.pyi | 21 +- DrissionPage/{ => _pages}/session_page.py | 672 +++++++++--------- DrissionPage/{ => _pages}/session_page.pyi | 10 +- DrissionPage/{ => _pages}/web_page.py | 12 +- DrissionPage/{ => _pages}/web_page.pyi | 15 +- DrissionPage/{ => _units}/action_chains.py | 4 +- DrissionPage/{ => _units}/action_chains.pyi | 7 +- .../{ => _units}/browser_download_manager.py | 2 +- .../{ => _units}/browser_download_manager.pyi | 4 +- DrissionPage/{ => _units}/network_listener.py | 6 +- .../{ => _units}/network_listener.pyi | 9 +- DrissionPage/{ => _units}/setter.py | 4 +- DrissionPage/{ => _units}/setter.pyi | 14 +- DrissionPage/{ => _units}/waiter.py | 4 +- DrissionPage/{ => _units}/waiter.pyi | 8 +- DrissionPage/common.py | 10 +- DrissionPage/common.pyi | 15 +- DrissionPage/easy_set.py | 4 +- DrissionPage/errors.py | 4 + 56 files changed, 541 insertions(+), 515 deletions(-) rename DrissionPage/{ => _base}/base.py (98%) rename DrissionPage/{ => _base}/base.pyi (99%) rename DrissionPage/{ => _base}/browser.py (97%) rename DrissionPage/{ => _base}/browser.pyi (86%) rename DrissionPage/{ => _base}/chromium_driver.py (100%) rename DrissionPage/{ => _base}/chromium_driver.pyi (100%) rename DrissionPage/{commons => _commons}/browser.py (100%) rename DrissionPage/{commons => _commons}/browser.pyi (78%) rename DrissionPage/{commons => _commons}/by.py (100%) rename DrissionPage/{commons => _commons}/cli.py (86%) rename DrissionPage/{commons => _commons}/constants.py (100%) rename DrissionPage/{commons => _commons}/keys.py (100%) rename DrissionPage/{commons => _commons}/locator.py (100%) rename DrissionPage/{commons => _commons}/locator.pyi (100%) rename DrissionPage/{commons => _commons}/tools.py (100%) rename DrissionPage/{commons => _commons}/tools.pyi (93%) rename DrissionPage/{commons => _commons}/web.py (100%) rename DrissionPage/{commons => _commons}/web.pyi (90%) rename DrissionPage/{configs => _configs}/chromium_options.py (99%) rename DrissionPage/{configs => _configs}/chromium_options.pyi (100%) rename DrissionPage/{configs => _configs}/configs.ini (100%) rename DrissionPage/{configs => _configs}/options_manage.py (100%) rename DrissionPage/{configs => _configs}/options_manage.pyi (100%) rename DrissionPage/{configs => _configs}/session_options.py (99%) rename DrissionPage/{configs => _configs}/session_options.pyi (100%) rename DrissionPage/{ => _elements}/chromium_element.py (99%) rename DrissionPage/{ => _elements}/chromium_element.pyi (96%) rename DrissionPage/{ => _elements}/session_element.py (98%) rename DrissionPage/{ => _elements}/session_element.pyi (92%) rename DrissionPage/{ => _pages}/chromium_base.py (97%) rename DrissionPage/{ => _pages}/chromium_base.pyi (92%) rename DrissionPage/{ => _pages}/chromium_frame.py (98%) rename DrissionPage/{ => _pages}/chromium_frame.pyi (93%) rename DrissionPage/{ => _pages}/chromium_page.py (96%) rename DrissionPage/{ => _pages}/chromium_page.pyi (89%) rename DrissionPage/{ => _pages}/chromium_tab.py (97%) rename DrissionPage/{ => _pages}/chromium_tab.pyi (88%) rename DrissionPage/{ => _pages}/session_page.py (95%) rename DrissionPage/{ => _pages}/session_page.pyi (94%) rename DrissionPage/{ => _pages}/web_page.py (97%) rename DrissionPage/{ => _pages}/web_page.pyi (94%) rename DrissionPage/{ => _units}/action_chains.py (98%) rename DrissionPage/{ => _units}/action_chains.pyi (92%) rename DrissionPage/{ => _units}/browser_download_manager.py (99%) rename DrissionPage/{ => _units}/browser_download_manager.pyi (95%) rename DrissionPage/{ => _units}/network_listener.py (99%) rename DrissionPage/{ => _units}/network_listener.pyi (95%) rename DrissionPage/{ => _units}/setter.py (99%) rename DrissionPage/{ => _units}/setter.pyi (92%) rename DrissionPage/{ => _units}/waiter.py (99%) rename DrissionPage/{ => _units}/waiter.pyi (93%) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 1f553d7..8a2eaf3 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -4,10 +4,10 @@ @Contact : g1879@qq.com """ # 常用页面类 -from .chromium_page import ChromiumPage -from .session_page import SessionPage -from .web_page import WebPage +from ._pages.chromium_page import ChromiumPage +from ._pages.session_page import SessionPage +from ._pages.web_page import WebPage # 启动配置类 -from .configs.chromium_options import ChromiumOptions -from .configs.session_options import SessionOptions +from ._configs.chromium_options import ChromiumOptions +from ._configs.session_options import SessionOptions diff --git a/DrissionPage/base.py b/DrissionPage/_base/base.py similarity index 98% rename from DrissionPage/base.py rename to DrissionPage/_base/base.py index 48880a6..51d7182 100644 --- a/DrissionPage/base.py +++ b/DrissionPage/_base/base.py @@ -10,10 +10,10 @@ from urllib.parse import quote from DownloadKit import DownloadKit -from .commons.constants import Settings, NoneElement -from .commons.locator import get_loc -from .commons.web import format_html -from .errors import ElementNotFoundError +from DrissionPage._commons.constants import Settings, NoneElement +from DrissionPage._commons.locator import get_loc +from DrissionPage._commons.web import format_html +from DrissionPage.errors import ElementNotFoundError class BaseParser(object): @@ -367,7 +367,7 @@ class BasePage(BaseParser): self.retry_times = 3 self.retry_interval = 2 self._DownloadKit = None - self._download_path = str(Path('.').absolute()) + self._download_path = str(Path('../..').absolute()) @property def title(self): diff --git a/DrissionPage/base.pyi b/DrissionPage/_base/base.pyi similarity index 99% rename from DrissionPage/base.pyi rename to DrissionPage/_base/base.pyi index 6c49719..3ec97a4 100644 --- a/DrissionPage/base.pyi +++ b/DrissionPage/_base/base.pyi @@ -8,7 +8,7 @@ from typing import Union, Tuple, List from DownloadKit import DownloadKit -from .commons.constants import NoneElement +from DrissionPage._commons.constants import NoneElement class BaseParser(object): diff --git a/DrissionPage/browser.py b/DrissionPage/_base/browser.py similarity index 97% rename from DrissionPage/browser.py rename to DrissionPage/_base/browser.py index 387ba2d..9019b7f 100644 --- a/DrissionPage/browser.py +++ b/DrissionPage/_base/browser.py @@ -1,7 +1,11 @@ # -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" from time import sleep -from .browser_download_manager import BrowserDownloadManager +from DrissionPage._units.browser_download_manager import BrowserDownloadManager from .chromium_driver import BrowserDriver diff --git a/DrissionPage/browser.pyi b/DrissionPage/_base/browser.pyi similarity index 86% rename from DrissionPage/browser.pyi rename to DrissionPage/_base/browser.pyi index e4672ef..cdee003 100644 --- a/DrissionPage/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -1,8 +1,12 @@ # -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" from typing import List, Optional, Union -from .browser_download_manager import BrowserDownloadManager -from .chromium_page import ChromiumPage +from DrissionPage._units.browser_download_manager import BrowserDownloadManager +from DrissionPage._pages.chromium_page import ChromiumPage from .chromium_driver import BrowserDriver diff --git a/DrissionPage/chromium_driver.py b/DrissionPage/_base/chromium_driver.py similarity index 100% rename from DrissionPage/chromium_driver.py rename to DrissionPage/_base/chromium_driver.py diff --git a/DrissionPage/chromium_driver.pyi b/DrissionPage/_base/chromium_driver.pyi similarity index 100% rename from DrissionPage/chromium_driver.pyi rename to DrissionPage/_base/chromium_driver.pyi diff --git a/DrissionPage/commons/browser.py b/DrissionPage/_commons/browser.py similarity index 100% rename from DrissionPage/commons/browser.py rename to DrissionPage/_commons/browser.py diff --git a/DrissionPage/commons/browser.pyi b/DrissionPage/_commons/browser.pyi similarity index 78% rename from DrissionPage/commons/browser.pyi rename to DrissionPage/_commons/browser.pyi index ede46db..0294aff 100644 --- a/DrissionPage/commons/browser.pyi +++ b/DrissionPage/_commons/browser.pyi @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from DrissionPage.configs.chromium_options import ChromiumOptions +from DrissionPage._configs.chromium_options import ChromiumOptions def connect_browser(option: ChromiumOptions) -> tuple: ... diff --git a/DrissionPage/commons/by.py b/DrissionPage/_commons/by.py similarity index 100% rename from DrissionPage/commons/by.py rename to DrissionPage/_commons/by.py diff --git a/DrissionPage/commons/cli.py b/DrissionPage/_commons/cli.py similarity index 86% rename from DrissionPage/commons/cli.py rename to DrissionPage/_commons/cli.py index cc82107..71f2c28 100644 --- a/DrissionPage/commons/cli.py +++ b/DrissionPage/_commons/cli.py @@ -1,6 +1,11 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" from click import command, option -from DrissionPage import ChromiumPage +from DrissionPage._pages.chromium_page import ChromiumPage from DrissionPage.easy_set import set_paths, configs_to_here as ch diff --git a/DrissionPage/commons/constants.py b/DrissionPage/_commons/constants.py similarity index 100% rename from DrissionPage/commons/constants.py rename to DrissionPage/_commons/constants.py diff --git a/DrissionPage/commons/keys.py b/DrissionPage/_commons/keys.py similarity index 100% rename from DrissionPage/commons/keys.py rename to DrissionPage/_commons/keys.py diff --git a/DrissionPage/commons/locator.py b/DrissionPage/_commons/locator.py similarity index 100% rename from DrissionPage/commons/locator.py rename to DrissionPage/_commons/locator.py diff --git a/DrissionPage/commons/locator.pyi b/DrissionPage/_commons/locator.pyi similarity index 100% rename from DrissionPage/commons/locator.pyi rename to DrissionPage/_commons/locator.pyi diff --git a/DrissionPage/commons/tools.py b/DrissionPage/_commons/tools.py similarity index 100% rename from DrissionPage/commons/tools.py rename to DrissionPage/_commons/tools.py diff --git a/DrissionPage/commons/tools.pyi b/DrissionPage/_commons/tools.pyi similarity index 93% rename from DrissionPage/commons/tools.pyi rename to DrissionPage/_commons/tools.pyi index ba8d6ee..83dcff1 100644 --- a/DrissionPage/commons/tools.pyi +++ b/DrissionPage/_commons/tools.pyi @@ -8,7 +8,7 @@ from pathlib import Path from typing import Union from types import FunctionType -from chromium_page import ChromiumPage +from DrissionPage._pages.chromium_page import ChromiumPage def get_usable_path(path: Union[str, Path], is_file: bool = True, parents: bool = True) -> Path: ... diff --git a/DrissionPage/commons/web.py b/DrissionPage/_commons/web.py similarity index 100% rename from DrissionPage/commons/web.py rename to DrissionPage/_commons/web.py diff --git a/DrissionPage/commons/web.pyi b/DrissionPage/_commons/web.pyi similarity index 90% rename from DrissionPage/commons/web.pyi rename to DrissionPage/_commons/web.pyi index f4b4931..4a365e9 100644 --- a/DrissionPage/commons/web.pyi +++ b/DrissionPage/_commons/web.pyi @@ -10,8 +10,8 @@ from requests import Session from requests.cookies import RequestsCookieJar from DrissionPage.base import DrissionElement, BasePage -from DrissionPage.chromium_element import ChromiumElement -from DrissionPage.chromium_base import ChromiumBase +from DrissionPage._chromium_element import ChromiumElement +from DrissionPage._chromium_base import ChromiumBase def get_ele_txt(e: DrissionElement) -> str: ... diff --git a/DrissionPage/configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py similarity index 99% rename from DrissionPage/configs/chromium_options.py rename to DrissionPage/_configs/chromium_options.py index 6f32d90..c44529a 100644 --- a/DrissionPage/configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -6,7 +6,7 @@ from pathlib import Path from tempfile import gettempdir, TemporaryDirectory -from DrissionPage.commons.tools import port_is_using, clean_folder +from DrissionPage._commons.tools import port_is_using, clean_folder from .options_manage import OptionsManager diff --git a/DrissionPage/configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi similarity index 100% rename from DrissionPage/configs/chromium_options.pyi rename to DrissionPage/_configs/chromium_options.pyi diff --git a/DrissionPage/configs/configs.ini b/DrissionPage/_configs/configs.ini similarity index 100% rename from DrissionPage/configs/configs.ini rename to DrissionPage/_configs/configs.ini diff --git a/DrissionPage/configs/options_manage.py b/DrissionPage/_configs/options_manage.py similarity index 100% rename from DrissionPage/configs/options_manage.py rename to DrissionPage/_configs/options_manage.py diff --git a/DrissionPage/configs/options_manage.pyi b/DrissionPage/_configs/options_manage.pyi similarity index 100% rename from DrissionPage/configs/options_manage.pyi rename to DrissionPage/_configs/options_manage.pyi diff --git a/DrissionPage/configs/session_options.py b/DrissionPage/_configs/session_options.py similarity index 99% rename from DrissionPage/configs/session_options.py rename to DrissionPage/_configs/session_options.py index 03fc800..1aa7c1d 100644 --- a/DrissionPage/configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -8,7 +8,7 @@ from pathlib import Path from requests import Session from requests.structures import CaseInsensitiveDict -from DrissionPage.commons.web import cookies_to_tuple, set_session_cookies +from DrissionPage._commons.web import cookies_to_tuple, set_session_cookies from .options_manage import OptionsManager diff --git a/DrissionPage/configs/session_options.pyi b/DrissionPage/_configs/session_options.pyi similarity index 100% rename from DrissionPage/configs/session_options.pyi rename to DrissionPage/_configs/session_options.pyi diff --git a/DrissionPage/chromium_element.py b/DrissionPage/_elements/chromium_element.py similarity index 99% rename from DrissionPage/chromium_element.py rename to DrissionPage/_elements/chromium_element.py index cf12697..f5fc1fd 100644 --- a/DrissionPage/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -7,17 +7,17 @@ from os.path import basename, sep from pathlib import Path from time import perf_counter, sleep -from .base import DrissionElement, BaseElement -from .commons.constants import FRAME_ELEMENT, NoneElement, Settings -from .commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions -from .commons.locator import get_loc -from .commons.tools import get_usable_path -from .commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, location_in_viewport, offset_scroll -from .errors import ContextLossError, ElementLossError, JavaScriptError, NoRectError, ElementNotFoundError, \ +from DrissionPage._base.base import DrissionElement, BaseElement +from DrissionPage._commons.constants import FRAME_ELEMENT, NoneElement, Settings +from DrissionPage._commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions +from DrissionPage._commons.locator import get_loc +from DrissionPage._commons.tools import get_usable_path +from DrissionPage._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, location_in_viewport, offset_scroll +from DrissionPage.errors import ContextLossError, ElementLossError, JavaScriptError, NoRectError, ElementNotFoundError, \ CDPError, NoResourceError, CanNotClickError from .session_element import make_session_ele -from .setter import ChromiumElementSetter -from .waiter import ChromiumElementWaiter +from DrissionPage._units.setter import ChromiumElementSetter +from DrissionPage._units.waiter import ChromiumElementWaiter class ChromiumElement(DrissionElement): @@ -1203,7 +1203,7 @@ def make_chromium_ele(page, node_id=None, obj_id=None): ele = ChromiumElement(page, obj_id=obj_id, node_id=node_id, backend_id=backend_id) if ele.tag in FRAME_ELEMENT: - from .chromium_frame import ChromiumFrame + from ._chromium_frame import ChromiumFrame ele = ChromiumFrame(page, ele) return ele diff --git a/DrissionPage/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi similarity index 96% rename from DrissionPage/chromium_element.pyi rename to DrissionPage/_elements/chromium_element.pyi index 03355d2..a7b0dcb 100644 --- a/DrissionPage/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -6,15 +6,15 @@ from pathlib import Path from typing import Union, Tuple, List, Any -from .base import DrissionElement, BaseElement -from .chromium_base import ChromiumBase -from .chromium_frame import ChromiumFrame -from .chromium_page import ChromiumPage -from .commons.constants import NoneElement -from .session_element import SessionElement -from .setter import ChromiumElementSetter -from .waiter import ChromiumElementWaiter -from .web_page import WebPage +from DrissionPage._base.base import DrissionElement, BaseElement +from DrissionPage._commons.constants import NoneElement +from DrissionPage._elements.session_element import SessionElement +from DrissionPage._pages.chromium_base import ChromiumBase +from DrissionPage._pages.chromium_frame import ChromiumFrame +from DrissionPage._pages.chromium_page import ChromiumPage +from DrissionPage._pages.web_page import WebPage +from DrissionPage._units.setter import ChromiumElementSetter +from DrissionPage._units.waiter import ChromiumElementWaiter class ChromiumElement(DrissionElement): diff --git a/DrissionPage/session_element.py b/DrissionPage/_elements/session_element.py similarity index 98% rename from DrissionPage/session_element.py rename to DrissionPage/_elements/session_element.py index 7e8bb88..fa241cb 100644 --- a/DrissionPage/session_element.py +++ b/DrissionPage/_elements/session_element.py @@ -9,10 +9,10 @@ from re import match, DOTALL from lxml.etree import tostring from lxml.html import HtmlElement, fromstring -from .base import DrissionElement, BasePage, BaseElement -from .commons.constants import NoneElement -from .commons.locator import get_loc -from .commons.web import get_ele_txt, make_absolute_link +from DrissionPage._base.base import DrissionElement, BasePage, BaseElement +from DrissionPage._commons.constants import NoneElement +from DrissionPage._commons.locator import get_loc +from DrissionPage._commons.web import get_ele_txt, make_absolute_link class SessionElement(DrissionElement): diff --git a/DrissionPage/session_element.pyi b/DrissionPage/_elements/session_element.pyi similarity index 92% rename from DrissionPage/session_element.pyi rename to DrissionPage/_elements/session_element.pyi index c55dcfe..9257354 100644 --- a/DrissionPage/session_element.pyi +++ b/DrissionPage/_elements/session_element.pyi @@ -7,12 +7,12 @@ from typing import Union, List, Tuple from lxml.html import HtmlElement -from .base import DrissionElement, BaseElement -from .chromium_base import ChromiumBase -from .chromium_element import ChromiumElement -from .chromium_frame import ChromiumFrame -from .commons.constants import NoneElement -from .session_page import SessionPage +from DrissionPage._base.base import DrissionElement, BaseElement +from DrissionPage._commons.constants import NoneElement +from DrissionPage._elements.chromium_element import ChromiumElement +from DrissionPage._pages.chromium_base import ChromiumBase +from DrissionPage._pages.chromium_frame import ChromiumFrame +from DrissionPage._pages.session_page import SessionPage class SessionElement(DrissionElement): diff --git a/DrissionPage/chromium_base.py b/DrissionPage/_pages/chromium_base.py similarity index 97% rename from DrissionPage/chromium_base.py rename to DrissionPage/_pages/chromium_base.py index d820918..c79b8be 100644 --- a/DrissionPage/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -13,20 +13,20 @@ from time import perf_counter, sleep, time from requests import get -from .action_chains import ActionChains -from .base import BasePage -from .chromium_driver import ChromiumDriver -from .chromium_element import ChromiumScroll, ChromiumElement, run_js, make_chromium_ele -from .commons.constants import HANDLE_ALERT_METHOD, ERROR, NoneElement -from .commons.locator import get_loc -from .commons.tools import get_usable_path, clean_folder -from .commons.web import location_in_viewport -from .errors import ContextLossError, ElementLossError, AlertExistsError, CDPError, TabClosedError, \ +from DrissionPage._base.base import BasePage +from DrissionPage._base.chromium_driver import ChromiumDriver +from DrissionPage._commons.constants import HANDLE_ALERT_METHOD, ERROR, NoneElement +from DrissionPage._commons.locator import get_loc +from DrissionPage._commons.tools import get_usable_path, clean_folder +from DrissionPage._commons.web import location_in_viewport +from DrissionPage._elements.chromium_element import ChromiumScroll, ChromiumElement, run_js, make_chromium_ele +from DrissionPage._elements.session_element import make_session_ele +from DrissionPage._units.network_listener import NetworkListener +from DrissionPage._units.setter import ChromiumBaseSetter +from DrissionPage._units.waiter import ChromiumBaseWaiter +from DrissionPage.errors import ContextLossError, ElementLossError, AlertExistsError, CDPError, TabClosedError, \ NoRectError, BrowserConnectError, GetDocumentError -from .network_listener import NetworkListener -from .session_element import make_session_ele -from .setter import ChromiumBaseSetter -from .waiter import ChromiumBaseWaiter +from _units.action_chains import ActionChains class ChromiumBase(BasePage): @@ -48,7 +48,7 @@ class ChromiumBase(BasePage): self._actions = None self._listener = None - self._download_path = str(Path('.').absolute()) + self._download_path = str(Path('../..').absolute()) if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): address = f'127.0.0.1:{address}' diff --git a/DrissionPage/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi similarity index 92% rename from DrissionPage/chromium_base.pyi rename to DrissionPage/_pages/chromium_base.pyi index 3a90a70..97020ad 100644 --- a/DrissionPage/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -8,18 +8,18 @@ from typing import Union, Tuple, List, Any from DataRecorder import Recorder -from .browser import Browser -from .action_chains import ActionChains -from .base import BasePage -from .chromium_driver import ChromiumDriver -from .chromium_element import ChromiumElement, ChromiumScroll -from .chromium_frame import ChromiumFrame -from .chromium_page import ChromiumPage -from .commons.constants import NoneElement -from .network_listener import NetworkListener -from .session_element import SessionElement -from .setter import ChromiumBaseSetter -from .waiter import ChromiumBaseWaiter +from DrissionPage._base.base import BasePage +from DrissionPage._base.browser import Browser +from DrissionPage._base.chromium_driver import ChromiumDriver +from DrissionPage._commons.constants import NoneElement +from DrissionPage._elements.chromium_element import ChromiumElement, ChromiumScroll +from DrissionPage._elements.session_element import SessionElement +from DrissionPage._pages.chromium_frame import ChromiumFrame +from DrissionPage._pages.chromium_page import ChromiumPage +from DrissionPage._units.action_chains import ActionChains +from DrissionPage._units.network_listener import NetworkListener +from DrissionPage._units.setter import ChromiumBaseSetter +from DrissionPage._units.waiter import ChromiumBaseWaiter class ChromiumBase(BasePage): diff --git a/DrissionPage/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py similarity index 98% rename from DrissionPage/chromium_frame.py rename to DrissionPage/_pages/chromium_frame.py index 71d3b0d..ca037f0 100644 --- a/DrissionPage/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -10,11 +10,11 @@ from time import sleep, perf_counter from requests import get -from .chromium_base import ChromiumBase, ChromiumPageScroll -from .chromium_element import ChromiumElement -from .errors import ContextLossError -from .setter import ChromiumFrameSetter -from .waiter import FrameWaiter +from DrissionPage._elements.chromium_element import ChromiumElement +from DrissionPage._pages.chromium_base import ChromiumBase, ChromiumPageScroll +from DrissionPage._units.setter import ChromiumFrameSetter +from DrissionPage._units.waiter import FrameWaiter +from DrissionPage.errors import ContextLossError class ChromiumFrame(ChromiumBase): diff --git a/DrissionPage/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi similarity index 93% rename from DrissionPage/chromium_frame.pyi rename to DrissionPage/_pages/chromium_frame.pyi index 086565a..1be67ad 100644 --- a/DrissionPage/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -6,12 +6,13 @@ from pathlib import Path from typing import Union, Tuple, List, Any -from DrissionPage import ChromiumPage, WebPage -from .chromium_tab import ChromiumTab -from .chromium_base import ChromiumBase, ChromiumPageScroll -from .chromium_element import ChromiumElement, Locations, ChromiumElementStates -from .setter import ChromiumFrameSetter -from .waiter import FrameWaiter +from DrissionPage._elements.chromium_element import ChromiumElement, Locations, ChromiumElementStates +from DrissionPage._pages.chromium_base import ChromiumBase, ChromiumPageScroll +from DrissionPage._pages.chromium_page import ChromiumPage +from DrissionPage._pages.chromium_tab import ChromiumTab +from DrissionPage._pages.web_page import WebPage +from DrissionPage._units.setter import ChromiumFrameSetter +from DrissionPage._units.waiter import FrameWaiter class ChromiumFrame(ChromiumBase): diff --git a/DrissionPage/chromium_page.py b/DrissionPage/_pages/chromium_page.py similarity index 96% rename from DrissionPage/chromium_page.py rename to DrissionPage/_pages/chromium_page.py index bbee74b..33f343b 100644 --- a/DrissionPage/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -8,14 +8,14 @@ from time import perf_counter, sleep from requests import get -from .browser import Browser -from .chromium_base import ChromiumBase, Timeout -from .chromium_driver import ChromiumDriver -from .chromium_tab import ChromiumTab -from .commons.browser import connect_browser -from .configs.chromium_options import ChromiumOptions -from .setter import ChromiumPageSetter -from .waiter import ChromiumPageWaiter +from DrissionPage._base.browser import Browser +from DrissionPage._base.chromium_driver import ChromiumDriver +from DrissionPage._commons.browser import connect_browser +from DrissionPage._configs.chromium_options import ChromiumOptions +from DrissionPage._pages.chromium_base import ChromiumBase, Timeout +from DrissionPage._pages.chromium_tab import ChromiumTab +from DrissionPage._units.setter import ChromiumPageSetter +from DrissionPage._units.waiter import ChromiumPageWaiter class ChromiumPage(ChromiumBase): diff --git a/DrissionPage/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi similarity index 89% rename from DrissionPage/chromium_page.pyi rename to DrissionPage/_pages/chromium_page.pyi index a152caa..884a452 100644 --- a/DrissionPage/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -5,13 +5,13 @@ """ from typing import Union, Tuple, List, Optional -from .browser import Browser -from .chromium_base import ChromiumBase -from .chromium_driver import ChromiumDriver -from .chromium_tab import ChromiumTab -from .configs.chromium_options import ChromiumOptions -from .setter import ChromiumPageSetter -from .waiter import ChromiumPageWaiter +from DrissionPage._base.browser import Browser +from DrissionPage._pages.chromium_base import ChromiumBase +from DrissionPage._base.chromium_driver import ChromiumDriver +from DrissionPage._pages.chromium_tab import ChromiumTab +from DrissionPage._configs.chromium_options import ChromiumOptions +from DrissionPage._units.setter import ChromiumPageSetter +from DrissionPage._units.waiter import ChromiumPageWaiter class ChromiumPage(ChromiumBase): diff --git a/DrissionPage/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py similarity index 97% rename from DrissionPage/chromium_tab.py rename to DrissionPage/_pages/chromium_tab.py index 556fff4..6e44f6d 100644 --- a/DrissionPage/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -5,13 +5,12 @@ """ from copy import copy -from .base import BasePage -from .chromium_base import ChromiumBase -from .commons.web import set_session_cookies, set_browser_cookies -from .session_page import SessionPage -from .setter import TabSetter -from .setter import WebPageTabSetter -from .waiter import ChromiumTabWaiter +from DrissionPage._base.base import BasePage +from DrissionPage._commons.web import set_session_cookies, set_browser_cookies +from DrissionPage._pages.chromium_base import ChromiumBase +from DrissionPage._pages.session_page import SessionPage +from DrissionPage._units.setter import TabSetter, WebPageTabSetter +from DrissionPage._units.waiter import ChromiumTabWaiter class ChromiumTab(ChromiumBase): diff --git a/DrissionPage/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi similarity index 88% rename from DrissionPage/chromium_tab.pyi rename to DrissionPage/_pages/chromium_tab.pyi index 8a6db8c..6f19d4b 100644 --- a/DrissionPage/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -7,17 +7,16 @@ from typing import Union, Tuple, Any, List from requests import Session, Response -from .browser import Browser -from .chromium_base import ChromiumBase -from .chromium_element import ChromiumElement -from .chromium_frame import ChromiumFrame -from .chromium_page import ChromiumPage, ChromiumTabRect -from .session_element import SessionElement -from .session_page import SessionPage -from .setter import TabSetter -from .setter import WebPageTabSetter -from .waiter import ChromiumTabWaiter -from .web_page import WebPage +from DrissionPage._base.browser import Browser +from DrissionPage._elements.chromium_element import ChromiumElement +from DrissionPage._elements.session_element import SessionElement +from DrissionPage._pages.chromium_base import ChromiumBase +from DrissionPage._pages.chromium_frame import ChromiumFrame +from DrissionPage._pages.chromium_page import ChromiumPage, ChromiumTabRect +from DrissionPage._pages.session_page import SessionPage +from DrissionPage._pages.web_page import WebPage +from DrissionPage._units.setter import TabSetter, WebPageTabSetter +from DrissionPage._units.waiter import ChromiumTabWaiter class ChromiumTab(ChromiumBase): diff --git a/DrissionPage/session_page.py b/DrissionPage/_pages/session_page.py similarity index 95% rename from DrissionPage/session_page.py rename to DrissionPage/_pages/session_page.py index 9bf1e1a..846ba9b 100644 --- a/DrissionPage/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -1,336 +1,336 @@ -# -*- coding:utf-8 -*- -""" -@Author : g1879 -@Contact : g1879@qq.com -""" -from re import search -from time import sleep -from urllib.parse import urlparse - -from requests import Session -from requests.structures import CaseInsensitiveDict -from tldextract import extract - -from .base import BasePage -from .commons.web import cookie_to_dict -from .configs.session_options import SessionOptions -from .session_element import SessionElement, make_session_ele -from .setter import SessionPageSetter - - -class SessionPage(BasePage): - """SessionPage封装了页面操作的常用功能，使用requests来获取、解析网页""" - - def __init__(self, session_or_options=None, timeout=None): - """ - :param session_or_options: Session对象或SessionOptions对象 - :param timeout: 连接超时时间，为None时从ini文件读取 - """ - super(SessionPage, SessionPage).__init__(self) - self._response = None - self._session = None - self._set = None - self._s_set_start_options(session_or_options, None) - self._s_set_runtime_settings() - self._create_session() - if timeout is not None: - self.timeout = timeout - - def _s_set_start_options(self, session_or_options, none): - """启动配置 - :param session_or_options: Session、SessionOptions - :param none: 用于后代继承 - :return: None - """ - if not session_or_options or isinstance(session_or_options, SessionOptions): - self._session_options = session_or_options or SessionOptions(session_or_options) - - elif isinstance(session_or_options, Session): - self._session_options = SessionOptions() - self._session = session_or_options - - def _s_set_runtime_settings(self): - """设置运行时用到的属性""" - self._timeout = self._session_options.timeout - self._download_path = self._session_options.download_path - - def _create_session(self): - """创建内建Session对象""" - if not self._session: - self._session = self._session_options.make_session() - - def __call__(self, loc_or_str, timeout=None): - """在内部查找元素 - 例：ele2 = ele1('@id=ele_id') - :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 不起实际作用，用于和ChromiumElement对应，便于无差别调用 - :return: SessionElement对象或属性文本 - """ - return self.ele(loc_or_str) - - # -----------------共有属性和方法------------------- - @property - def title(self): - """返回网页title""" - ele = self._ele('xpath://title', raise_err=False) - return ele.text if ele else None - - @property - def url(self): - """返回当前访问url""" - return self._url - - @property - def _session_url(self): - """返回当前访问url""" - return self._url - - @property - def html(self): - """返回页面的html文本""" - return self.response.text if self.response else '' - - @property - def json(self): - """当返回内容是json格式时，返回对应的字典，非json格式时返回None""" - try: - return self.response.json() - except Exception: - return None - - @property - def user_agent(self): - """返回user agent""" - return self.session.headers.get('user-agent', '') - - @property - def session(self): - """返回session对象""" - return self._session - - @property - def response(self): - """返回访问url得到的response对象""" - return self._response - - @property - def set(self): - """返回用于等待的对象""" - if self._set is None: - self._set = SessionPageSetter(self) - return self._set - - def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None, **kwargs): - """用get方式跳转到url - :param url: 目标url - :param show_errmsg: 是否显示和抛出异常 - :param retry: 重试次数 - :param interval: 重试间隔（秒） - :param timeout: 连接超时时间（秒） - :param kwargs: 连接参数 - :return: url是否可用 - """ - return self._s_connect(url, 'get', None, show_errmsg, retry, interval, **kwargs) - - def ele(self, loc_or_ele, timeout=None): - """返回页面中符合条件的第一个元素、属性或节点文本 - :param loc_or_ele: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 - :param timeout: 不起实际作用，用于和ChromiumElement对应，便于无差别调用 - :return: SessionElement对象或属性、文本 - """ - return self._ele(loc_or_ele) - - def eles(self, loc_or_str, timeout=None): - """返回页面中所有符合条件的元素、属性或节点文本 - :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 不起实际作用，用于和ChromiumElement对应，便于无差别调用 - :return: SessionElement对象或属性、文本组成的列表 - """ - return self._ele(loc_or_str, single=False) - - def s_ele(self, loc_or_ele=None): - """返回页面中符合条件的第一个元素、属性或节点文本 - :param loc_or_ele: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 - :return: SessionElement对象或属性、文本 - """ - return make_session_ele(self.html) if loc_or_ele is None else self._ele(loc_or_ele) - - def s_eles(self, loc_or_str): - """返回页面中符合条件的所有元素、属性或节点文本 - :param loc_or_str: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 - :return: SessionElement对象或属性、文本 - """ - return self._ele(loc_or_str, single=False) - - def _find_elements(self, loc_or_ele, timeout=None, single=True, raise_err=None): - """返回页面中符合条件的元素、属性或节点文本，默认返回第一个 - :param loc_or_ele: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 - :param timeout: 不起实际作用，用于和父类对应 - :param single: True则返回第一个，False则返回全部 - :param raise_err: 找不到元素是是否抛出异常，为None时根据全局设置 - :return: SessionElement对象 - """ - return loc_or_ele if isinstance(loc_or_ele, SessionElement) else make_session_ele(self, loc_or_ele, single) - - def get_cookies(self, as_dict=False, all_domains=False, all_info=False): - """返回cookies - :param as_dict: 是否以字典方式返回，False则以list返回 - :param all_domains: 是否返回所有域的cookies - :param all_info: 是否返回所有信息，False则只返回name、value、domain - :return: cookies信息 - """ - if all_domains: - cookies = self.session.cookies - else: - if self.url: - ex_url = extract(self._session_url) - domain = f'{ex_url.domain}.{ex_url.suffix}' if ex_url.suffix else ex_url.domain - - cookies = tuple(x for x in self.session.cookies if domain in x.domain or x.domain == '') - else: - cookies = tuple(x for x in self.session.cookies) - - if as_dict: - return {x.name: x.value for x in cookies} - elif all_info: - return [cookie_to_dict(cookie) for cookie in cookies] - else: - r = [] - for c in cookies: - c = cookie_to_dict(c) - r.append({'name': c['name'], 'value': c['value'], 'domain': c['domain']}) - return r - - def post(self, url, data=None, show_errmsg=False, retry=None, interval=None, **kwargs): - """用post方式跳转到url - :param url: 目标url - :param data: 提交的数据 - :param show_errmsg: 是否显示和抛出异常 - :param retry: 重试次数 - :param interval: 重试间隔（秒） - :param kwargs: 连接参数 - :return: url是否可用 - """ - return self._s_connect(url, 'post', data, show_errmsg, retry, interval, **kwargs) - - def _s_connect(self, url, mode, data=None, show_errmsg=False, retry=None, interval=None, **kwargs): - """执行get或post连接 - :param url: 目标url - :param mode: 'get' 或 'post' - :param data: 提交的数据 - :param show_errmsg: 是否显示和抛出异常 - :param retry: 重试次数 - :param interval: 重试间隔（秒） - :param kwargs: 连接参数 - :return: url是否可用 - """ - retry, interval = self._before_connect(url, retry, interval) - self._response, info = self._make_response(self._url, mode, data, retry, interval, show_errmsg, **kwargs) - - if self._response is None: - self._url_available = False - - else: - if self._response.ok: - self._url_available = True - - else: - if show_errmsg: - raise ConnectionError(f'状态码：{self._response.status_code}.') - self._url_available = False - - return self._url_available - - def _make_response(self, url, mode='get', data=None, retry=None, interval=None, show_errmsg=False, **kwargs): - """生成Response对象 - :param url: 目标url - :param mode: 'get' 或 'post' - :param data: post方式要提交的数据 - :param show_errmsg: 是否显示和抛出异常 - :param kwargs: 其它参数 - :return: tuple，第一位为Response或None，第二位为出错信息或'Success' - """ - kwargs = CaseInsensitiveDict(kwargs) - if 'headers' not in kwargs: - kwargs['headers'] = {} - else: - kwargs['headers'] = CaseInsensitiveDict(kwargs['headers']) - - # 设置referer和host值 - parsed_url = urlparse(url) - hostname = parsed_url.hostname - scheme = parsed_url.scheme - if not check_headers(kwargs, self.session.headers, 'Referer'): - kwargs['headers']['Referer'] = self.url if self.url else f'{scheme}://{hostname}' - if 'Host' not in kwargs['headers']: - kwargs['headers']['Host'] = hostname - - if not check_headers(kwargs, self.session.headers, 'timeout'): - kwargs['timeout'] = self.timeout - - r = err = None - retry = retry if retry is not None else self.retry_times - interval = interval if interval is not None else self.retry_interval - for i in range(retry + 1): - try: - if mode == 'get': - r = self.session.get(url, **kwargs) - elif mode == 'post': - r = self.session.post(url, data=data, **kwargs) - - if r: - return set_charset(r), 'Success' - - except Exception as e: - err = e - - # if r and r.status_code in (403, 404): - # break - - if i < retry: - sleep(interval) - if show_errmsg: - print(f'重试 {url}') - - if r is None: - if show_errmsg: - if err: - raise err - else: - raise ConnectionError('连接失败') - return None, '连接失败' if err is None else err - - if not r.ok: - if show_errmsg: - raise ConnectionError(f'状态码：{r.status_code}') - return r, f'状态码：{r.status_code}' - - -def check_headers(kwargs, headers, arg): - """检查kwargs或headers中是否有arg所示属性""" - return arg in kwargs['headers'] or arg in headers - - -def set_charset(response): - """设置Response对象的编码""" - # 在headers中获取编码 - content_type = response.headers.get('content-type', '').lower() - if not content_type.endswith(';'): - content_type += ';' - charset = search(r'charset[=: ]*(.*)?;?', content_type) - - if charset: - response.encoding = charset.group(1) - - # 在headers中获取不到编码，且如果是网页 - elif content_type.replace(' ', '').startswith('text/html'): - re_result = search(b']+).*?>', response.content) - - if re_result: - charset = re_result.group(1).decode() - else: - charset = response.apparent_encoding - - response.encoding = charset - - return response +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" +from re import search +from time import sleep +from urllib.parse import urlparse + +from requests import Session +from requests.structures import CaseInsensitiveDict +from tldextract import extract + +from DrissionPage._base.base import BasePage +from DrissionPage._commons.web import cookie_to_dict +from DrissionPage._configs.session_options import SessionOptions +from DrissionPage._elements.session_element import SessionElement, make_session_ele +from DrissionPage._units.setter import SessionPageSetter + + +class SessionPage(BasePage): + """SessionPage封装了页面操作的常用功能，使用requests来获取、解析网页""" + + def __init__(self, session_or_options=None, timeout=None): + """ + :param session_or_options: Session对象或SessionOptions对象 + :param timeout: 连接超时时间，为None时从ini文件读取 + """ + super(SessionPage, SessionPage).__init__(self) + self._response = None + self._session = None + self._set = None + self._s_set_start_options(session_or_options, None) + self._s_set_runtime_settings() + self._create_session() + if timeout is not None: + self.timeout = timeout + + def _s_set_start_options(self, session_or_options, none): + """启动配置 + :param session_or_options: Session、SessionOptions + :param none: 用于后代继承 + :return: None + """ + if not session_or_options or isinstance(session_or_options, SessionOptions): + self._session_options = session_or_options or SessionOptions(session_or_options) + + elif isinstance(session_or_options, Session): + self._session_options = SessionOptions() + self._session = session_or_options + + def _s_set_runtime_settings(self): + """设置运行时用到的属性""" + self._timeout = self._session_options.timeout + self._download_path = self._session_options.download_path + + def _create_session(self): + """创建内建Session对象""" + if not self._session: + self._session = self._session_options.make_session() + + def __call__(self, loc_or_str, timeout=None): + """在内部查找元素 + 例：ele2 = ele1('@id=ele_id') + :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 + :param timeout: 不起实际作用，用于和ChromiumElement对应，便于无差别调用 + :return: SessionElement对象或属性文本 + """ + return self.ele(loc_or_str) + + # -----------------共有属性和方法------------------- + @property + def title(self): + """返回网页title""" + ele = self._ele('xpath://title', raise_err=False) + return ele.text if ele else None + + @property + def url(self): + """返回当前访问url""" + return self._url + + @property + def _session_url(self): + """返回当前访问url""" + return self._url + + @property + def html(self): + """返回页面的html文本""" + return self.response.text if self.response else '' + + @property + def json(self): + """当返回内容是json格式时，返回对应的字典，非json格式时返回None""" + try: + return self.response.json() + except Exception: + return None + + @property + def user_agent(self): + """返回user agent""" + return self.session.headers.get('user-agent', '') + + @property + def session(self): + """返回session对象""" + return self._session + + @property + def response(self): + """返回访问url得到的response对象""" + return self._response + + @property + def set(self): + """返回用于等待的对象""" + if self._set is None: + self._set = SessionPageSetter(self) + return self._set + + def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None, **kwargs): + """用get方式跳转到url + :param url: 目标url + :param show_errmsg: 是否显示和抛出异常 + :param retry: 重试次数 + :param interval: 重试间隔（秒） + :param timeout: 连接超时时间（秒） + :param kwargs: 连接参数 + :return: url是否可用 + """ + return self._s_connect(url, 'get', None, show_errmsg, retry, interval, **kwargs) + + def ele(self, loc_or_ele, timeout=None): + """返回页面中符合条件的第一个元素、属性或节点文本 + :param loc_or_ele: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 + :param timeout: 不起实际作用，用于和ChromiumElement对应，便于无差别调用 + :return: SessionElement对象或属性、文本 + """ + return self._ele(loc_or_ele) + + def eles(self, loc_or_str, timeout=None): + """返回页面中所有符合条件的元素、属性或节点文本 + :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 + :param timeout: 不起实际作用，用于和ChromiumElement对应，便于无差别调用 + :return: SessionElement对象或属性、文本组成的列表 + """ + return self._ele(loc_or_str, single=False) + + def s_ele(self, loc_or_ele=None): + """返回页面中符合条件的第一个元素、属性或节点文本 + :param loc_or_ele: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 + :return: SessionElement对象或属性、文本 + """ + return make_session_ele(self.html) if loc_or_ele is None else self._ele(loc_or_ele) + + def s_eles(self, loc_or_str): + """返回页面中符合条件的所有元素、属性或节点文本 + :param loc_or_str: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 + :return: SessionElement对象或属性、文本 + """ + return self._ele(loc_or_str, single=False) + + def _find_elements(self, loc_or_ele, timeout=None, single=True, raise_err=None): + """返回页面中符合条件的元素、属性或节点文本，默认返回第一个 + :param loc_or_ele: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 + :param timeout: 不起实际作用，用于和父类对应 + :param single: True则返回第一个，False则返回全部 + :param raise_err: 找不到元素是是否抛出异常，为None时根据全局设置 + :return: SessionElement对象 + """ + return loc_or_ele if isinstance(loc_or_ele, SessionElement) else make_session_ele(self, loc_or_ele, single) + + def get_cookies(self, as_dict=False, all_domains=False, all_info=False): + """返回cookies + :param as_dict: 是否以字典方式返回，False则以list返回 + :param all_domains: 是否返回所有域的cookies + :param all_info: 是否返回所有信息，False则只返回name、value、domain + :return: cookies信息 + """ + if all_domains: + cookies = self.session.cookies + else: + if self.url: + ex_url = extract(self._session_url) + domain = f'{ex_url.domain}.{ex_url.suffix}' if ex_url.suffix else ex_url.domain + + cookies = tuple(x for x in self.session.cookies if domain in x.domain or x.domain == '') + else: + cookies = tuple(x for x in self.session.cookies) + + if as_dict: + return {x.name: x.value for x in cookies} + elif all_info: + return [cookie_to_dict(cookie) for cookie in cookies] + else: + r = [] + for c in cookies: + c = cookie_to_dict(c) + r.append({'name': c['name'], 'value': c['value'], 'domain': c['domain']}) + return r + + def post(self, url, data=None, show_errmsg=False, retry=None, interval=None, **kwargs): + """用post方式跳转到url + :param url: 目标url + :param data: 提交的数据 + :param show_errmsg: 是否显示和抛出异常 + :param retry: 重试次数 + :param interval: 重试间隔（秒） + :param kwargs: 连接参数 + :return: url是否可用 + """ + return self._s_connect(url, 'post', data, show_errmsg, retry, interval, **kwargs) + + def _s_connect(self, url, mode, data=None, show_errmsg=False, retry=None, interval=None, **kwargs): + """执行get或post连接 + :param url: 目标url + :param mode: 'get' 或 'post' + :param data: 提交的数据 + :param show_errmsg: 是否显示和抛出异常 + :param retry: 重试次数 + :param interval: 重试间隔（秒） + :param kwargs: 连接参数 + :return: url是否可用 + """ + retry, interval = self._before_connect(url, retry, interval) + self._response, info = self._make_response(self._url, mode, data, retry, interval, show_errmsg, **kwargs) + + if self._response is None: + self._url_available = False + + else: + if self._response.ok: + self._url_available = True + + else: + if show_errmsg: + raise ConnectionError(f'状态码：{self._response.status_code}.') + self._url_available = False + + return self._url_available + + def _make_response(self, url, mode='get', data=None, retry=None, interval=None, show_errmsg=False, **kwargs): + """生成Response对象 + :param url: 目标url + :param mode: 'get' 或 'post' + :param data: post方式要提交的数据 + :param show_errmsg: 是否显示和抛出异常 + :param kwargs: 其它参数 + :return: tuple，第一位为Response或None，第二位为出错信息或'Success' + """ + kwargs = CaseInsensitiveDict(kwargs) + if 'headers' not in kwargs: + kwargs['headers'] = {} + else: + kwargs['headers'] = CaseInsensitiveDict(kwargs['headers']) + + # 设置referer和host值 + parsed_url = urlparse(url) + hostname = parsed_url.hostname + scheme = parsed_url.scheme + if not check_headers(kwargs, self.session.headers, 'Referer'): + kwargs['headers']['Referer'] = self.url if self.url else f'{scheme}://{hostname}' + if 'Host' not in kwargs['headers']: + kwargs['headers']['Host'] = hostname + + if not check_headers(kwargs, self.session.headers, 'timeout'): + kwargs['timeout'] = self.timeout + + r = err = None + retry = retry if retry is not None else self.retry_times + interval = interval if interval is not None else self.retry_interval + for i in range(retry + 1): + try: + if mode == 'get': + r = self.session.get(url, **kwargs) + elif mode == 'post': + r = self.session.post(url, data=data, **kwargs) + + if r: + return set_charset(r), 'Success' + + except Exception as e: + err = e + + # if r and r.status_code in (403, 404): + # break + + if i < retry: + sleep(interval) + if show_errmsg: + print(f'重试 {url}') + + if r is None: + if show_errmsg: + if err: + raise err + else: + raise ConnectionError('连接失败') + return None, '连接失败' if err is None else err + + if not r.ok: + if show_errmsg: + raise ConnectionError(f'状态码：{r.status_code}') + return r, f'状态码：{r.status_code}' + + +def check_headers(kwargs, headers, arg): + """检查kwargs或headers中是否有arg所示属性""" + return arg in kwargs['headers'] or arg in headers + + +def set_charset(response): + """设置Response对象的编码""" + # 在headers中获取编码 + content_type = response.headers.get('content-type', '').lower() + if not content_type.endswith(';'): + content_type += ';' + charset = search(r'charset[=: ]*(.*)?;?', content_type) + + if charset: + response.encoding = charset.group(1) + + # 在headers中获取不到编码，且如果是网页 + elif content_type.replace(' ', '').startswith('text/html'): + re_result = search(b']+).*?>', response.content) + + if re_result: + charset = re_result.group(1).decode() + else: + charset = response.apparent_encoding + + response.encoding = charset + + return response diff --git a/DrissionPage/session_page.pyi b/DrissionPage/_pages/session_page.pyi similarity index 94% rename from DrissionPage/session_page.pyi rename to DrissionPage/_pages/session_page.pyi index 76bba05..145f122 100644 --- a/DrissionPage/session_page.pyi +++ b/DrissionPage/_pages/session_page.pyi @@ -8,11 +8,11 @@ from typing import Any, Union, Tuple, List from requests import Session, Response from requests.structures import CaseInsensitiveDict -from .base import BasePage -from .commons.constants import NoneElement -from .configs.session_options import SessionOptions -from .session_element import SessionElement -from .setter import SessionPageSetter +from DrissionPage._base.base import BasePage +from DrissionPage._commons.constants import NoneElement +from DrissionPage._configs.session_options import SessionOptions +from DrissionPage._elements.session_element import SessionElement +from DrissionPage._units.setter import SessionPageSetter class SessionPage(BasePage): diff --git a/DrissionPage/web_page.py b/DrissionPage/_pages/web_page.py similarity index 97% rename from DrissionPage/web_page.py rename to DrissionPage/_pages/web_page.py index 05dd9f5..058ad99 100644 --- a/DrissionPage/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -3,13 +3,13 @@ @Author : g1879 @Contact : g1879@qq.com """ -from .base import BasePage -from .chromium_page import ChromiumPage -from .chromium_tab import WebPageTab -from .commons.web import set_session_cookies, set_browser_cookies -from .configs.chromium_options import ChromiumOptions +from DrissionPage._base.base import BasePage +from DrissionPage._commons.web import set_session_cookies, set_browser_cookies +from DrissionPage._configs.chromium_options import ChromiumOptions +from DrissionPage._pages.chromium_page import ChromiumPage +from DrissionPage._pages.chromium_tab import WebPageTab +from DrissionPage._units.setter import WebPageSetter from .session_page import SessionPage -from .setter import WebPageSetter class WebPage(SessionPage, ChromiumPage, BasePage): diff --git a/DrissionPage/web_page.pyi b/DrissionPage/_pages/web_page.pyi similarity index 94% rename from DrissionPage/web_page.pyi rename to DrissionPage/_pages/web_page.pyi index a70dc0e..de706f9 100644 --- a/DrissionPage/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -5,20 +5,19 @@ """ from typing import Union, Tuple, List, Any -from DownloadKit import DownloadKit from requests import Session, Response -from .base import BasePage -from .chromium_driver import ChromiumDriver -from .chromium_element import ChromiumElement +from ._base import BasePage +from ._chromium_driver import ChromiumDriver +from ._chromium_element import ChromiumElement from .chromium_frame import ChromiumFrame from .chromium_page import ChromiumPage from .chromium_tab import WebPageTab -from .configs.chromium_options import ChromiumOptions -from .configs.session_options import SessionOptions -from .session_element import SessionElement +from ._configs.chromium_options import ChromiumOptions +from ._configs.session_options import SessionOptions +from ._session_element import SessionElement from .session_page import SessionPage -from .setter import WebPageSetter +from ._units.setter import WebPageSetter class WebPage(SessionPage, ChromiumPage, BasePage): diff --git a/DrissionPage/action_chains.py b/DrissionPage/_units/action_chains.py similarity index 98% rename from DrissionPage/action_chains.py rename to DrissionPage/_units/action_chains.py index 0b5724a..fd733ce 100644 --- a/DrissionPage/action_chains.py +++ b/DrissionPage/_units/action_chains.py @@ -5,8 +5,8 @@ """ from time import sleep -from .commons.keys import modifierBit, keyDescriptionForString -from .commons.web import location_in_viewport +from DrissionPage._commons.keys import modifierBit, keyDescriptionForString +from DrissionPage._commons.web import location_in_viewport class ActionChains: diff --git a/DrissionPage/action_chains.pyi b/DrissionPage/_units/action_chains.pyi similarity index 92% rename from DrissionPage/action_chains.pyi rename to DrissionPage/_units/action_chains.pyi index ca6061a..83163dd 100644 --- a/DrissionPage/action_chains.pyi +++ b/DrissionPage/_units/action_chains.pyi @@ -5,10 +5,9 @@ """ from typing import Union, Tuple -from .chromium_base import ChromiumBase -from .chromium_driver import ChromiumDriver -from .chromium_element import ChromiumElement -from .chromium_page import ChromiumPage +from DrissionPage._base.chromium_driver import ChromiumDriver +from DrissionPage._elements.chromium_element import ChromiumElement +from DrissionPage._pages.chromium_base import ChromiumBase class ActionChains: diff --git a/DrissionPage/browser_download_manager.py b/DrissionPage/_units/browser_download_manager.py similarity index 99% rename from DrissionPage/browser_download_manager.py rename to DrissionPage/_units/browser_download_manager.py index 2d6cba5..546076d 100644 --- a/DrissionPage/browser_download_manager.py +++ b/DrissionPage/_units/browser_download_manager.py @@ -4,7 +4,7 @@ from pathlib import Path from shutil import move from time import sleep, perf_counter -from .commons.tools import get_usable_path +from DrissionPage._commons.tools import get_usable_path class BrowserDownloadManager(object): diff --git a/DrissionPage/browser_download_manager.pyi b/DrissionPage/_units/browser_download_manager.pyi similarity index 95% rename from DrissionPage/browser_download_manager.pyi rename to DrissionPage/_units/browser_download_manager.pyi index 6a9a434..0d4b266 100644 --- a/DrissionPage/browser_download_manager.pyi +++ b/DrissionPage/_units/browser_download_manager.pyi @@ -1,8 +1,8 @@ from pathlib import Path from typing import Dict, Optional, Union -from .browser import Browser -from .chromium_page import ChromiumPage +from DrissionPage._base.browser import Browser +from DrissionPage._pages.chromium_page import ChromiumPage class BrowserDownloadManager(object): diff --git a/DrissionPage/network_listener.py b/DrissionPage/_units/network_listener.py similarity index 99% rename from DrissionPage/network_listener.py rename to DrissionPage/_units/network_listener.py index ee11c08..220b0d0 100644 --- a/DrissionPage/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -1,4 +1,8 @@ # -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" from base64 import b64decode from json import JSONDecodeError, loads from queue import Queue @@ -8,7 +12,7 @@ from time import perf_counter, sleep from requests.structures import CaseInsensitiveDict -from .errors import CDPError +from DrissionPage.errors import CDPError class NetworkListener(object): diff --git a/DrissionPage/network_listener.pyi b/DrissionPage/_units/network_listener.pyi similarity index 95% rename from DrissionPage/network_listener.pyi rename to DrissionPage/_units/network_listener.pyi index 76c5e72..3472032 100644 --- a/DrissionPage/network_listener.pyi +++ b/DrissionPage/_units/network_listener.pyi @@ -1,10 +1,15 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" from queue import Queue from typing import Union, Dict, List, Iterable, Tuple from requests.structures import CaseInsensitiveDict -from chromium_base import ChromiumBase -from chromium_driver import ChromiumDriver +from DrissionPage._base.chromium_driver import ChromiumDriver +from DrissionPage._pages.chromium_base import ChromiumBase class NetworkListener(object): diff --git a/DrissionPage/setter.py b/DrissionPage/_units/setter.py similarity index 99% rename from DrissionPage/setter.py rename to DrissionPage/_units/setter.py index 3decfca..94f2262 100644 --- a/DrissionPage/setter.py +++ b/DrissionPage/_units/setter.py @@ -7,8 +7,8 @@ from pathlib import Path from requests.structures import CaseInsensitiveDict -from .commons.tools import show_or_hide_browser -from .commons.web import set_browser_cookies, set_session_cookies +from DrissionPage._commons.tools import show_or_hide_browser +from DrissionPage._commons.web import set_browser_cookies, set_session_cookies class ChromiumBaseSetter(object): diff --git a/DrissionPage/setter.pyi b/DrissionPage/_units/setter.pyi similarity index 92% rename from DrissionPage/setter.pyi rename to DrissionPage/_units/setter.pyi index 0ed4134..994ad87 100644 --- a/DrissionPage/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -11,13 +11,13 @@ from requests.adapters import HTTPAdapter from requests.auth import HTTPBasicAuth from requests.cookies import RequestsCookieJar -from .chromium_base import ChromiumBase, ChromiumPageScroll -from .chromium_element import ChromiumElement -from .chromium_frame import ChromiumFrame -from .chromium_page import ChromiumPage -from .chromium_tab import ChromiumTab -from .session_page import SessionPage -from .web_page import WebPage +from DrissionPage._elements.chromium_element import ChromiumElement +from DrissionPage._pages.chromium_base import ChromiumBase, ChromiumPageScroll +from DrissionPage._pages.chromium_frame import ChromiumFrame +from DrissionPage._pages.chromium_page import ChromiumPage +from DrissionPage._pages.chromium_tab import ChromiumTab +from DrissionPage._pages.session_page import SessionPage +from DrissionPage._pages.web_page import WebPage FILE_EXISTS = Literal['skip', 'rename', 'overwrite', 's', 'r', 'o'] diff --git a/DrissionPage/waiter.py b/DrissionPage/_units/waiter.py similarity index 99% rename from DrissionPage/waiter.py rename to DrissionPage/_units/waiter.py index 71dfc46..06a98f8 100644 --- a/DrissionPage/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -1,8 +1,8 @@ # -*- coding:utf-8 -*- from time import sleep, perf_counter -from .commons.constants import Settings -from .errors import WaitTimeoutError +from DrissionPage._commons.constants import Settings +from DrissionPage.errors import WaitTimeoutError class ChromiumBaseWaiter(object): diff --git a/DrissionPage/waiter.pyi b/DrissionPage/_units/waiter.pyi similarity index 93% rename from DrissionPage/waiter.pyi rename to DrissionPage/_units/waiter.pyi index 823ee3b..a05a921 100644 --- a/DrissionPage/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -5,11 +5,11 @@ """ from typing import Union +from DrissionPage._elements.chromium_element import ChromiumElement +from DrissionPage._pages.chromium_base import ChromiumBase +from DrissionPage._pages.chromium_frame import ChromiumFrame +from DrissionPage._pages.chromium_page import ChromiumPage from .browser_download_manager import DownloadMission -from .chromium_base import ChromiumBase -from .chromium_element import ChromiumElement -from .chromium_frame import ChromiumFrame -from .chromium_page import ChromiumPage class ChromiumBaseWaiter(object): diff --git a/DrissionPage/common.py b/DrissionPage/common.py index f1225cc..2c67c17 100644 --- a/DrissionPage/common.py +++ b/DrissionPage/common.py @@ -6,9 +6,9 @@ """ from FlowViewer import Listener, RequestMan -from .session_element import make_session_ele +from ._elements.session_element import make_session_ele -from .action_chains import ActionChains -from .commons.keys import Keys -from .commons.by import By -from .commons.constants import Settings +from ._units.action_chains import ActionChains +from ._commons.keys import Keys +from ._commons.by import By +from ._commons.constants import Settings diff --git a/DrissionPage/common.pyi b/DrissionPage/common.pyi index 54677db..f1d57c4 100644 --- a/DrissionPage/common.pyi +++ b/DrissionPage/common.pyi @@ -1,7 +1,10 @@ # -*- coding:utf-8 -*- -from .session_element import make_session_ele as make_session_ele - -from .action_chains import ActionChains as ActionChains -from .commons.keys import Keys as Keys -from .commons.by import By as By -from .commons.constants import Settings as Settings +""" +@Author : g1879 +@Contact : g1879@qq.com +""" +from ._commons.by import By as By +from ._commons.constants import Settings as Settings +from ._commons.keys import Keys as Keys +from ._elements.session_element import make_session_ele as make_session_ele +from ._units.action_chains import ActionChains as ActionChains diff --git a/DrissionPage/easy_set.py b/DrissionPage/easy_set.py index 7ed75a7..c70cb68 100644 --- a/DrissionPage/easy_set.py +++ b/DrissionPage/easy_set.py @@ -7,8 +7,8 @@ from os import popen from pathlib import Path from re import search -from .configs.chromium_options import ChromiumOptions -from .configs.options_manage import OptionsManager +from ._configs.chromium_options import ChromiumOptions +from ._configs.options_manage import OptionsManager def configs_to_here(save_name=None): diff --git a/DrissionPage/errors.py b/DrissionPage/errors.py index 7bab148..589fb95 100644 --- a/DrissionPage/errors.py +++ b/DrissionPage/errors.py @@ -1,4 +1,8 @@ # -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" class BaseError(Exception): From f2b522b25eb8ff506bd61b153300bfe7f11cb281 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 20 Oct 2023 15:07:08 +0800 Subject: [PATCH 041/182] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/base.py | 8 +++--- DrissionPage/_base/base.pyi | 2 +- DrissionPage/_base/browser.py | 2 +- DrissionPage/_base/browser.pyi | 4 +-- DrissionPage/_commons/browser.py | 4 +-- DrissionPage/_commons/browser.pyi | 2 +- DrissionPage/_commons/cli.py | 4 +-- DrissionPage/_commons/constants.py | 2 +- DrissionPage/_commons/tools.pyi | 2 +- DrissionPage/_commons/web.pyi | 6 ++--- DrissionPage/_configs/chromium_options.py | 2 +- DrissionPage/_configs/session_options.py | 2 +- DrissionPage/_elements/chromium_element.py | 22 ++++++++-------- DrissionPage/_elements/chromium_element.pyi | 18 ++++++------- DrissionPage/_elements/session_element.py | 8 +++--- DrissionPage/_elements/session_element.pyi | 12 ++++----- DrissionPage/_pages/chromium_base.py | 26 +++++++++---------- DrissionPage/_pages/chromium_base.pyi | 24 ++++++++--------- DrissionPage/_pages/chromium_frame.py | 10 +++---- DrissionPage/_pages/chromium_frame.pyi | 14 +++++----- DrissionPage/_pages/chromium_page.py | 16 ++++++------ DrissionPage/_pages/chromium_page.pyi | 14 +++++----- DrissionPage/_pages/chromium_tab.py | 12 ++++----- DrissionPage/_pages/chromium_tab.pyi | 20 +++++++------- DrissionPage/_pages/session_page.py | 10 +++---- DrissionPage/_pages/session_page.pyi | 10 +++---- DrissionPage/_pages/web_page.py | 12 ++++----- DrissionPage/_pages/web_page.pyi | 14 +++++----- DrissionPage/_units/action_chains.py | 4 +-- DrissionPage/_units/action_chains.pyi | 6 ++--- .../_units/browser_download_manager.py | 6 ++++- .../_units/browser_download_manager.pyi | 4 +-- DrissionPage/_units/network_listener.py | 2 +- DrissionPage/_units/network_listener.pyi | 4 +-- DrissionPage/_units/setter.py | 4 +-- DrissionPage/_units/setter.pyi | 14 +++++----- DrissionPage/_units/waiter.py | 4 +-- DrissionPage/_units/waiter.pyi | 8 +++--- 38 files changed, 171 insertions(+), 167 deletions(-) diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index 51d7182..fe6f9a8 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -10,10 +10,10 @@ from urllib.parse import quote from DownloadKit import DownloadKit -from DrissionPage._commons.constants import Settings, NoneElement -from DrissionPage._commons.locator import get_loc -from DrissionPage._commons.web import format_html -from DrissionPage.errors import ElementNotFoundError +from .._commons.constants import Settings, NoneElement +from .._commons.locator import get_loc +from .._commons.web import format_html +from ..errors import ElementNotFoundError class BaseParser(object): diff --git a/DrissionPage/_base/base.pyi b/DrissionPage/_base/base.pyi index 3ec97a4..2b9228c 100644 --- a/DrissionPage/_base/base.pyi +++ b/DrissionPage/_base/base.pyi @@ -8,7 +8,7 @@ from typing import Union, Tuple, List from DownloadKit import DownloadKit -from DrissionPage._commons.constants import NoneElement +from .._commons.constants import NoneElement class BaseParser(object): diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 9019b7f..126afab 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -5,8 +5,8 @@ """ from time import sleep -from DrissionPage._units.browser_download_manager import BrowserDownloadManager from .chromium_driver import BrowserDriver +from .._units.browser_download_manager import BrowserDownloadManager class Browser(object): diff --git a/DrissionPage/_base/browser.pyi b/DrissionPage/_base/browser.pyi index cdee003..3a0b818 100644 --- a/DrissionPage/_base/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -5,9 +5,9 @@ """ from typing import List, Optional, Union -from DrissionPage._units.browser_download_manager import BrowserDownloadManager -from DrissionPage._pages.chromium_page import ChromiumPage from .chromium_driver import BrowserDriver +from .._pages.chromium_page import ChromiumPage +from .._units.browser_download_manager import BrowserDownloadManager class Browser(object): diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_commons/browser.py index 326bc7c..80106c4 100644 --- a/DrissionPage/_commons/browser.py +++ b/DrissionPage/_commons/browser.py @@ -12,7 +12,7 @@ from platform import system from requests import get as requests_get -from DrissionPage.errors import BrowserConnectError +from ..errors import BrowserConnectError from .tools import port_is_using @@ -42,7 +42,7 @@ def connect_browser(option): # 传入的路径找不到，主动在ini文件、注册表、系统变量中找 except FileNotFoundError: - from DrissionPage.easy_set import get_chrome_path + from ..easy_set import get_chrome_path chrome_path = get_chrome_path(show_msg=False) if not chrome_path: diff --git a/DrissionPage/_commons/browser.pyi b/DrissionPage/_commons/browser.pyi index 0294aff..a6fee1c 100644 --- a/DrissionPage/_commons/browser.pyi +++ b/DrissionPage/_commons/browser.pyi @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from DrissionPage._configs.chromium_options import ChromiumOptions +from .._configs.chromium_options import ChromiumOptions def connect_browser(option: ChromiumOptions) -> tuple: ... diff --git a/DrissionPage/_commons/cli.py b/DrissionPage/_commons/cli.py index 71f2c28..82e991a 100644 --- a/DrissionPage/_commons/cli.py +++ b/DrissionPage/_commons/cli.py @@ -5,8 +5,8 @@ """ from click import command, option -from DrissionPage._pages.chromium_page import ChromiumPage -from DrissionPage.easy_set import set_paths, configs_to_here as ch +from .._pages.chromium_page import ChromiumPage +from ..easy_set import set_paths, configs_to_here as ch @command() diff --git a/DrissionPage/_commons/constants.py b/DrissionPage/_commons/constants.py index 2a219c7..9192448 100644 --- a/DrissionPage/_commons/constants.py +++ b/DrissionPage/_commons/constants.py @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from DrissionPage.errors import ElementNotFoundError +from ..errors import ElementNotFoundError HANDLE_ALERT_METHOD = 'Page.handleJavaScriptDialog' FRAME_ELEMENT = ('iframe', 'frame') diff --git a/DrissionPage/_commons/tools.pyi b/DrissionPage/_commons/tools.pyi index 83dcff1..b430ba9 100644 --- a/DrissionPage/_commons/tools.pyi +++ b/DrissionPage/_commons/tools.pyi @@ -8,7 +8,7 @@ from pathlib import Path from typing import Union from types import FunctionType -from DrissionPage._pages.chromium_page import ChromiumPage +from .._pages.chromium_page import ChromiumPage def get_usable_path(path: Union[str, Path], is_file: bool = True, parents: bool = True) -> Path: ... diff --git a/DrissionPage/_commons/web.pyi b/DrissionPage/_commons/web.pyi index 4a365e9..eae78ce 100644 --- a/DrissionPage/_commons/web.pyi +++ b/DrissionPage/_commons/web.pyi @@ -9,9 +9,9 @@ from typing import Union from requests import Session from requests.cookies import RequestsCookieJar -from DrissionPage.base import DrissionElement, BasePage -from DrissionPage._chromium_element import ChromiumElement -from DrissionPage._chromium_base import ChromiumBase +from .._elements.chromium_element import ChromiumElement +from .._pages.chromium_base import ChromiumBase +from ..base import DrissionElement, BasePage def get_ele_txt(e: DrissionElement) -> str: ... diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index c44529a..38809a4 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -6,8 +6,8 @@ from pathlib import Path from tempfile import gettempdir, TemporaryDirectory -from DrissionPage._commons.tools import port_is_using, clean_folder from .options_manage import OptionsManager +from .._commons.tools import port_is_using, clean_folder class ChromiumOptions(object): diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index 1aa7c1d..4339b08 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -8,8 +8,8 @@ from pathlib import Path from requests import Session from requests.structures import CaseInsensitiveDict -from DrissionPage._commons.web import cookies_to_tuple, set_session_cookies from .options_manage import OptionsManager +from .._commons.web import cookies_to_tuple, set_session_cookies class SessionOptions(object): diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index f5fc1fd..e8cb50e 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -7,17 +7,17 @@ from os.path import basename, sep from pathlib import Path from time import perf_counter, sleep -from DrissionPage._base.base import DrissionElement, BaseElement -from DrissionPage._commons.constants import FRAME_ELEMENT, NoneElement, Settings -from DrissionPage._commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions -from DrissionPage._commons.locator import get_loc -from DrissionPage._commons.tools import get_usable_path -from DrissionPage._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, location_in_viewport, offset_scroll -from DrissionPage.errors import ContextLossError, ElementLossError, JavaScriptError, NoRectError, ElementNotFoundError, \ +from .._base.base import DrissionElement, BaseElement +from .._commons.constants import FRAME_ELEMENT, NoneElement, Settings +from .._commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions +from .._commons.locator import get_loc +from .._commons.tools import get_usable_path +from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, location_in_viewport, offset_scroll +from ..errors import ContextLossError, ElementLossError, JavaScriptError, NoRectError, ElementNotFoundError, \ CDPError, NoResourceError, CanNotClickError from .session_element import make_session_ele -from DrissionPage._units.setter import ChromiumElementSetter -from DrissionPage._units.waiter import ChromiumElementWaiter +from .._units.setter import ChromiumElementSetter +from .._units.waiter import ChromiumElementWaiter class ChromiumElement(DrissionElement): @@ -641,7 +641,7 @@ class ChromiumElement(DrissionElement): points = [(int(current_x + i * (width / num)), int(current_y + i * (height / num))) for i in range(1, num)] points.append((target_x, target_y)) - from .action_chains import ActionChains + from .._units.action_chains import ActionChains actions = ActionChains(self.page) actions.hold(self) @@ -1203,7 +1203,7 @@ def make_chromium_ele(page, node_id=None, obj_id=None): ele = ChromiumElement(page, obj_id=obj_id, node_id=node_id, backend_id=backend_id) if ele.tag in FRAME_ELEMENT: - from ._chromium_frame import ChromiumFrame + from .._pages.chromium_frame import ChromiumFrame ele = ChromiumFrame(page, ele) return ele diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index a7b0dcb..7ee4253 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -6,15 +6,15 @@ from pathlib import Path from typing import Union, Tuple, List, Any -from DrissionPage._base.base import DrissionElement, BaseElement -from DrissionPage._commons.constants import NoneElement -from DrissionPage._elements.session_element import SessionElement -from DrissionPage._pages.chromium_base import ChromiumBase -from DrissionPage._pages.chromium_frame import ChromiumFrame -from DrissionPage._pages.chromium_page import ChromiumPage -from DrissionPage._pages.web_page import WebPage -from DrissionPage._units.setter import ChromiumElementSetter -from DrissionPage._units.waiter import ChromiumElementWaiter +from .._base.base import DrissionElement, BaseElement +from .._commons.constants import NoneElement +from .._elements.session_element import SessionElement +from .._pages.chromium_base import ChromiumBase +from .._pages.chromium_frame import ChromiumFrame +from .._pages.chromium_page import ChromiumPage +from .._pages.web_page import WebPage +from .._units.setter import ChromiumElementSetter +from .._units.waiter import ChromiumElementWaiter class ChromiumElement(DrissionElement): diff --git a/DrissionPage/_elements/session_element.py b/DrissionPage/_elements/session_element.py index fa241cb..5fd09f4 100644 --- a/DrissionPage/_elements/session_element.py +++ b/DrissionPage/_elements/session_element.py @@ -9,10 +9,10 @@ from re import match, DOTALL from lxml.etree import tostring from lxml.html import HtmlElement, fromstring -from DrissionPage._base.base import DrissionElement, BasePage, BaseElement -from DrissionPage._commons.constants import NoneElement -from DrissionPage._commons.locator import get_loc -from DrissionPage._commons.web import get_ele_txt, make_absolute_link +from .._base.base import DrissionElement, BasePage, BaseElement +from .._commons.constants import NoneElement +from .._commons.locator import get_loc +from .._commons.web import get_ele_txt, make_absolute_link class SessionElement(DrissionElement): diff --git a/DrissionPage/_elements/session_element.pyi b/DrissionPage/_elements/session_element.pyi index 9257354..6d3efc9 100644 --- a/DrissionPage/_elements/session_element.pyi +++ b/DrissionPage/_elements/session_element.pyi @@ -7,12 +7,12 @@ from typing import Union, List, Tuple from lxml.html import HtmlElement -from DrissionPage._base.base import DrissionElement, BaseElement -from DrissionPage._commons.constants import NoneElement -from DrissionPage._elements.chromium_element import ChromiumElement -from DrissionPage._pages.chromium_base import ChromiumBase -from DrissionPage._pages.chromium_frame import ChromiumFrame -from DrissionPage._pages.session_page import SessionPage +from .._base.base import DrissionElement, BaseElement +from .._commons.constants import NoneElement +from .._elements.chromium_element import ChromiumElement +from .._pages.chromium_base import ChromiumBase +from .._pages.chromium_frame import ChromiumFrame +from .._pages.session_page import SessionPage class SessionElement(DrissionElement): diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index c79b8be..e0259c4 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -13,20 +13,20 @@ from time import perf_counter, sleep, time from requests import get -from DrissionPage._base.base import BasePage -from DrissionPage._base.chromium_driver import ChromiumDriver -from DrissionPage._commons.constants import HANDLE_ALERT_METHOD, ERROR, NoneElement -from DrissionPage._commons.locator import get_loc -from DrissionPage._commons.tools import get_usable_path, clean_folder -from DrissionPage._commons.web import location_in_viewport -from DrissionPage._elements.chromium_element import ChromiumScroll, ChromiumElement, run_js, make_chromium_ele -from DrissionPage._elements.session_element import make_session_ele -from DrissionPage._units.network_listener import NetworkListener -from DrissionPage._units.setter import ChromiumBaseSetter -from DrissionPage._units.waiter import ChromiumBaseWaiter -from DrissionPage.errors import ContextLossError, ElementLossError, AlertExistsError, CDPError, TabClosedError, \ +from .._base.base import BasePage +from .._base.chromium_driver import ChromiumDriver +from .._commons.constants import HANDLE_ALERT_METHOD, ERROR, NoneElement +from .._commons.locator import get_loc +from .._commons.tools import get_usable_path, clean_folder +from .._commons.web import location_in_viewport +from .._elements.chromium_element import ChromiumScroll, ChromiumElement, run_js, make_chromium_ele +from .._elements.session_element import make_session_ele +from .._units.action_chains import ActionChains +from .._units.network_listener import NetworkListener +from .._units.setter import ChromiumBaseSetter +from .._units.waiter import ChromiumBaseWaiter +from ..errors import ContextLossError, ElementLossError, AlertExistsError, CDPError, TabClosedError, \ NoRectError, BrowserConnectError, GetDocumentError -from _units.action_chains import ActionChains class ChromiumBase(BasePage): diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 97020ad..33f4030 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -8,18 +8,18 @@ from typing import Union, Tuple, List, Any from DataRecorder import Recorder -from DrissionPage._base.base import BasePage -from DrissionPage._base.browser import Browser -from DrissionPage._base.chromium_driver import ChromiumDriver -from DrissionPage._commons.constants import NoneElement -from DrissionPage._elements.chromium_element import ChromiumElement, ChromiumScroll -from DrissionPage._elements.session_element import SessionElement -from DrissionPage._pages.chromium_frame import ChromiumFrame -from DrissionPage._pages.chromium_page import ChromiumPage -from DrissionPage._units.action_chains import ActionChains -from DrissionPage._units.network_listener import NetworkListener -from DrissionPage._units.setter import ChromiumBaseSetter -from DrissionPage._units.waiter import ChromiumBaseWaiter +from .._base.base import BasePage +from .._base.browser import Browser +from .._base.chromium_driver import ChromiumDriver +from .._commons.constants import NoneElement +from .._elements.chromium_element import ChromiumElement, ChromiumScroll +from .._elements.session_element import SessionElement +from .._pages.chromium_frame import ChromiumFrame +from .._pages.chromium_page import ChromiumPage +from .._units.action_chains import ActionChains +from .._units.network_listener import NetworkListener +from .._units.setter import ChromiumBaseSetter +from .._units.waiter import ChromiumBaseWaiter class ChromiumBase(BasePage): diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index ca037f0..f83f322 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -10,11 +10,11 @@ from time import sleep, perf_counter from requests import get -from DrissionPage._elements.chromium_element import ChromiumElement -from DrissionPage._pages.chromium_base import ChromiumBase, ChromiumPageScroll -from DrissionPage._units.setter import ChromiumFrameSetter -from DrissionPage._units.waiter import FrameWaiter -from DrissionPage.errors import ContextLossError +from .._elements.chromium_element import ChromiumElement +from .._pages.chromium_base import ChromiumBase, ChromiumPageScroll +from .._units.setter import ChromiumFrameSetter +from .._units.waiter import FrameWaiter +from ..errors import ContextLossError class ChromiumFrame(ChromiumBase): diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 1be67ad..8b4e0bf 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -6,13 +6,13 @@ from pathlib import Path from typing import Union, Tuple, List, Any -from DrissionPage._elements.chromium_element import ChromiumElement, Locations, ChromiumElementStates -from DrissionPage._pages.chromium_base import ChromiumBase, ChromiumPageScroll -from DrissionPage._pages.chromium_page import ChromiumPage -from DrissionPage._pages.chromium_tab import ChromiumTab -from DrissionPage._pages.web_page import WebPage -from DrissionPage._units.setter import ChromiumFrameSetter -from DrissionPage._units.waiter import FrameWaiter +from .chromium_base import ChromiumBase, ChromiumPageScroll +from .chromium_page import ChromiumPage +from .chromium_tab import ChromiumTab +from .web_page import WebPage +from .._elements.chromium_element import ChromiumElement, Locations, ChromiumElementStates +from .._units.setter import ChromiumFrameSetter +from .._units.waiter import FrameWaiter class ChromiumFrame(ChromiumBase): diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 33f343b..64ed600 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -8,14 +8,14 @@ from time import perf_counter, sleep from requests import get -from DrissionPage._base.browser import Browser -from DrissionPage._base.chromium_driver import ChromiumDriver -from DrissionPage._commons.browser import connect_browser -from DrissionPage._configs.chromium_options import ChromiumOptions -from DrissionPage._pages.chromium_base import ChromiumBase, Timeout -from DrissionPage._pages.chromium_tab import ChromiumTab -from DrissionPage._units.setter import ChromiumPageSetter -from DrissionPage._units.waiter import ChromiumPageWaiter +from .._base.browser import Browser +from .._base.chromium_driver import ChromiumDriver +from .._commons.browser import connect_browser +from .._configs.chromium_options import ChromiumOptions +from .._pages.chromium_base import ChromiumBase, Timeout +from .._pages.chromium_tab import ChromiumTab +from .._units.setter import ChromiumPageSetter +from .._units.waiter import ChromiumPageWaiter class ChromiumPage(ChromiumBase): diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index 884a452..9f55304 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -5,13 +5,13 @@ """ from typing import Union, Tuple, List, Optional -from DrissionPage._base.browser import Browser -from DrissionPage._pages.chromium_base import ChromiumBase -from DrissionPage._base.chromium_driver import ChromiumDriver -from DrissionPage._pages.chromium_tab import ChromiumTab -from DrissionPage._configs.chromium_options import ChromiumOptions -from DrissionPage._units.setter import ChromiumPageSetter -from DrissionPage._units.waiter import ChromiumPageWaiter +from .._base.browser import Browser +from .._pages.chromium_base import ChromiumBase +from .._base.chromium_driver import ChromiumDriver +from .._pages.chromium_tab import ChromiumTab +from .._configs.chromium_options import ChromiumOptions +from .._units.setter import ChromiumPageSetter +from .._units.waiter import ChromiumPageWaiter class ChromiumPage(ChromiumBase): diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 6e44f6d..f5d6cff 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -5,12 +5,12 @@ """ from copy import copy -from DrissionPage._base.base import BasePage -from DrissionPage._commons.web import set_session_cookies, set_browser_cookies -from DrissionPage._pages.chromium_base import ChromiumBase -from DrissionPage._pages.session_page import SessionPage -from DrissionPage._units.setter import TabSetter, WebPageTabSetter -from DrissionPage._units.waiter import ChromiumTabWaiter +from .._base.base import BasePage +from .._commons.web import set_session_cookies, set_browser_cookies +from .._pages.chromium_base import ChromiumBase +from .._pages.session_page import SessionPage +from .._units.setter import TabSetter, WebPageTabSetter +from .._units.waiter import ChromiumTabWaiter class ChromiumTab(ChromiumBase): diff --git a/DrissionPage/_pages/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi index 6f19d4b..9bb19a5 100644 --- a/DrissionPage/_pages/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -7,16 +7,16 @@ from typing import Union, Tuple, Any, List from requests import Session, Response -from DrissionPage._base.browser import Browser -from DrissionPage._elements.chromium_element import ChromiumElement -from DrissionPage._elements.session_element import SessionElement -from DrissionPage._pages.chromium_base import ChromiumBase -from DrissionPage._pages.chromium_frame import ChromiumFrame -from DrissionPage._pages.chromium_page import ChromiumPage, ChromiumTabRect -from DrissionPage._pages.session_page import SessionPage -from DrissionPage._pages.web_page import WebPage -from DrissionPage._units.setter import TabSetter, WebPageTabSetter -from DrissionPage._units.waiter import ChromiumTabWaiter +from .chromium_base import ChromiumBase +from .chromium_frame import ChromiumFrame +from .chromium_page import ChromiumPage, ChromiumTabRect +from .session_page import SessionPage +from .web_page import WebPage +from .._base.browser import Browser +from .._elements.chromium_element import ChromiumElement +from .._elements.session_element import SessionElement +from .._units.setter import TabSetter, WebPageTabSetter +from .._units.waiter import ChromiumTabWaiter class ChromiumTab(ChromiumBase): diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index 846ba9b..5a92319 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -11,11 +11,11 @@ from requests import Session from requests.structures import CaseInsensitiveDict from tldextract import extract -from DrissionPage._base.base import BasePage -from DrissionPage._commons.web import cookie_to_dict -from DrissionPage._configs.session_options import SessionOptions -from DrissionPage._elements.session_element import SessionElement, make_session_ele -from DrissionPage._units.setter import SessionPageSetter +from .._base.base import BasePage +from .._commons.web import cookie_to_dict +from .._configs.session_options import SessionOptions +from .._elements.session_element import SessionElement, make_session_ele +from .._units.setter import SessionPageSetter class SessionPage(BasePage): diff --git a/DrissionPage/_pages/session_page.pyi b/DrissionPage/_pages/session_page.pyi index 145f122..22f6963 100644 --- a/DrissionPage/_pages/session_page.pyi +++ b/DrissionPage/_pages/session_page.pyi @@ -8,11 +8,11 @@ from typing import Any, Union, Tuple, List from requests import Session, Response from requests.structures import CaseInsensitiveDict -from DrissionPage._base.base import BasePage -from DrissionPage._commons.constants import NoneElement -from DrissionPage._configs.session_options import SessionOptions -from DrissionPage._elements.session_element import SessionElement -from DrissionPage._units.setter import SessionPageSetter +from .._base.base import BasePage +from .._commons.constants import NoneElement +from .._configs.session_options import SessionOptions +from .._elements.session_element import SessionElement +from .._units.setter import SessionPageSetter class SessionPage(BasePage): diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 058ad99..e6b055f 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -3,13 +3,13 @@ @Author : g1879 @Contact : g1879@qq.com """ -from DrissionPage._base.base import BasePage -from DrissionPage._commons.web import set_session_cookies, set_browser_cookies -from DrissionPage._configs.chromium_options import ChromiumOptions -from DrissionPage._pages.chromium_page import ChromiumPage -from DrissionPage._pages.chromium_tab import WebPageTab -from DrissionPage._units.setter import WebPageSetter +from .chromium_page import ChromiumPage +from .chromium_tab import WebPageTab from .session_page import SessionPage +from .._base.base import BasePage +from .._commons.web import set_session_cookies, set_browser_cookies +from .._configs.chromium_options import ChromiumOptions +from .._units.setter import WebPageSetter class WebPage(SessionPage, ChromiumPage, BasePage): diff --git a/DrissionPage/_pages/web_page.pyi b/DrissionPage/_pages/web_page.pyi index de706f9..ce350da 100644 --- a/DrissionPage/_pages/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -7,17 +7,17 @@ from typing import Union, Tuple, List, Any from requests import Session, Response -from ._base import BasePage -from ._chromium_driver import ChromiumDriver -from ._chromium_element import ChromiumElement from .chromium_frame import ChromiumFrame from .chromium_page import ChromiumPage from .chromium_tab import WebPageTab -from ._configs.chromium_options import ChromiumOptions -from ._configs.session_options import SessionOptions -from ._session_element import SessionElement from .session_page import SessionPage -from ._units.setter import WebPageSetter +from .._base.base import BasePage +from .._base.chromium_driver import ChromiumDriver +from .._configs.chromium_options import ChromiumOptions +from .._configs.session_options import SessionOptions +from .._elements.chromium_element import ChromiumElement +from .._elements.session_element import SessionElement +from .._units.setter import WebPageSetter class WebPage(SessionPage, ChromiumPage, BasePage): diff --git a/DrissionPage/_units/action_chains.py b/DrissionPage/_units/action_chains.py index fd733ce..0469bef 100644 --- a/DrissionPage/_units/action_chains.py +++ b/DrissionPage/_units/action_chains.py @@ -5,8 +5,8 @@ """ from time import sleep -from DrissionPage._commons.keys import modifierBit, keyDescriptionForString -from DrissionPage._commons.web import location_in_viewport +from .._commons.keys import modifierBit, keyDescriptionForString +from .._commons.web import location_in_viewport class ActionChains: diff --git a/DrissionPage/_units/action_chains.pyi b/DrissionPage/_units/action_chains.pyi index 83163dd..471c797 100644 --- a/DrissionPage/_units/action_chains.pyi +++ b/DrissionPage/_units/action_chains.pyi @@ -5,9 +5,9 @@ """ from typing import Union, Tuple -from DrissionPage._base.chromium_driver import ChromiumDriver -from DrissionPage._elements.chromium_element import ChromiumElement -from DrissionPage._pages.chromium_base import ChromiumBase +from .._base.chromium_driver import ChromiumDriver +from .._elements.chromium_element import ChromiumElement +from .._pages.chromium_base import ChromiumBase class ActionChains: diff --git a/DrissionPage/_units/browser_download_manager.py b/DrissionPage/_units/browser_download_manager.py index 546076d..0a0518c 100644 --- a/DrissionPage/_units/browser_download_manager.py +++ b/DrissionPage/_units/browser_download_manager.py @@ -1,10 +1,14 @@ # -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" from os.path import sep from pathlib import Path from shutil import move from time import sleep, perf_counter -from DrissionPage._commons.tools import get_usable_path +from .._commons.tools import get_usable_path class BrowserDownloadManager(object): diff --git a/DrissionPage/_units/browser_download_manager.pyi b/DrissionPage/_units/browser_download_manager.pyi index 0d4b266..c4d640e 100644 --- a/DrissionPage/_units/browser_download_manager.pyi +++ b/DrissionPage/_units/browser_download_manager.pyi @@ -1,8 +1,8 @@ from pathlib import Path from typing import Dict, Optional, Union -from DrissionPage._base.browser import Browser -from DrissionPage._pages.chromium_page import ChromiumPage +from .._base.browser import Browser +from .._pages.chromium_page import ChromiumPage class BrowserDownloadManager(object): diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index 220b0d0..c2f49b1 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -12,7 +12,7 @@ from time import perf_counter, sleep from requests.structures import CaseInsensitiveDict -from DrissionPage.errors import CDPError +from ..errors import CDPError class NetworkListener(object): diff --git a/DrissionPage/_units/network_listener.pyi b/DrissionPage/_units/network_listener.pyi index 3472032..e2b7e7a 100644 --- a/DrissionPage/_units/network_listener.pyi +++ b/DrissionPage/_units/network_listener.pyi @@ -8,8 +8,8 @@ from typing import Union, Dict, List, Iterable, Tuple from requests.structures import CaseInsensitiveDict -from DrissionPage._base.chromium_driver import ChromiumDriver -from DrissionPage._pages.chromium_base import ChromiumBase +from .._base.chromium_driver import ChromiumDriver +from .._pages.chromium_base import ChromiumBase class NetworkListener(object): diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index 94f2262..8102489 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -7,8 +7,8 @@ from pathlib import Path from requests.structures import CaseInsensitiveDict -from DrissionPage._commons.tools import show_or_hide_browser -from DrissionPage._commons.web import set_browser_cookies, set_session_cookies +from .._commons.tools import show_or_hide_browser +from .._commons.web import set_browser_cookies, set_session_cookies class ChromiumBaseSetter(object): diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index 994ad87..989b368 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -11,13 +11,13 @@ from requests.adapters import HTTPAdapter from requests.auth import HTTPBasicAuth from requests.cookies import RequestsCookieJar -from DrissionPage._elements.chromium_element import ChromiumElement -from DrissionPage._pages.chromium_base import ChromiumBase, ChromiumPageScroll -from DrissionPage._pages.chromium_frame import ChromiumFrame -from DrissionPage._pages.chromium_page import ChromiumPage -from DrissionPage._pages.chromium_tab import ChromiumTab -from DrissionPage._pages.session_page import SessionPage -from DrissionPage._pages.web_page import WebPage +from .._elements.chromium_element import ChromiumElement +from .._pages.chromium_base import ChromiumBase, ChromiumPageScroll +from .._pages.chromium_frame import ChromiumFrame +from .._pages.chromium_page import ChromiumPage +from .._pages.chromium_tab import ChromiumTab +from .._pages.session_page import SessionPage +from .._pages.web_page import WebPage FILE_EXISTS = Literal['skip', 'rename', 'overwrite', 's', 'r', 'o'] diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 06a98f8..971e21a 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -1,8 +1,8 @@ # -*- coding:utf-8 -*- from time import sleep, perf_counter -from DrissionPage._commons.constants import Settings -from DrissionPage.errors import WaitTimeoutError +from .._commons.constants import Settings +from ..errors import WaitTimeoutError class ChromiumBaseWaiter(object): diff --git a/DrissionPage/_units/waiter.pyi b/DrissionPage/_units/waiter.pyi index a05a921..d9d4441 100644 --- a/DrissionPage/_units/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -5,11 +5,11 @@ """ from typing import Union -from DrissionPage._elements.chromium_element import ChromiumElement -from DrissionPage._pages.chromium_base import ChromiumBase -from DrissionPage._pages.chromium_frame import ChromiumFrame -from DrissionPage._pages.chromium_page import ChromiumPage from .browser_download_manager import DownloadMission +from .._elements.chromium_element import ChromiumElement +from .._pages.chromium_base import ChromiumBase +from .._pages.chromium_frame import ChromiumFrame +from .._pages.chromium_page import ChromiumPage class ChromiumBaseWaiter(object): From 2d5f1687c835c1ff01d61c68f2a84890a222e3f5 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 20 Oct 2023 17:15:36 +0800 Subject: [PATCH 042/182] =?UTF-8?q?new=5Ftab()=E6=94=B9=E5=9B=9E=E8=BF=94?= =?UTF-8?q?=E5=9B=9Etab=5Fid=EF=BC=9B=E5=A2=9E=E5=8A=A0tab.set.activate()?= =?UTF-8?q?=EF=BC=9BTab=E5=AF=B9=E8=B1=A1=E8=83=BD=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E8=87=AA=E5=B7=B1=E7=9A=84=E7=AA=97=E5=8F=A3=E7=9F=A9=E5=BD=A2?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=EF=BC=9B=E4=BF=AE=E5=A4=8D=E6=B5=8F=E8=A7=88?= =?UTF-8?q?=E5=99=A8=E6=9C=80=E5=B0=8F=E5=8C=96=E6=97=B6=E6=A8=A1=E6=8B=9F?= =?UTF-8?q?=E6=93=8D=E4=BD=9C=E4=B8=8D=E5=93=8D=E5=BA=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 9 ++- DrissionPage/_base/browser.pyi | 2 +- DrissionPage/_pages/chromium_base.py | 1 + DrissionPage/_pages/chromium_page.py | 82 +-------------------------- DrissionPage/_pages/chromium_page.pyi | 44 ++------------ DrissionPage/_pages/chromium_tab.py | 6 +- DrissionPage/_pages/chromium_tab.pyi | 6 +- DrissionPage/_pages/web_page.py | 8 --- DrissionPage/_pages/web_page.pyi | 2 - DrissionPage/_units/setter.py | 4 ++ DrissionPage/_units/setter.pyi | 2 + DrissionPage/_units/tab_rect.py | 76 +++++++++++++++++++++++++ DrissionPage/_units/tab_rect.pyi | 42 ++++++++++++++ 13 files changed, 148 insertions(+), 136 deletions(-) create mode 100644 DrissionPage/_units/tab_rect.py create mode 100644 DrissionPage/_units/tab_rect.pyi diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 126afab..e0ef55a 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -127,9 +127,12 @@ class Browser(object): """ self._driver.get(f'http://{self.address}/json/activate/{tab_id}') - def get_window_bounds(self): - """返回浏览器窗口位置和大小信息""" - return self.run_cdp('Browser.getWindowForTarget', targetId=self.id)['bounds'] + def get_window_bounds(self, tab_id=None): + """返回浏览器窗口位置和大小信息 + :param tab_id: 标签页id + :return: 窗口大小字典 + """ + return self.run_cdp('Browser.getWindowForTarget', targetId=tab_id or self.id)['bounds'] def quit(self): """关闭浏览器""" diff --git a/DrissionPage/_base/browser.pyi b/DrissionPage/_base/browser.pyi index 3a0b818..81bd82a 100644 --- a/DrissionPage/_base/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -46,7 +46,7 @@ class Browser(object): def activate_tab(self, tab_id: str) -> None: ... - def get_window_bounds(self) -> dict: ... + def get_window_bounds(self, tab_id: str = None) -> dict: ... def connect_to_page(self) -> None: ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index e0259c4..c2c2267 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -105,6 +105,7 @@ class ChromiumBase(BasePage): self._driver.call_method('DOM.enable') self._driver.call_method('Page.enable') + self._driver.call_method('Emulation.setFocusEmulationEnabled', enabled=True) self._driver.set_listener('Page.frameStoppedLoading', self._onFrameStoppedLoading) self._driver.set_listener('Page.frameStartedLoading', self._onFrameStartedLoading) diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 64ed600..cfe8855 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -15,6 +15,7 @@ from .._configs.chromium_options import ChromiumOptions from .._pages.chromium_base import ChromiumBase, Timeout from .._pages.chromium_tab import ChromiumTab from .._units.setter import ChromiumPageSetter +from .._units.tab_rect import ChromiumTabRect from .._units.waiter import ChromiumPageWaiter @@ -155,7 +156,7 @@ class ChromiumPage(ChromiumBase): """ return self._browser.find_tabs(title, url, tab_type, single) - def _new_tab(self, url=None, switch_to=False): + def new_tab(self, url=None, switch_to=False): """新建一个标签页,该标签页在最后面 :param url: 新标签页跳转到的网址 :param switch_to: 新建标签页后是否把焦点移过去 @@ -184,14 +185,6 @@ class ChromiumPage(ChromiumBase): return tid - def new_tab(self, url=None, switch_to=False): - """新建一个标签页,该标签页在最后面 - :param url: 新标签页跳转到的网址 - :param switch_to: 新建标签页后是否把焦点移过去 - :return: 新标签页对象 - """ - return ChromiumTab(self, self._new_tab(url, switch_to)) - def to_main_tab(self): """跳转到主标签页""" self.to_tab(self._main_tab) @@ -326,77 +319,6 @@ class ChromiumPage(ChromiumBase): self._driver.has_alert = True -class ChromiumTabRect(object): - def __init__(self, page): - self._page = page - - @property - def window_state(self): - """返回窗口状态：normal、fullscreen、maximized、 minimized""" - return self._get_browser_rect()['windowState'] - - @property - def browser_location(self): - """返回浏览器在屏幕上的坐标，左上角为(0, 0)""" - r = self._get_browser_rect() - if r['windowState'] in ('maximized', 'fullscreen'): - return 0, 0 - return r['left'] + 7, r['top'] - - @property - def page_location(self): - """返回页面左上角在屏幕中坐标，左上角为(0, 0)""" - w, h = self.viewport_location - r = self._get_page_rect()['layoutViewport'] - return w - r['pageX'], h - r['pageY'] - - @property - def viewport_location(self): - """返回视口在屏幕中坐标，左上角为(0, 0)""" - w_bl, h_bl = self.browser_location - w_bs, h_bs = self.browser_size - w_vs, h_vs = self.viewport_size_with_scrollbar - return w_bl + w_bs - w_vs, h_bl + h_bs - h_vs - - @property - def browser_size(self): - """返回浏览器大小""" - r = self._get_browser_rect() - if r['windowState'] == 'fullscreen': - return r['width'], r['height'] - elif r['windowState'] == 'maximized': - return r['width'] - 16, r['height'] - 16 - else: - return r['width'] - 16, r['height'] - 7 - - @property - def page_size(self): - """返回页面总宽高，格式：(宽, 高)""" - r = self._get_page_rect()['contentSize'] - return r['width'], r['height'] - - @property - def viewport_size(self): - """返回视口宽高，不包括滚动条，格式：(宽, 高)""" - r = self._get_page_rect()['visualViewport'] - return r['clientWidth'], r['clientHeight'] - - @property - def viewport_size_with_scrollbar(self): - """返回视口宽高，包括滚动条，格式：(宽, 高)""" - r = self._page.run_js('return window.innerWidth.toString() + " " + window.innerHeight.toString();') - w, h = r.split(' ') - return int(w), int(h) - - def _get_page_rect(self): - """获取页面范围信息""" - return self._page.run_cdp_loaded('Page.getLayoutMetrics') - - def _get_browser_rect(self): - """获取浏览器范围信息""" - return self._page.browser.get_window_bounds() - - class Alert(object): """用于保存alert信息的类""" diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index 9f55304..9a02407 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -6,11 +6,12 @@ from typing import Union, Tuple, List, Optional from .._base.browser import Browser -from .._pages.chromium_base import ChromiumBase from .._base.chromium_driver import ChromiumDriver -from .._pages.chromium_tab import ChromiumTab from .._configs.chromium_options import ChromiumOptions +from .._pages.chromium_base import ChromiumBase +from .._pages.chromium_tab import ChromiumTab from .._units.setter import ChromiumPageSetter +from .._units.tab_rect import ChromiumTabRect from .._units.waiter import ChromiumPageWaiter @@ -24,7 +25,7 @@ class ChromiumPage(ChromiumBase): self._main_tab: str = ... self._alert: Alert = ... self._browser: Browser = ... - self._rect: ChromiumTabRect = ... + self._rect: Optional[ChromiumTabRect] = ... def _handle_options(self, addr_driver_opts: Union[str, ChromiumDriver, ChromiumOptions]) -> str: ... @@ -64,9 +65,7 @@ class ChromiumPage(ChromiumBase): def find_tabs(self, title: str = None, url: str = None, tab_type: Union[str, list, tuple] = None, single: bool = True) -> Union[str, List[str]]: ... - def _new_tab(self, url: str = None, switch_to: bool = False) -> str: ... - - def new_tab(self, url: str = None, switch_to: bool = False) -> ChromiumTab: ... + def new_tab(self, url: str = None, switch_to: bool = False) -> str: ... def to_main_tab(self) -> None: ... @@ -91,39 +90,6 @@ class ChromiumPage(ChromiumBase): def _on_alert_open(self, **kwargs): ... -class ChromiumTabRect(object): - def __init__(self, page: ChromiumPage): - self._page: ChromiumPage = ... - - @property - def window_state(self) -> str: ... - - @property - def browser_location(self) -> Tuple[int, int]: ... - - @property - def page_location(self) -> Tuple[int, int]: ... - - @property - def viewport_location(self) -> Tuple[int, int]: ... - - @property - def browser_size(self) -> Tuple[int, int]: ... - - @property - def page_size(self) -> Tuple[int, int]: ... - - @property - def viewport_size(self) -> Tuple[int, int]: ... - - @property - def viewport_size_with_scrollbar(self) -> Tuple[int, int]: ... - - def _get_page_rect(self) -> dict: ... - - def _get_browser_rect(self) -> dict: ... - - class Alert(object): def __init__(self): diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index f5d6cff..8944559 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -10,6 +10,7 @@ from .._commons.web import set_session_cookies, set_browser_cookies from .._pages.chromium_base import ChromiumBase from .._pages.session_page import SessionPage from .._units.setter import TabSetter, WebPageTabSetter +from .._units.tab_rect import ChromiumTabRect from .._units.waiter import ChromiumTabWaiter @@ -24,6 +25,7 @@ class ChromiumTab(ChromiumBase): self._page = page self._browser = page.browser super().__init__(page.address, tab_id, page.timeout) + self._rect = None def _d_set_runtime_settings(self): """重写设置浏览器运行参数方法""" @@ -45,7 +47,9 @@ class ChromiumTab(ChromiumBase): @property def rect(self): """返回获取窗口坐标和大小的对象""" - return self.page.rect + if self._rect is None: + self._rect = ChromiumTabRect(self) + return self._rect @property def set(self): diff --git a/DrissionPage/_pages/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi index 9bb19a5..923c8c7 100644 --- a/DrissionPage/_pages/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -3,13 +3,14 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union, Tuple, Any, List +from typing import Union, Tuple, Any, List, Optional from requests import Session, Response +from .._units.tab_rect import ChromiumTabRect from .chromium_base import ChromiumBase from .chromium_frame import ChromiumFrame -from .chromium_page import ChromiumPage, ChromiumTabRect +from .chromium_page import ChromiumPage from .session_page import SessionPage from .web_page import WebPage from .._base.browser import Browser @@ -24,6 +25,7 @@ class ChromiumTab(ChromiumBase): def __init__(self, page: ChromiumPage, tab_id: str = None): self._page: ChromiumPage = ... self._browser: Browser = ... + self._rect: Optional[ChromiumTabRect] = ... def _d_set_runtime_settings(self) -> None: ... diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index e6b055f..f91aa78 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -294,14 +294,6 @@ class WebPage(SessionPage, ChromiumPage, BasePage): """ return tab_id if isinstance(tab_id, WebPageTab) else WebPageTab(self, tab_id or self.tab_id) - def new_tab(self, url=None, switch_to=False): - """新建一个标签页,该标签页在最后面 - :param url: 新标签页跳转到的网址 - :param switch_to: 新建标签页后是否把焦点移过去 - :return: 新标签页对象 - """ - return WebPageTab(self, self._new_tab(url, switch_to)) - def close_driver(self): """关闭driver及浏览器""" if self._has_driver: diff --git a/DrissionPage/_pages/web_page.pyi b/DrissionPage/_pages/web_page.pyi index ce350da..e05cb9f 100644 --- a/DrissionPage/_pages/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -121,8 +121,6 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def get_tab(self, tab_id: Union[str, WebPageTab] = None) -> WebPageTab: ... - def new_tab(self, url: str = None, switch_to: bool = False) -> WebPageTab: ... - def close_driver(self) -> None: ... def close_session(self) -> None: ... diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index 8102489..f0de74a 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -152,6 +152,10 @@ class TabSetter(ChromiumBaseSetter): self._page.browser._dl_mgr.set_file_exists(self._page.tab_id, mode) + def activate(self): + """使标签页处于最前面""" + self._page.browser.activate_tab(self._page.tab_id) + class ChromiumPageSetter(TabSetter): def main_tab(self, tab_id=None): diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index 989b368..f44413e 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -62,6 +62,8 @@ class TabSetter(ChromiumBaseSetter): def when_download_file_exists(self, mode: FILE_EXISTS) -> None: ... + def activate(self) -> None: ... + class ChromiumPageSetter(TabSetter): _page: ChromiumPage = ... diff --git a/DrissionPage/_units/tab_rect.py b/DrissionPage/_units/tab_rect.py new file mode 100644 index 0000000..6655932 --- /dev/null +++ b/DrissionPage/_units/tab_rect.py @@ -0,0 +1,76 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" + + +class ChromiumTabRect(object): + def __init__(self, page): + self._page = page + + @property + def window_state(self): + """返回窗口状态：normal、fullscreen、maximized、 minimized""" + return self._get_browser_rect()['windowState'] + + @property + def browser_location(self): + """返回浏览器在屏幕上的坐标，左上角为(0, 0)""" + r = self._get_browser_rect() + if r['windowState'] in ('maximized', 'fullscreen'): + return 0, 0 + return r['left'] + 7, r['top'] + + @property + def page_location(self): + """返回页面左上角在屏幕中坐标，左上角为(0, 0)""" + w, h = self.viewport_location + r = self._get_page_rect()['layoutViewport'] + return w - r['pageX'], h - r['pageY'] + + @property + def viewport_location(self): + """返回视口在屏幕中坐标，左上角为(0, 0)""" + w_bl, h_bl = self.browser_location + w_bs, h_bs = self.browser_size + w_vs, h_vs = self.viewport_size_with_scrollbar + return w_bl + w_bs - w_vs, h_bl + h_bs - h_vs + + @property + def browser_size(self): + """返回浏览器大小""" + r = self._get_browser_rect() + if r['windowState'] == 'fullscreen': + return r['width'], r['height'] + elif r['windowState'] == 'maximized': + return r['width'] - 16, r['height'] - 16 + else: + return r['width'] - 16, r['height'] - 7 + + @property + def page_size(self): + """返回页面总宽高，格式：(宽, 高)""" + r = self._get_page_rect()['contentSize'] + return r['width'], r['height'] + + @property + def viewport_size(self): + """返回视口宽高，不包括滚动条，格式：(宽, 高)""" + r = self._get_page_rect()['visualViewport'] + return r['clientWidth'], r['clientHeight'] + + @property + def viewport_size_with_scrollbar(self): + """返回视口宽高，包括滚动条，格式：(宽, 高)""" + r = self._page.run_js('return window.innerWidth.toString() + " " + window.innerHeight.toString();') + w, h = r.split(' ') + return int(w), int(h) + + def _get_page_rect(self): + """获取页面范围信息""" + return self._page.run_cdp_loaded('Page.getLayoutMetrics') + + def _get_browser_rect(self): + """获取浏览器范围信息""" + return self._page.browser.get_window_bounds(self._page.tab_id) diff --git a/DrissionPage/_units/tab_rect.pyi b/DrissionPage/_units/tab_rect.pyi new file mode 100644 index 0000000..95ebadc --- /dev/null +++ b/DrissionPage/_units/tab_rect.pyi @@ -0,0 +1,42 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" +from typing import Tuple, Union + +from .._pages.chromium_page import ChromiumPage +from .._pages.chromium_tab import ChromiumTab + + +class ChromiumTabRect(object): + def __init__(self, page: Union[ChromiumPage, ChromiumTab]): + self._page: Union[ChromiumPage, ChromiumTab] = ... + + @property + def window_state(self) -> str: ... + + @property + def browser_location(self) -> Tuple[int, int]: ... + + @property + def page_location(self) -> Tuple[int, int]: ... + + @property + def viewport_location(self) -> Tuple[int, int]: ... + + @property + def browser_size(self) -> Tuple[int, int]: ... + + @property + def page_size(self) -> Tuple[int, int]: ... + + @property + def viewport_size(self) -> Tuple[int, int]: ... + + @property + def viewport_size_with_scrollbar(self) -> Tuple[int, int]: ... + + def _get_page_rect(self) -> dict: ... + + def _get_browser_rect(self) -> dict: ... From 8e8394a889752289fc268513803ff65d3432b009 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 20 Oct 2023 18:24:01 +0800 Subject: [PATCH 043/182] =?UTF-8?q?=E5=BE=AE=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 2 ++ DrissionPage/_elements/chromium_element.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index e0ef55a..beb6416 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -149,3 +149,5 @@ class Browser(object): if f' {self.process_id} ' not in p.read(): break sleep(.2) + + Browser.BROWSERS.pop(self.id) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index e8cb50e..bb318a9 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -25,7 +25,7 @@ class ChromiumElement(DrissionElement): def __init__(self, page, node_id=None, obj_id=None, backend_id=None): """node_id、obj_id和backend_id必须至少传入一个 - :param page: 元素所在ChromePage页面对象 + :param page: 元素所在页面对象 :param node_id: cdp中的node id :param obj_id: js中的object id :param backend_id: backend id From 116bfe7e2fb9a0fa2565b8a888dc521b611a0607 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sat, 21 Oct 2023 10:42:26 +0800 Subject: [PATCH 044/182] =?UTF-8?q?ChromiumOptions=E5=A2=9E=E5=8A=A0existi?= =?UTF-8?q?ng=5Fonly()=E5=92=8Cis=5Fexisting=5Fonly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 2 +- DrissionPage/_base/browser.pyi | 2 +- DrissionPage/_commons/browser.py | 46 ++++++++++--------- DrissionPage/_commons/browser.pyi | 7 ++- DrissionPage/_configs/chromium_options.py | 19 +++++++- DrissionPage/_configs/chromium_options.pyi | 9 ++-- DrissionPage/_configs/configs.ini | 5 +- ...ownload_manager.py => download_manager.py} | 0 ...nload_manager.pyi => download_manager.pyi} | 0 DrissionPage/_units/waiter.pyi | 2 +- 10 files changed, 60 insertions(+), 32 deletions(-) rename DrissionPage/_units/{browser_download_manager.py => download_manager.py} (100%) rename DrissionPage/_units/{browser_download_manager.pyi => download_manager.pyi} (100%) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index beb6416..4c7aee0 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -6,7 +6,7 @@ from time import sleep from .chromium_driver import BrowserDriver -from .._units.browser_download_manager import BrowserDownloadManager +from .._units.download_manager import BrowserDownloadManager class Browser(object): diff --git a/DrissionPage/_base/browser.pyi b/DrissionPage/_base/browser.pyi index 81bd82a..8146d35 100644 --- a/DrissionPage/_base/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -7,7 +7,7 @@ from typing import List, Optional, Union from .chromium_driver import BrowserDriver from .._pages.chromium_page import ChromiumPage -from .._units.browser_download_manager import BrowserDownloadManager +from .._units.download_manager import BrowserDownloadManager class Browser(object): diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_commons/browser.py index 80106c4..25909bb 100644 --- a/DrissionPage/_commons/browser.py +++ b/DrissionPage/_commons/browser.py @@ -19,24 +19,19 @@ from .tools import port_is_using def connect_browser(option): """连接或启动浏览器 :param option: ChromiumOptions对象 - :return: chrome 路径和进程对象组成的元组 + :return: None """ debugger_address = option.debugger_address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') chrome_path = option.browser_path ip, port = debugger_address.split(':') - if ip != '127.0.0.1': + if ip != '127.0.0.1' or port_is_using(ip, port) or option.is_existing_only: test_connect(ip, port) - return None, None - - if port_is_using(ip, port): - test_connect(ip, port) - return None, None - - args = get_launch_args(option) - set_prefs(option) + return # ----------创建浏览器进程---------- + args = get_launch_args(option) + set_prefs(option) try: debugger = _run_browser(port, chrome_path, args) @@ -63,7 +58,7 @@ def get_launch_args(opt): result = set() has_user_path = False remote_allow = False - headless = False + headless = None for i in opt.arguments: if i.startswith(('--load-extension=', '--remote-debugging-port=')): continue @@ -74,7 +69,14 @@ def get_launch_args(opt): elif i.startswith('--remote-allow-origins='): remote_allow = True elif i.startswith('--headless'): - headless = True + if i == '--headless=false': + headless = False + continue + elif i == '--headless': + i = '--headless=new' + headless = True + else: + headless = True result.add(i) @@ -87,7 +89,7 @@ def get_launch_args(opt): if not remote_allow: result.add('--remote-allow-origins=*') - if not headless and system().lower() == 'linux': + if headless is not None and system().lower() == 'linux': from os import popen r = popen('systemctl list-units | grep graphical.target') if 'graphical.target' not in r.read(): @@ -146,27 +148,29 @@ def set_prefs(opt): dump(prefs_dict, f) -def test_connect(ip, port): +def test_connect(ip, port, timeout=30): """测试浏览器是否可用 :param ip: 浏览器ip :param port: 浏览器端口 + :param timeout: 超时时间 :return: None """ - end_time = perf_counter() + 30 + end_time = perf_counter() + timeout while perf_counter() < end_time: try: - tabs = requests_get(f'http://{ip}:{port}/json', timeout=10, headers={'Connection': 'close'}, proxies={'http': None, 'https': None}).json() + tabs = requests_get(f'http://{ip}:{port}/json', timeout=10, headers={'Connection': 'close'}, + proxies={'http': None, 'https': None}).json() for tab in tabs: if tab['type'] == 'page': return except Exception: sleep(.2) - if ip in ('127.0.0.1', 'localhost'): - raise BrowserConnectError(f'\n连接浏览器失败，可能原因：\n1、浏览器未启动\n2、{port}端口不是Chromium内核浏览器\n' - f'3、该浏览器未允许控制\n4、和已打开的浏览器冲突\n' - f'请尝试用ChromiumOptions指定别的端口和指定浏览器路径') - raise BrowserConnectError(f'{ip}:{port}浏览器无法链接。') + raise BrowserConnectError(f'\n{ip}:{port}浏览器无法链接。\n请确认：\n1、该端口为浏览器\n' + f'2、已添加--remote-allow-origins=*和--remote-debugging-port={port}启动项\n' + f'3、用户文件夹没有和已打开的浏览器冲突\n' + f'4、如为无界面系统，请使用headless模式\n' + f'可使用ChromiumOptions设置端口和用户文件夹路径。') def _run_browser(port, path: str, args) -> Popen: diff --git a/DrissionPage/_commons/browser.pyi b/DrissionPage/_commons/browser.pyi index a6fee1c..8c041ef 100644 --- a/DrissionPage/_commons/browser.pyi +++ b/DrissionPage/_commons/browser.pyi @@ -3,13 +3,18 @@ @Author : g1879 @Contact : g1879@qq.com """ +from typing import Union + from .._configs.chromium_options import ChromiumOptions -def connect_browser(option: ChromiumOptions) -> tuple: ... +def connect_browser(option: ChromiumOptions) -> None: ... def get_launch_args(opt: ChromiumOptions) -> list: ... def set_prefs(opt: ChromiumOptions) -> None: ... + + +def test_connect(ip: str, port: Union[int, str], timeout: float = 30) -> None: ... diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index 38809a4..df0c21d 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -35,6 +35,7 @@ class ChromiumOptions(object): self._page_load_strategy = options.get('page_load_strategy', 'normal') self._proxy = om.proxies.get('http', None) self._system_user_path = options.get('system_user_path', False) + self._existing_only = options.get('existing_only', False) user_path = user = False for arg in self._arguments: @@ -71,6 +72,7 @@ class ChromiumOptions(object): self._proxy = None self._auto_port = False self._system_user_path = False + self._existing_only = False @property def download_path(self): @@ -138,6 +140,11 @@ class ChromiumOptions(object): """返回是否使用系统安装的浏览器所使用的用户数据文件夹""" return self._system_user_path + @property + def is_existing_only(self): + """返回是否只接管现有浏览器方式""" + return self._existing_only + def set_argument(self, arg, value=None): """设置浏览器配置的argument属性 :param arg: 属性名 @@ -242,7 +249,7 @@ class ChromiumOptions(object): :param on_off: 开或关 :return: 当前对象 """ - on_off = 'new' if on_off else False + on_off = 'new' if on_off else 'false' return self.set_argument('--headless', on_off) def set_no_imgs(self, on_off=True): @@ -354,6 +361,14 @@ class ChromiumOptions(object): self._auto_port = False return self + def existing_only(self, on_off=True): + """设置只接管已有浏览器，不自动启动新的 + :param on_off: 是否开启自动获取端口号 + :return: 当前对象 + """ + self._existing_only = on_off + return self + def save(self, path=None): """保存设置到文件 :param path: ini文件的路径， None 保存到当前读取的配置文件，传入 'default' 保存到默认ini文件 @@ -380,7 +395,7 @@ class ChromiumOptions(object): # 设置chrome_options attrs = ('debugger_address', 'binary_location', 'arguments', 'extensions', 'user', 'page_load_strategy', - 'auto_port', 'system_user_path') + 'auto_port', 'system_user_path', 'is_existing_only') for i in attrs: om.set_item('chrome_options', i, self.__getattribute__(f'_{i}')) # 设置代理 diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index 68937fc..bcd3a0a 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -25,6 +25,7 @@ class ChromiumOptions(object): self._prefs_to_del: list = ... self._auto_port: bool = ... self._system_user_path: bool = ... + self._existing_only: bool = ... @property def download_path(self) -> str: ... @@ -53,9 +54,6 @@ class ChromiumOptions(object): @property def arguments(self) -> list: ... - @debugger_address.setter - def debugger_address(self, address: str): ... - @property def extensions(self) -> list: ... @@ -65,6 +63,9 @@ class ChromiumOptions(object): @property def system_user_path(self) -> bool: ... + @property + def is_existing_only(self) -> bool: ... + def set_argument(self, arg: str, value: Union[str, None, bool] = None) -> ChromiumOptions: ... def remove_argument(self, value: str) -> ChromiumOptions: ... @@ -106,6 +107,8 @@ class ChromiumOptions(object): def auto_port(self, on_off: bool = True) -> ChromiumOptions: ... + def existing_only(self, on_off: bool = True) -> ChromiumOptions: ... + def save(self, path: Union[str, Path] = None) -> str: ... def save_to_default(self) -> str: ... diff --git a/DrissionPage/_configs/configs.ini b/DrissionPage/_configs/configs.ini index 54d20ab..0190176 100644 --- a/DrissionPage/_configs/configs.ini +++ b/DrissionPage/_configs/configs.ini @@ -1,5 +1,5 @@ [paths] -download_path = +download_path = [chrome_options] debugger_address = 127.0.0.1:9222 @@ -11,6 +11,7 @@ page_load_strategy = normal user = Default auto_port = False system_user_path = False +is_existing_only = False [session_options] headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8', 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'connection': 'keep-alive', 'accept-charset': 'GB2312,utf-8;q=0.7,*;q=0.7'} @@ -21,6 +22,6 @@ page_load = 30 script = 30 [proxies] -http = +http = https = diff --git a/DrissionPage/_units/browser_download_manager.py b/DrissionPage/_units/download_manager.py similarity index 100% rename from DrissionPage/_units/browser_download_manager.py rename to DrissionPage/_units/download_manager.py diff --git a/DrissionPage/_units/browser_download_manager.pyi b/DrissionPage/_units/download_manager.pyi similarity index 100% rename from DrissionPage/_units/browser_download_manager.pyi rename to DrissionPage/_units/download_manager.pyi diff --git a/DrissionPage/_units/waiter.pyi b/DrissionPage/_units/waiter.pyi index d9d4441..f15dedc 100644 --- a/DrissionPage/_units/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -5,7 +5,7 @@ """ from typing import Union -from .browser_download_manager import DownloadMission +from .download_manager import DownloadMission from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame From f79a91b5a1edf69e6035241828f0a547ebe95652 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sat, 21 Oct 2023 15:58:29 +0800 Subject: [PATCH 045/182] =?UTF-8?q?new=5Ftab()=E8=BF=94=E5=9B=9E=E5=AF=B9?= =?UTF-8?q?=E8=B1=A1=EF=BC=9BChromiumOptions=E5=A2=9E=E5=8A=A0=E5=87=A0?= =?UTF-8?q?=E4=B8=AA=E8=AE=BE=E7=BD=AE=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_configs/chromium_options.py | 70 ++++++++++++++++++---- DrissionPage/_configs/chromium_options.pyi | 12 ++++ DrissionPage/_pages/chromium_page.py | 10 +++- DrissionPage/_pages/chromium_page.pyi | 4 +- DrissionPage/_pages/web_page.py | 8 +++ DrissionPage/_pages/web_page.pyi | 2 + 6 files changed, 93 insertions(+), 13 deletions(-) diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index df0c21d..d49ebba 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -316,30 +316,78 @@ class ChromiumOptions(object): :return: 当前对象 """ if browser_path is not None: - self._binary_location = str(browser_path) - self._auto_port = False + self.set_browser_path(browser_path) if local_port is not None: - self._debugger_address = f'127.0.0.1:{local_port}' - self._auto_port = False + self.set_local_port(local_port) if debugger_address is not None: - self.debugger_address = debugger_address + self.set_debugger_address(debugger_address) if download_path is not None: - self._download_path = str(download_path) + self.set_download_path(download_path) if user_data_path is not None: - u = str(user_data_path) - self.set_argument('--user-data-dir', u) - self._user_data_path = u - self._auto_port = False + self.set_user_data_path(user_data_path) if cache_path is not None: - self.set_argument('--disk-cache-dir', str(cache_path)) + self.set_cache_path(cache_path) return self + def set_local_port(self, port): + """设置本地启动端口 + :param port: 端口号 + :return: 当前对象 + """ + self._debugger_address = f'127.0.0.1:{port}' + self._auto_port = False + return self + + def set_debugger_address(self, address): + """设置浏览器地址，格式'ip:port' + :param address: 浏览器地址 + :return: 当前对象 + """ + self.debugger_address = address + return self + + def set_browser_path(self, path): + """设置浏览器可执行文件路径 + :param path: 浏览器路径 + :return: 当前对象 + """ + self._binary_location = str(path) + self._auto_port = False + return self + + def set_download_path(self, path): + """设置下载文件保存路径 + :param path: 下载路径 + :return: 当前对象 + """ + self._download_path = str(path) + return self + + def set_user_data_path(self, path): + """设置用户文件夹路径 + :param path: 用户文件夹路径 + :return: 当前对象 + """ + u = str(path) + self.set_argument('--user-data-dir', u) + self._user_data_path = u + self._auto_port = False + return self + + def set_cache_path(self, path): + """设置缓存路径 + :param path: 缓存路径 + :return: 当前对象 + """ + self.set_argument('--disk-cache-dir', str(path)) + return self + def use_system_user_path(self, on_off=True): """设置是否使用系统安装的浏览器默认用户文件夹 :param on_off: 开或关 diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index bcd3a0a..17d1906 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -99,6 +99,18 @@ class ChromiumOptions(object): def set_page_load_strategy(self, value: str) -> ChromiumOptions: ... + def set_browser_path(self, path: Union[str, Path]) -> ChromiumOptions: ... + + def set_local_port(self, port: Union[str, int]) -> ChromiumOptions: ... + + def set_debugger_address(self, address: str) -> ChromiumOptions: ... + + def set_download_path(self, path: Union[str, Path]) -> ChromiumOptions: ... + + def set_user_data_path(self, path: Union[str, Path]) -> ChromiumOptions: ... + + def set_cache_path(self, path: Union[str, Path]) -> ChromiumOptions: ... + def set_paths(self, browser_path: Union[str, Path] = None, local_port: Union[int, str] = None, debugger_address: str = None, download_path: Union[str, Path] = None, user_data_path: Union[str, Path] = None, cache_path: Union[str, Path] = None) -> ChromiumOptions: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index cfe8855..efea162 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -156,7 +156,7 @@ class ChromiumPage(ChromiumBase): """ return self._browser.find_tabs(title, url, tab_type, single) - def new_tab(self, url=None, switch_to=False): + def _new_tab(self, url=None, switch_to=False): """新建一个标签页,该标签页在最后面 :param url: 新标签页跳转到的网址 :param switch_to: 新建标签页后是否把焦点移过去 @@ -185,6 +185,14 @@ class ChromiumPage(ChromiumBase): return tid + def new_tab(self, url=None, switch_to=False): + """新建一个标签页,该标签页在最后面 + :param url: 新标签页跳转到的网址 + :param switch_to: 新建标签页后是否把焦点移过去 + :return: switch_to为False时返回新标签页对象，否则返回当前对象， + """ + return self if switch_to else ChromiumTab(self, self._new_tab(url, switch_to)) + def to_main_tab(self): """跳转到主标签页""" self.to_tab(self._main_tab) diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index 9a02407..15b6714 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -65,7 +65,9 @@ class ChromiumPage(ChromiumBase): def find_tabs(self, title: str = None, url: str = None, tab_type: Union[str, list, tuple] = None, single: bool = True) -> Union[str, List[str]]: ... - def new_tab(self, url: str = None, switch_to: bool = False) -> str: ... + def _new_tab(self, url=None, switch_to=False) -> str: ... + + def new_tab(self, url: str = None, switch_to: bool = False) -> Union[ChromiumTab, ChromiumPage]: ... def to_main_tab(self) -> None: ... diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index f91aa78..bc940b2 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -294,6 +294,14 @@ class WebPage(SessionPage, ChromiumPage, BasePage): """ return tab_id if isinstance(tab_id, WebPageTab) else WebPageTab(self, tab_id or self.tab_id) + def new_tab(self, url=None, switch_to=False): + """新建一个标签页,该标签页在最后面 + :param url: 新标签页跳转到的网址 + :param switch_to: 新建标签页后是否把焦点移过去 + :return: switch_to为False时返回新标签页对象，否则返回当前对象， + """ + return self if switch_to else WebPageTab(self, self._new_tab(url, switch_to)) + def close_driver(self): """关闭driver及浏览器""" if self._has_driver: diff --git a/DrissionPage/_pages/web_page.pyi b/DrissionPage/_pages/web_page.pyi index e05cb9f..7939f5a 100644 --- a/DrissionPage/_pages/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -121,6 +121,8 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def get_tab(self, tab_id: Union[str, WebPageTab] = None) -> WebPageTab: ... + def new_tab(self, url: str = None, switch_to: bool = False) -> Union[WebPageTab, WebPage]: ... + def close_driver(self) -> None: ... def close_session(self) -> None: ... From e5a2a254735cdf48afef21567be2bccb06569336 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 22 Oct 2023 10:00:53 +0800 Subject: [PATCH 046/182] =?UTF-8?q?get=5Fsrc()=E5=A2=9E=E5=8A=A0=E6=94=AF?= =?UTF-8?q?=E6=8C=81blob?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/chromium_driver.py | 9 ++-- DrissionPage/_base/chromium_driver.pyi | 6 ++- DrissionPage/_configs/chromium_options.py | 6 +-- DrissionPage/_elements/chromium_element.py | 63 ++++++++++++++++------ DrissionPage/_pages/chromium_page.py | 4 +- DrissionPage/_units/download_manager.py | 13 +++-- DrissionPage/_units/download_manager.pyi | 5 +- 7 files changed, 68 insertions(+), 38 deletions(-) diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index cd89be0..94a9664 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -137,12 +137,9 @@ class ChromiumDriver(object): except Empty: continue - if event['method'] in self.event_handlers: - try: - self.event_handlers[event['method']](**event['params']) - except Exception as e: - raise - # raise RuntimeError(f"\n回调函数错误：\n{e}") + function = self.event_handlers.get(event['method']) + if function: + function(**event['params']) self.event_queue.task_done() diff --git a/DrissionPage/_base/chromium_driver.pyi b/DrissionPage/_base/chromium_driver.pyi index 9e233a0..aeba4f9 100644 --- a/DrissionPage/_base/chromium_driver.pyi +++ b/DrissionPage/_base/chromium_driver.pyi @@ -5,7 +5,9 @@ """ from queue import Queue from threading import Thread, Event -from typing import Union, Callable, Dict +from typing import Union, Callable, Dict, Optional + +from websocket import WebSocket class GenericAttr(object): @@ -24,7 +26,7 @@ class ChromiumDriver(object): has_alert: bool _websocket_url: str _cur_id: int - _ws = None + _ws: Optional[WebSocket] _recv_th: Thread _handle_event_th: Thread _stopped: Event diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index d49ebba..67dfe89 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -117,8 +117,7 @@ class ChromiumOptions(object): @debugger_address.setter def debugger_address(self, address): """设置浏览器地址，格式ip:port""" - address = address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') - self._debugger_address = address + self.set_debugger_address(address) @property def arguments(self): @@ -349,7 +348,8 @@ class ChromiumOptions(object): :param address: 浏览器地址 :return: 当前对象 """ - self.debugger_address = address + address = address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') + self._debugger_address = address return self def set_browser_path(self, path): diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index bb318a9..c52501f 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -454,31 +454,64 @@ class ChromiumElement(DrissionElement): while not self.run_js(js) and perf_counter() < end_time: sleep(.1) + src = self.attr('src') + is_blob = src.startswith('blob') result = None end_time = perf_counter() + timeout while perf_counter() < end_time: - src = self.prop('currentSrc') - if not src: - continue + if is_blob: + js = """ + function fetchData(url) { + return new Promise((resolve, reject) => { + var xhr = new XMLHttpRequest(); + xhr.responseType = 'blob'; + xhr.onload = function() { + var reader = new FileReader(); + reader.onloadend = function() {resolve(reader.result);} + reader.readAsDataURL(xhr.response); + }; + xhr.open('GET', url, true); + xhr.send(); + }); + } + """ + try: + result = self.page.run_js(js, src) + break + except: + continue - node = self.page.run_cdp('DOM.describeNode', backendNodeId=self._backend_id)['node'] - frame = node.get('frameId', None) - frame = frame or self.page._target_id + else: + src = self.prop('currentSrc') + if not src: + continue - try: - result = self.page.run_cdp('Page.getResourceContent', frameId=frame, url=src) - break - except CDPError: - sleep(.1) + node = self.page.run_cdp('DOM.describeNode', backendNodeId=self._backend_id)['node'] + frame = node.get('frameId', None) + frame = frame or self.page._target_id + + try: + result = self.page.run_cdp('Page.getResourceContent', frameId=frame, url=src) + break + except CDPError: + sleep(.1) if not result: return None - if result['base64Encoded'] and base64_to_bytes: - from base64 import b64decode - return b64decode(result['content']) + if is_blob: + if base64_to_bytes: + from base64 import b64decode + return b64decode(result.split(',', 1)[1]) + else: + return result + else: - return result['content'] + if result['base64Encoded'] and base64_to_bytes: + from base64 import b64decode + return b64decode(result['content']) + else: + return result['content'] def save(self, path=None, name=None, timeout=None): """保存图片或其它有src属性的元素的资源 diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index efea162..6c6ba37 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -49,11 +49,11 @@ class ChromiumPage(ChromiumBase): # 接收浏览器地址和端口 elif isinstance(addr_driver_opts, str): self._driver_options = ChromiumOptions() - self._driver_options.debugger_address = addr_driver_opts + self._driver_options.set_debugger_address(addr_driver_opts) elif isinstance(addr_driver_opts, ChromiumDriver): self._driver_options = ChromiumOptions(False) - self._driver_options.debugger_address = addr_driver_opts.address + self._driver_options.set_debugger_address(addr_driver_opts.address) self._driver = addr_driver_opts else: diff --git a/DrissionPage/_units/download_manager.py b/DrissionPage/_units/download_manager.py index 0a0518c..029ccb9 100644 --- a/DrissionPage/_units/download_manager.py +++ b/DrissionPage/_units/download_manager.py @@ -24,7 +24,6 @@ class BrowserDownloadManager(object): t = TabDownloadSettings(self._page.tab_id) t.path = self._page.download_path self._missions = {} # {guid: DownloadMission} - self._tabs_settings = {self._page.tab_id: t} # {tab_id: TabDownloadSettings} self._tab_missions = {} # {tab_id: DownloadMission} self._flags = {} # {tab_id: [bool, DownloadMission]} @@ -44,7 +43,7 @@ class BrowserDownloadManager(object): :param path: 下载路径 :return: None """ - self._tabs_settings.setdefault(tab_id, TabDownloadSettings(tab_id)).path = str(Path(path).absolute()) + TabDownloadSettings(tab_id).path = str(Path(path).absolute()) if tab_id == self._page.tab_id: self._browser.run_cdp('Browser.setDownloadBehavior', downloadPath=str(Path(path).absolute()), behavior='allowAndName', eventsEnabled=True) @@ -55,7 +54,7 @@ class BrowserDownloadManager(object): :param rename: 文件名 :return: None """ - self._tabs_settings.setdefault(tab_id, TabDownloadSettings(tab_id)).rename = rename + TabDownloadSettings(tab_id).rename = rename def set_file_exists(self, tab_id, mode): """设置某个tab下载文件重名时执行的策略 @@ -63,7 +62,7 @@ class BrowserDownloadManager(object): :param mode: 下载路径 :return: None """ - self._tabs_settings.setdefault(tab_id, TabDownloadSettings(tab_id)).when_file_exists = mode + TabDownloadSettings(tab_id).when_file_exists = mode def set_flag(self, tab_id, flag): """设置某个tab的重命名文件名 @@ -125,7 +124,6 @@ class BrowserDownloadManager(object): :param tab_id: 标签页id :return: None """ - self._tabs_settings.pop(tab_id) self._tab_missions.pop(tab_id) self._flags.pop(tab_id) TabDownloadSettings.TABS.pop(tab_id) @@ -135,7 +133,7 @@ class BrowserDownloadManager(object): guid = kwargs['guid'] tab_id = self._browser._frames.get(kwargs['frameId'], self._page.tab_id) - settings = TabDownloadSettings(tab_id) + settings = TabDownloadSettings(tab_id if tab_id in TabDownloadSettings.TABS else self._page.tab_id) if settings.rename: tmp = kwargs['suggestedFilename'].rsplit('.', 1) ext_name = tmp[-1] if len(tmp) > 1 else '' @@ -165,7 +163,8 @@ class BrowserDownloadManager(object): else: self._tab_missions.setdefault(tab_id, []).append(guid) - self._flags[tab_id] = m + if self.get_flag(tab_id) is not None: + self._flags[tab_id] = m def _onDownloadProgress(self, **kwargs): """下载状态变化时执行""" diff --git a/DrissionPage/_units/download_manager.pyi b/DrissionPage/_units/download_manager.pyi index c4d640e..5153aa6 100644 --- a/DrissionPage/_units/download_manager.pyi +++ b/DrissionPage/_units/download_manager.pyi @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Dict, Optional, Union +from typing import Dict, Optional, Union, Literal from .._base.browser import Browser from .._pages.chromium_page import ChromiumPage @@ -10,7 +10,6 @@ class BrowserDownloadManager(object): _page: ChromiumPage = ... _missions: Dict[str, DownloadMission] = ... _tab_missions: dict = ... - _tabs_settings: Dict[str, TabDownloadSettings] = ... _flags: dict = ... def __init__(self, browser: Browser): ... @@ -22,7 +21,7 @@ class BrowserDownloadManager(object): def set_rename(self, tab_id: str, rename: str) -> None: ... - def set_file_exists(self, tab_id: str, mode: str) -> None: ... + def set_file_exists(self, tab_id: str, mode: Literal['rename', 'skip', 'overwrite']) -> None: ... def set_flag(self, tab_id: str, flag: Optional[bool, DownloadMission]) -> None: ... From 06a215d93a6b0f4f00eae6933b24823cc44194dd Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 23 Oct 2023 01:03:46 +0800 Subject: [PATCH 047/182] =?UTF-8?q?=E5=9F=BA=E6=9C=AC=E5=AE=8C=E6=88=90Lis?= =?UTF-8?q?tener=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 122 +----------- DrissionPage/_elements/chromium_element.pyi | 24 +-- DrissionPage/_units/clicker.py | 115 +++++++++++ DrissionPage/_units/clicker.pyi | 27 +++ DrissionPage/_units/network_listener.py | 205 ++++++++++++-------- DrissionPage/_units/network_listener.pyi | 45 ++--- DrissionPage/_units/waiter.py | 8 + DrissionPage/_units/waiter.pyi | 6 +- 8 files changed, 308 insertions(+), 244 deletions(-) create mode 100644 DrissionPage/_units/clicker.py create mode 100644 DrissionPage/_units/clicker.pyi diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index c52501f..f3e8e84 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -7,17 +7,18 @@ from os.path import basename, sep from pathlib import Path from time import perf_counter, sleep +from .session_element import make_session_ele from .._base.base import DrissionElement, BaseElement from .._commons.constants import FRAME_ELEMENT, NoneElement, Settings from .._commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions from .._commons.locator import get_loc from .._commons.tools import get_usable_path from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, location_in_viewport, offset_scroll -from ..errors import ContextLossError, ElementLossError, JavaScriptError, NoRectError, ElementNotFoundError, \ - CDPError, NoResourceError, CanNotClickError -from .session_element import make_session_ele +from .._units.clicker import Clicker from .._units.setter import ChromiumElementSetter from .._units.waiter import ChromiumElementWaiter +from ..errors import ContextLossError, ElementLossError, JavaScriptError, ElementNotFoundError, \ + CDPError, NoResourceError class ChromiumElement(DrissionElement): @@ -37,7 +38,7 @@ class ChromiumElement(DrissionElement): self._set = None self._states = None self._pseudo = None - self._click = None + self._clicker = None self._tag = None self._wait = None @@ -183,9 +184,9 @@ class ChromiumElement(DrissionElement): @property def click(self): """返回用于点击的对象""" - if self._click is None: - self._click = Click(self) - return self._click + if self._clicker is None: + self._clicker = Clicker(self) + return self._clicker @property def wait(self): @@ -502,7 +503,7 @@ class ChromiumElement(DrissionElement): if is_blob: if base64_to_bytes: from base64 import b64decode - return b64decode(result.split(',', 1)[1]) + return b64decode(result.split(',', 1)[-1]) else: return result @@ -1598,111 +1599,6 @@ class Locations(object): return x + sx, y + sy -class Click(object): - def __init__(self, ele): - """ - :param ele: ChromiumElement - """ - self._ele = ele - - def __call__(self, by_js=False, timeout=1): - """点击元素 - 如果遇到遮挡，可选择是否用js点击 - :param by_js: 是否用js点击，为None时先用模拟点击，遇到遮挡改用js，为True时直接用js点击，为False时只用模拟点击 - :param timeout: 模拟点击的超时时间，等待元素可见、不被遮挡、进入视口 - :return: 是否点击成功 - """ - return self.left(by_js, timeout) - - def left(self, by_js=False, timeout=1): - """点击元素，可选择是否用js点击 - :param by_js: 是否用js点击，为None时先用模拟点击，遇到遮挡改用js，为True时直接用js点击，为False时只用模拟点击 - :param timeout: 模拟点击的超时时间，等待元素可见、不被遮挡、进入视口 - :return: 是否点击成功 - """ - if not by_js: - try: - self._ele.scroll.to_see() - can_click = False - - timeout = self._ele.page.timeout if timeout is None else timeout - if timeout == 0: - if self._ele.states.is_in_viewport and self._ele.states.is_enabled and self._ele.states.is_displayed: - can_click = True - else: - end_time = perf_counter() + timeout - while perf_counter() < end_time: - if self._ele.states.is_in_viewport and self._ele.states.is_enabled and self._ele.states.is_displayed: - can_click = True - break - - if not self._ele.states.is_in_viewport: - by_js = True - - elif can_click and (by_js is False or not self._ele.states.is_covered): - client_x, client_y = self._ele.locations.viewport_midpoint if self._ele.tag == 'input' \ - else self._ele.locations.viewport_click_point - self._click(client_x, client_y) - return True - - except NoRectError: - by_js = True - - if by_js is not False: - self._ele.run_js('this.click();') - return True - if Settings.raise_when_click_failed: - raise CanNotClickError - - return False - - def right(self): - """右键单击""" - self._ele.page.scroll.to_see(self._ele) - x, y = self._ele.locations.viewport_click_point - self._click(x, y, 'right') - - def middle(self): - """中键单击""" - self._ele.page.scroll.to_see(self._ele) - x, y = self._ele.locations.viewport_click_point - self._click(x, y, 'middle') - - def at(self, offset_x=None, offset_y=None, button='left', count=1): - """带偏移量点击本元素，相对于左上角坐标。不传入x或y值时点击元素中间点 - :param offset_x: 相对元素左上角坐标的x轴偏移量 - :param offset_y: 相对元素左上角坐标的y轴偏移量 - :param button: 点击哪个键，可选 left, middle, right, back, forward - :param count: 点击次数 - :return: None - """ - self._ele.page.scroll.to_see(self._ele) - if offset_x is None and offset_y is None: - w, h = self._ele.size - offset_x = w // 2 - offset_y = h // 2 - x, y = offset_scroll(self._ele, offset_x, offset_y) - self._click(x, y, button, count) - - def twice(self): - """双击元素""" - self.at(count=2) - - def _click(self, client_x, client_y, button='left', count=1): - """实施点击 - :param client_x: 视口中的x坐标 - :param client_y: 视口中的y坐标 - :param button: 'left' 'right' 'middle' 'back' 'forward' - :param count: 点击次数 - :return: None - """ - self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mousePressed', - x=client_x, y=client_y, button=button, clickCount=count) - # sleep(.05) - self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mouseReleased', - x=client_x, y=client_y, button=button) - - class ChromiumScroll(object): """用于滚动的对象""" diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 7ee4253..253dc7c 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -6,6 +6,7 @@ from pathlib import Path from typing import Union, Tuple, List, Any +from .._units.clicker import Clicker from .._base.base import DrissionElement, BaseElement from .._commons.constants import NoneElement from .._elements.session_element import SessionElement @@ -30,7 +31,7 @@ class ChromiumElement(DrissionElement): self._doc_id: str = ... self._ids: ChromiumElementIds = ... self._scroll: ChromiumElementScroll = ... - self._click: Click = ... + self._clicker: Clicker = ... self._select: ChromiumSelect = ... self._wait: ChromiumElementWaiter = ... self._locations: Locations = ... @@ -94,7 +95,7 @@ class ChromiumElement(DrissionElement): def scroll(self) -> ChromiumElementScroll: ... @property - def click(self) -> Click: ... + def click(self) -> Clicker: ... def parent(self, level_or_loc: Union[tuple, str, int] = 1, index: int = 1) -> Union[ChromiumElement, None]: ... @@ -437,25 +438,6 @@ class Locations(object): def _get_page_coord(self, x: int, y: int) -> Tuple[int, int]: ... -class Click(object): - def __init__(self, ele: ChromiumElement): - self._ele: ChromiumElement = ... - - def __call__(self, by_js: Union[None, bool] = False, timeout: float = 1) -> bool: ... - - def left(self, by_js: Union[None, bool] = False, timeout: float = 1) -> bool: ... - - def right(self) -> None: ... - - def middle(self) -> None: ... - - def at(self, offset_x: int = None, offset_y: int = None, button: str = 'left', count: int = 1) -> None: ... - - def twice(self, by_js: bool = False) -> None: ... - - def _click(self, client_x: int, client_y: int, button: str = 'left', count: int = 1) -> None: ... - - class ChromiumScroll(object): def __init__(self, page_or_ele: Union[ChromiumBase, ChromiumElement, ChromiumFrame]): self.t1: str = ... diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py new file mode 100644 index 0000000..c075d35 --- /dev/null +++ b/DrissionPage/_units/clicker.py @@ -0,0 +1,115 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" +from time import perf_counter + +from .._commons.constants import Settings +from .._commons.web import offset_scroll +from ..errors import NoRectError, CanNotClickError + + +class Clicker(object): + def __init__(self, ele): + """ + :param ele: ChromiumElement + """ + self._ele = ele + + def __call__(self, by_js=False, timeout=1): + """点击元素 + 如果遇到遮挡，可选择是否用js点击 + :param by_js: 是否用js点击，为None时先用模拟点击，遇到遮挡改用js，为True时直接用js点击，为False时只用模拟点击 + :param timeout: 模拟点击的超时时间，等待元素可见、不被遮挡、进入视口 + :return: 是否点击成功 + """ + return self.left(by_js, timeout) + + def left(self, by_js=False, timeout=1): + """点击元素，可选择是否用js点击 + :param by_js: 是否用js点击，为None时先用模拟点击，遇到遮挡改用js，为True时直接用js点击，为False时只用模拟点击 + :param timeout: 模拟点击的超时时间，等待元素可见、不被遮挡、进入视口 + :return: 是否点击成功 + """ + if not by_js: + try: + self._ele.scroll.to_see() + can_click = False + + timeout = self._ele.page.timeout if timeout is None else timeout + if timeout == 0: + if self._ele.states.is_in_viewport and self._ele.states.is_enabled and self._ele.states.is_displayed: + can_click = True + else: + end_time = perf_counter() + timeout + while perf_counter() < end_time: + if self._ele.states.is_in_viewport and self._ele.states.is_enabled and self._ele.states.is_displayed: + can_click = True + break + + if not self._ele.states.is_in_viewport: + by_js = True + + elif can_click and (by_js is False or not self._ele.states.is_covered): + client_x, client_y = self._ele.locations.viewport_midpoint if self._ele.tag == 'input' \ + else self._ele.locations.viewport_click_point + self._click(client_x, client_y) + return True + + except NoRectError: + by_js = True + + if by_js is not False: + self._ele.run_js('this.click();') + return True + if Settings.raise_when_click_failed: + raise CanNotClickError + + return False + + def right(self): + """右键单击""" + self._ele.page.scroll.to_see(self._ele) + x, y = self._ele.locations.viewport_click_point + self._click(x, y, 'right') + + def middle(self): + """中键单击""" + self._ele.page.scroll.to_see(self._ele) + x, y = self._ele.locations.viewport_click_point + self._click(x, y, 'middle') + + def at(self, offset_x=None, offset_y=None, button='left', count=1): + """带偏移量点击本元素，相对于左上角坐标。不传入x或y值时点击元素中间点 + :param offset_x: 相对元素左上角坐标的x轴偏移量 + :param offset_y: 相对元素左上角坐标的y轴偏移量 + :param button: 点击哪个键，可选 left, middle, right, back, forward + :param count: 点击次数 + :return: None + """ + self._ele.page.scroll.to_see(self._ele) + if offset_x is None and offset_y is None: + w, h = self._ele.size + offset_x = w // 2 + offset_y = h // 2 + x, y = offset_scroll(self._ele, offset_x, offset_y) + self._click(x, y, button, count) + + def twice(self): + """双击元素""" + self.at(count=2) + + def _click(self, client_x, client_y, button='left', count=1): + """实施点击 + :param client_x: 视口中的x坐标 + :param client_y: 视口中的y坐标 + :param button: 'left' 'right' 'middle' 'back' 'forward' + :param count: 点击次数 + :return: None + """ + self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mousePressed', + x=client_x, y=client_y, button=button, clickCount=count) + # sleep(.05) + self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mouseReleased', + x=client_x, y=client_y, button=button) diff --git a/DrissionPage/_units/clicker.pyi b/DrissionPage/_units/clicker.pyi new file mode 100644 index 0000000..dbdcf58 --- /dev/null +++ b/DrissionPage/_units/clicker.pyi @@ -0,0 +1,27 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" +from typing import Union + +from .._elements.chromium_element import ChromiumElement + + +class Clicker(object): + def __init__(self, ele: ChromiumElement): + self._ele: ChromiumElement = ... + + def __call__(self, by_js: Union[None, bool] = False, timeout: float = 1) -> bool: ... + + def left(self, by_js: Union[None, bool] = False, timeout: float = 1) -> bool: ... + + def right(self) -> None: ... + + def middle(self) -> None: ... + + def at(self, offset_x: int = None, offset_y: int = None, button: str = 'left', count: int = 1) -> None: ... + + def twice(self, by_js: bool = False) -> None: ... + + def _click(self, client_x: int, client_y: int, button: str = 'left', count: int = 1) -> None: ... diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index c2f49b1..c2eada4 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -7,7 +7,6 @@ from base64 import b64decode from json import JSONDecodeError, loads from queue import Queue from re import search -from threading import Thread from time import perf_counter, sleep from requests.structures import CaseInsensitiveDict @@ -23,24 +22,24 @@ class NetworkListener(object): :param page: ChromiumBase对象 """ self._page = page - self._driver = self._page.driver + self._driver = page.driver + self._driver.call_method('Network.enable') - self._tmp = None # 临存捕捉到的数据 + self._caught = None # 临存捕捉到的数据 self._request_ids = None # 暂存须要拦截的请求id - self._total_count = None # 当次监听的数量上限 - self._caught_count = None # 当次已监听到的数量 - self._begin_time = None # 当次监听开始时间 - self._timeout = None # 当次监听超时时间 - self.listening = False self._targets = None # 默认监听所有 self.tab_id = None # 当前tab的id - self._results = [] self._is_regex = False self._method = None + @property + def targets(self): + """返回监听目标""" + return self._targets + def set_targets(self, targets=True, is_regex=False, method=None): """指定要等待的数据包 :param targets: 要匹配的数据包url特征，可用list等传入多个，为True时获取所有 @@ -54,10 +53,7 @@ class NetworkListener(object): if targets is True: targets = '' - if isinstance(targets, str): - self._targets = {targets} - else: - self._targets = set(targets) + self._targets = {targets} if isinstance(targets, str) else set(targets) self._is_regex = is_regex @@ -69,93 +65,128 @@ class NetworkListener(object): else: raise TypeError('method参数只能是str、list、tuple、set类型。') - def listen(self, targets=None, count=None, timeout=None): - """拦截目标请求，直到超时或达到拦截个数，每次拦截前清空结果 - 可监听多个目标，请求url包含这些字符串就会被记录 - :param targets: 要监听的目标字符串或其组成的列表，True监听所有，None则保留之前的目标不变 - :param count: 要记录的个数，到达个数停止监听 - :param timeout: 监听最长时间，到时间即使未达到记录个数也停止，None为无限长 + def listen(self, targets=None, is_regex=False, method=None): + """拦截目标请求，每次拦截前清空结果 + :param targets: 要匹配的数据包url特征，可用list等传入多个，为True时获取所有 + :param is_regex: 设置的target是否正则表达式 + :param method: 设置监听的请求类型，可用list等指定多个，为None时监听全部 :return: None """ if targets: - self.set_targets(targets) + self.set_targets(targets, is_regex, method) self.listening = True - self._results = [] self._request_ids = {} - self._tmp = Queue(maxsize=0) + self._caught = Queue(maxsize=0) - self._caught_count = 0 - self._begin_time = perf_counter() - self._timeout = timeout + self._set_callback() - self._set_callback_func() - - self._total_count = len(self._targets) if not count else count - - Thread(target=self._wait_to_stop).start() - - def stop(self): - """停止监听""" - self._stop() - self.listening = False - - def wait(self): - """等待监听结束""" - while self.listening: - sleep(.2) - return self._results - - def get_results(self, target=None): - """获取结果列表 - :param target: 要获取的目标，为None时获取全部 - :return: 结果数据组成的列表 + def wait(self, count=1, timeout=None, fix_count=True): + """等待符合要求的数据包到达指定数量 + :param count: 需要捕捉的数据包数量 + :param timeout: 超时时间，为None无限等待 + :param fix_count: 是否必须满足总数要求，发生超时，为True返回False，为False返回已捕捉到的数据包 + :return: count为1时返回数据包对象，大于1时返回列表，超时且fix_count为True时返回False """ - return self._results if target is None else [i for i in self._results if i.target == target] + if not self.listening: + raise RuntimeError('监听未启动或已暂停。') + if not timeout: + while self._caught.qsize() < count: + sleep(.05) + fail = False - def _wait_to_stop(self): - """当收到停止信号、到达须获取结果数、到时间就停止""" - while self._is_continue(): - sleep(.2) - self.stop() + else: + end = perf_counter() + count + while True: + if perf_counter() > end: + fail = True + break + if self._caught.qsize() >= count: + fail = False + break - def _is_continue(self): - """是否继续当前监听""" - return self.listening \ - and (self._total_count is None or self._caught_count < self._total_count) \ - and (self._timeout is None or perf_counter() - self._begin_time < self._timeout) + if fail: + if fix_count or not self._caught.qsize(): + return False + else: + return [self._caught.get_nowait() for _ in range(self._caught.qsize())] - def steps(self, gap=1): + if count == 1: + return self._caught.get_nowait() + + return [self._caught.get_nowait() for _ in range(count)] + + def steps(self, count=None, timeout=None, gap=1): """用于单步操作，可实现没收到若干个数据包执行一步操作（如翻页） + :param count: 需捕获的数据包总数，为None表示无限 + :param timeout: 每个数据包等待时间，为None表示无限 :param gap: 每接收到多少个数据包触发 :return: 用于在接收到监听目标时触发动作的可迭代对象 """ - if not isinstance(gap, int) or gap < 1: - raise ValueError('gap参数必须为大于0的整数。') - while self.listening or not self._tmp.empty(): - while self._tmp.qsize() >= gap: - yield self._tmp.get(False) if gap == 1 else [self._tmp.get(False) for _ in range(gap)] + caught = 0 + end = perf_counter() + timeout if timeout else None + while True: + if timeout and perf_counter() > end: + return + if self._caught.qsize() >= gap: + yield self._caught.get_nowait() if gap == 1 else [self._caught.get_nowait() for _ in range(gap)] + if timeout: + end = perf_counter() + timeout + if count: + caught += gap + if caught >= count: + return + sleep(.05) - sleep(.1) + def stop(self): + """停止监听，清空已监听到的列表""" + if self.listening: + self.pause() + self.clear() - def _set_callback_func(self): + def pause(self, clear=True): + """暂停监听 + :param clear: 是否清空已获取队列 + :return: None + """ + if self.listening: + self._driver.set_listener('Network.requestWillBeSent', None) + self._driver.set_listener('Network.responseReceived', None) + self._driver.set_listener('Network.loadingFinished', None) + self._driver.set_listener('Network.loadingFailed', None) + self.listening = False + if clear: + self.clear() + + def go_on(self): + """继续暂停的监听""" + if self.listening: + return + self._set_callback() + self.listening = True + + def clear(self): + """清空结果""" + self._request_ids = {} + self._caught.queue.clear() + + def _set_callback(self): """设置监听请求的回调函数""" self._driver.set_listener('Network.requestWillBeSent', self._requestWillBeSent) self._driver.set_listener('Network.responseReceived', self._response_received) self._driver.set_listener('Network.loadingFinished', self._loading_finished) self._driver.set_listener('Network.loadingFailed', self._loading_failed) - self._driver.call_method('Network.enable') - - def _stop(self) -> None: - """停止监听前要做的工作""" - self._driver.set_listener('Network.requestWillBeSent', None) - self._driver.set_listener('Network.responseReceived', None) - self._driver.set_listener('Network.loadingFinished', None) - self._driver.set_listener('Network.loadingFailed', None) - # self._driver.call_method('Network.disable') def _requestWillBeSent(self, **kwargs): """接收到请求时的回调函数""" + if not self._targets: + self._request_ids[kwargs['requestId']] = DataPacket(self._page.tab_id, None, kwargs) + if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): + self._request_ids[kwargs['requestId']]._raw_post_data = \ + self._page.run_cdp('Network.getRequestPostData', requestId=kwargs['requestId'])['postData'] + + return + for target in self._targets: if ((self._is_regex and search(target, kwargs['request']['url'])) or (not self._is_regex and target in kwargs['request']['url'])) and ( @@ -191,9 +222,11 @@ class NetworkListener(object): dp._raw_body = body dp._base64_body = is_base64 - self._tmp.put(dp) - self._results.append(dp) - self._caught_count += 1 + self._caught.put(dp) + try: + self._request_ids.pop(request_id) + except: + pass def _loading_failed(self, **kwargs): """请求失败时的回调方法""" @@ -203,21 +236,23 @@ class NetworkListener(object): dp.errorText = kwargs['errorText'] dp._resource_type = kwargs['type'] - self._tmp.put(dp) - self._results.append(dp) - self._caught_count += 1 + self._caught.put(dp) + try: + self._request_ids.pop(request_id) + except: + pass class DataPacket(object): """返回的数据包管理类""" - def __init__(self, tab, target, raw_request): + def __init__(self, tab_id, target, raw_request): """ - :param tab: 产生这个数据包的tab的id + :param tab_id: 产生这个数据包的tab的id :param target: 监听目标 :param raw_request: 原始request数据，从cdp获得 """ - self.tab = tab + self.tab = tab_id self.target = target self._raw_request = raw_request @@ -232,6 +267,10 @@ class DataPacket(object): self.errorText = None self._resource_type = None + def __repr__(self): + t = f'"{self.target}"' if self.target is not None else None + return f'' + @property def url(self): return self.request.url diff --git a/DrissionPage/_units/network_listener.pyi b/DrissionPage/_units/network_listener.pyi index e2b7e7a..c52f85a 100644 --- a/DrissionPage/_units/network_listener.pyi +++ b/DrissionPage/_units/network_listener.pyi @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from queue import Queue -from typing import Union, Dict, List, Iterable, Tuple +from typing import Union, Dict, List, Iterable, Tuple, Optional from requests.structures import CaseInsensitiveDict @@ -15,36 +15,36 @@ from .._pages.chromium_base import ChromiumBase class NetworkListener(object): def __init__(self, page: ChromiumBase): self._page: ChromiumBase = ... - self._total_count: int = ... - self._caught_count: int = ... self._targets: Union[str, dict] = ... - self._results: list = ... self._method: set = ... - self._tmp: Queue = ... + self._caught: Queue = ... self._is_regex: bool = ... self._driver: ChromiumDriver = ... self._request_ids: dict = ... self.listening: bool = ... - self._timeout: float = ... - self._begin_time: float = ... + + @property + def targets(self) -> Optional[set]: ... def set_targets(self, targets: Union[str, list, tuple, set, None] = None, is_regex: bool = False, - count: int = None, method: Union[str, list, tuple, set] = None) -> None: ... + method: Union[str, list, tuple, set] = None) -> None: ... def stop(self) -> None: ... - def wait(self):... + def pause(self, clear: bool = True) -> None: ... + + def go_on(self) -> None: ... + + def wait(self, count: int = 1, timeout: float = None, fix_count: bool = True): ... @property def results(self) -> Union[DataPacket, Dict[str, List[DataPacket]], False]: ... def clear(self) -> None: ... - def listen(self, targets: Union[str, List[str], Tuple, bool, None] = ..., count: int = ..., - timeout: float = ...) -> Union[DataPacket, Dict[str, List[DataPacket]], False]: ... - - def _listen(self, timeout: float = None, - any_one: bool = False) -> Union[DataPacket, Dict[str, List[DataPacket]], False]: ... + def listen(self, targets: Union[str, List[str], Tuple, bool, None] = None, is_regex: bool = False, + method: Union[str, list, tuple, set] = None) \ + -> Union[DataPacket, Dict[str, List[DataPacket]], False]: ... def _requestWillBeSent(self, **kwargs) -> None: ... @@ -54,24 +54,17 @@ class NetworkListener(object): def _loading_failed(self, **kwargs) -> None: ... - def _request_paused(self, **kwargs) -> None: ... + def steps(self, count: int = None, timeout: float = None, + gap=1) -> Iterable[Union[DataPacket, List[DataPacket]]]: ... - def _wait_to_stop(self) -> None: ... - - def _is_continue(self) -> bool: ... - - def steps(self, gap=1) -> Iterable[Union[DataPacket, List[DataPacket]]]: ... - - def _set_callback_func(self) -> None: ... - - def _stop(self) -> None: ... + def _set_callback(self) -> None: ... class DataPacket(object): """返回的数据包管理类""" - def __init__(self, tab: str, target: str, raw_info: dict): - self.tab: str = ... + def __init__(self, tab_id: str, target: Optional[str], raw_info: dict): + self.tab_id: str = ... self.target: str = ... self._raw_request: dict = ... self._raw_response: dict = ... diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 971e21a..11d4860 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -126,6 +126,14 @@ class ChromiumBaseWaiter(object): """ return self._change('title', text, exclude, timeout, raise_err) + def data_packets(self, count=1, timeout=None, fix_count: bool = True): + """等待符合要求的数据包到达指定数量 + :param count: 需要捕捉的数据包数量 + :param timeout: 超时时间，为None无限等待 + :param fix_count: 是否必须满足总数要求，发生超时，为True返回False，为False返回已捕捉到的数据包 + :return: count为1时返回数据包对象，大于1时返回列表，超时且fix_count为True时返回False""" + return self._driver.listener.wait(count, timeout, fix_count) + def _change(self, arg, text, exclude=False, timeout=None, raise_err=None): """等待指定属性变成包含或不包含指定文本 :param arg: 要被匹配的属性 diff --git a/DrissionPage/_units/waiter.pyi b/DrissionPage/_units/waiter.pyi index f15dedc..d8ad640 100644 --- a/DrissionPage/_units/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -3,9 +3,10 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union +from typing import Union, List from .download_manager import DownloadMission +from .network_listener import DataPacket from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame @@ -46,6 +47,9 @@ class ChromiumBaseWaiter(object): def title_change(self, text: str, exclude: bool = False, timeout: float = None, raise_err: bool = None) -> bool: ... + def data_packets(self, count: int = 1, timeout: float = None, + fix_count: bool = True) -> Union[List[DataPacket], DataPacket, None]: ... + def _change(self, arg: str, text: str, exclude: bool = False, timeout: float = None, raise_err: bool = None) -> bool: ... From 90c715aeae834d60a1786f89c05f3fbb53a88e35 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 23 Oct 2023 17:44:19 +0800 Subject: [PATCH 048/182] =?UTF-8?q?=E5=A2=9E=E5=8A=A0ele.states.has=5Frect?= =?UTF-8?q?=E5=92=8Cele.wait.stop=5Fmoving()=EF=BC=9Bcommon=E5=88=A0?= =?UTF-8?q?=E9=99=A4FlowViewer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 12 +++++- DrissionPage/_elements/chromium_element.pyi | 3 ++ DrissionPage/_pages/chromium_base.py | 4 +- DrissionPage/_pages/chromium_base.pyi | 2 +- DrissionPage/_units/network_listener.py | 15 +++---- DrissionPage/_units/network_listener.pyi | 4 +- DrissionPage/_units/waiter.py | 46 +++++++++++++++++---- DrissionPage/_units/waiter.pyi | 2 + DrissionPage/common.py | 5 +-- 9 files changed, 69 insertions(+), 24 deletions(-) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index f3e8e84..07176bf 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -18,7 +18,7 @@ from .._units.clicker import Clicker from .._units.setter import ChromiumElementSetter from .._units.waiter import ChromiumElementWaiter from ..errors import ContextLossError, ElementLossError, JavaScriptError, ElementNotFoundError, \ - CDPError, NoResourceError + CDPError, NoResourceError, NoRectError class ChromiumElement(DrissionElement): @@ -1465,7 +1465,7 @@ class ChromiumElementStates(object): @property def is_in_viewport(self): - """返回元素是否出现在视口中，以元素可以接受点击的点为判断""" + """返回元素是否出现在视口中，以元素click_point为判断""" x, y = self._ele.locations.click_point return location_in_viewport(self._ele.page, x, y) if x else False @@ -1491,6 +1491,14 @@ class ChromiumElementStates(object): return False + @property + def has_rect(self): + """返回元素是否拥有位置和大小，没有返回False，有返回大小元组""" + try: + return self._ele.size + except NoRectError: + return False + class ShadowRootStates(object): def __init__(self, ele): diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 253dc7c..b10dd0a 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -239,6 +239,9 @@ class ChromiumElementStates(object): @property def is_covered(self) -> bool: ... + @property + def has_rect(self) -> Union[bool, Tuple[int, int]]: ... + class ChromiumShadowRoot(BaseElement): diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index c2c2267..35d93c5 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -255,6 +255,8 @@ class ChromiumBase(BasePage): def _onFileChooserOpened(self, **kwargs): """文件选择框打开时执行""" if self._upload_list: + if 'backendNodeId' not in kwargs: + raise TypeError('该输入框无法接管，请改用对元素输入路径的方法设置。') files = self._upload_list if kwargs['mode'] == 'selectMultiple' else self._upload_list[:1] self.run_cdp('DOM.setFileInputFiles', files=files, backendNodeId=kwargs['backendNodeId']) @@ -419,7 +421,7 @@ class ChromiumBase(BasePage): return self._actions @property - def listener(self): + def listen(self): """返回用于聆听数据包的对象""" if self._listener is None: self._listener = NetworkListener(self) diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 33f4030..447e86c 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -154,7 +154,7 @@ class ChromiumBase(BasePage): def actions(self) -> ActionChains: ... @property - def listener(self) -> NetworkListener: ... + def listen(self) -> NetworkListener: ... def run_js(self, script: str, *args: Any, as_expr: bool = False) -> Any: ... diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index c2eada4..6376ee1 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -11,6 +11,7 @@ from time import perf_counter, sleep from requests.structures import CaseInsensitiveDict +from .._base.chromium_driver import ChromiumDriver from ..errors import CDPError @@ -22,7 +23,7 @@ class NetworkListener(object): :param page: ChromiumBase对象 """ self._page = page - self._driver = page.driver + self._driver = ChromiumDriver(page.tab_id, 'page', page.address) self._driver.call_method('Network.enable') self._caught = None # 临存捕捉到的数据 @@ -65,7 +66,7 @@ class NetworkListener(object): else: raise TypeError('method参数只能是str、list、tuple、set类型。') - def listen(self, targets=None, is_regex=False, method=None): + def start(self, targets=None, is_regex=False, method=None): """拦截目标请求，每次拦截前清空结果 :param targets: 要匹配的数据包url特征，可用list等传入多个，为True时获取所有 :param is_regex: 设置的target是否正则表达式 @@ -180,10 +181,10 @@ class NetworkListener(object): def _requestWillBeSent(self, **kwargs): """接收到请求时的回调函数""" if not self._targets: - self._request_ids[kwargs['requestId']] = DataPacket(self._page.tab_id, None, kwargs) + self._request_ids[kwargs['requestId']] = DataPacket(self._driver.id, None, kwargs) if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): self._request_ids[kwargs['requestId']]._raw_post_data = \ - self._page.run_cdp('Network.getRequestPostData', requestId=kwargs['requestId'])['postData'] + self._driver.call_method('Network.getRequestPostData', requestId=kwargs['requestId'])['postData'] return @@ -191,11 +192,11 @@ class NetworkListener(object): if ((self._is_regex and search(target, kwargs['request']['url'])) or (not self._is_regex and target in kwargs['request']['url'])) and ( not self._method or kwargs['request']['method'] in self._method): - self._request_ids[kwargs['requestId']] = DataPacket(self._page.tab_id, target, kwargs) + self._request_ids[kwargs['requestId']] = DataPacket(self._driver.id, target, kwargs) if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): self._request_ids[kwargs['requestId']]._raw_post_data = \ - self._page.run_cdp('Network.getRequestPostData', requestId=kwargs['requestId'])['postData'] + self._driver.call_method('Network.getRequestPostData', requestId=kwargs['requestId'])['postData'] break @@ -212,7 +213,7 @@ class NetworkListener(object): dp = self._request_ids.get(request_id) if dp: try: - r = self._page.run_cdp('Network.getResponseBody', requestId=request_id) + r = self._driver.call_method('Network.getResponseBody', requestId=request_id) body = r['body'] is_base64 = r['base64Encoded'] except CDPError: diff --git a/DrissionPage/_units/network_listener.pyi b/DrissionPage/_units/network_listener.pyi index c52f85a..3b85ad4 100644 --- a/DrissionPage/_units/network_listener.pyi +++ b/DrissionPage/_units/network_listener.pyi @@ -42,8 +42,8 @@ class NetworkListener(object): def clear(self) -> None: ... - def listen(self, targets: Union[str, List[str], Tuple, bool, None] = None, is_regex: bool = False, - method: Union[str, list, tuple, set] = None) \ + def start(self, targets: Union[str, List[str], Tuple, bool, None] = None, is_regex: bool = False, + method: Union[str, list, tuple, set] = None) \ -> Union[DataPacket, Dict[str, List[DataPacket]], False]: ... def _requestWillBeSent(self, **kwargs) -> None: ... diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 11d4860..4baca34 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -2,7 +2,7 @@ from time import sleep, perf_counter from .._commons.constants import Settings -from ..errors import WaitTimeoutError +from ..errors import WaitTimeoutError, NoRectError class ChromiumBaseWaiter(object): @@ -132,7 +132,7 @@ class ChromiumBaseWaiter(object): :param timeout: 超时时间，为None无限等待 :param fix_count: 是否必须满足总数要求，发生超时，为True返回False，为False返回已捕捉到的数据包 :return: count为1时返回数据包对象，大于1时返回列表，超时且fix_count为True时返回False""" - return self._driver.listener.wait(count, timeout, fix_count) + return self._driver.listen.wait(count, timeout, fix_count) def _change(self, arg, text, exclude=False, timeout=None, raise_err=None): """等待指定属性变成包含或不包含指定文本 @@ -309,7 +309,7 @@ class ChromiumElementWaiter(object): def covered(self, timeout=None, raise_err=None): """等待当前元素被遮盖 - :param timeout:超时时间，为None使用元素所在页面timeout属性 + :param timeout: 超时时间，为None使用元素所在页面timeout属性 :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ @@ -317,7 +317,7 @@ class ChromiumElementWaiter(object): def not_covered(self, timeout=None, raise_err=None): """等待当前元素被遮盖 - :param timeout:超时时间，为None使用元素所在页面timeout属性 + :param timeout: 超时时间，为None使用元素所在页面timeout属性 :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ @@ -325,7 +325,7 @@ class ChromiumElementWaiter(object): def enabled(self, timeout=None, raise_err=None): """等待当前元素变成可用 - :param timeout:超时时间，为None使用元素所在页面timeout属性 + :param timeout: 超时时间，为None使用元素所在页面timeout属性 :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ @@ -333,7 +333,7 @@ class ChromiumElementWaiter(object): def disabled(self, timeout=None, raise_err=None): """等待当前元素变成可用 - :param timeout:超时时间，为None使用元素所在页面timeout属性 + :param timeout: 超时时间，为None使用元素所在页面timeout属性 :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ @@ -341,7 +341,7 @@ class ChromiumElementWaiter(object): def disabled_or_delete(self, timeout=None, raise_err=None): """等待当前元素变成不可用或从DOM移除 - :param timeout:超时时间，为None使用元素所在页面timeout属性 + :param timeout: 超时时间，为None使用元素所在页面timeout属性 :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ @@ -358,6 +358,38 @@ class ChromiumElementWaiter(object): else: return False + def stop_moving(self, gap=.1, timeout=None, raise_err=None): + """等待当前元素停止运动 + :param gap: 检测间隔时间 + :param timeout: 超时时间，为None使用元素所在页面timeout属性 + :param raise_err: 等待失败时是否报错，为None时根据Settings设置 + :return: 是否等待成功 + """ + if timeout is None: + timeout = self._page.timeout + end_time = perf_counter() + timeout + while perf_counter() < end_time: + try: + size = self._ele.states.has_rect + location = self._ele.location + break + except NoRectError: + pass + else: + raise NoRectError + + while perf_counter() < end_time: + sleep(gap) + if self._ele.size == size and location == self._ele.location: + return True + size = self._ele.size + location = self._ele.location + + if raise_err is True or Settings.raise_when_wait_failed is True: + raise WaitTimeoutError('等待元素停止运动失败。') + else: + return False + def _wait_state(self, attr, mode=False, timeout=None, raise_err=None): """等待元素某个bool状态到达指定状态 :param attr: 状态名称 diff --git a/DrissionPage/_units/waiter.pyi b/DrissionPage/_units/waiter.pyi index d8ad640..3bbd19c 100644 --- a/DrissionPage/_units/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -92,6 +92,8 @@ class ChromiumElementWaiter(object): def disabled_or_delete(self, timeout: float = None, raise_err: bool = None) -> bool: ... + def stop_moving(self, gap: float = .1, timeout: float = None, raise_err: bool = None) -> bool: ... + def _wait_state(self, attr: str, mode: bool = False, timeout: float = None, raise_err: bool = None) -> bool: ... diff --git a/DrissionPage/common.py b/DrissionPage/common.py index 2c67c17..1dab5d2 100644 --- a/DrissionPage/common.py +++ b/DrissionPage/common.py @@ -2,13 +2,10 @@ """ @Author : g1879 @Contact : g1879@qq.com -实用工具 """ -from FlowViewer import Listener, RequestMan - from ._elements.session_element import make_session_ele - from ._units.action_chains import ActionChains from ._commons.keys import Keys from ._commons.by import By from ._commons.constants import Settings +from ._commons.tools import wait_until From 301569a9cb6db0c4dade6c2d720bdd9db2877619 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 24 Oct 2023 16:54:35 +0800 Subject: [PATCH 049/182] =?UTF-8?q?=E5=AE=8C=E5=96=84=E7=9B=91=E5=90=AC?= =?UTF-8?q?=E5=99=A8=E9=80=BB=E8=BE=91=EF=BC=9B=E4=BF=AE=E5=A4=8Dto=5Ftab(?= =?UTF-8?q?)=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 8 +++-- DrissionPage/_pages/chromium_base.pyi | 2 +- DrissionPage/_pages/chromium_frame.py | 3 +- DrissionPage/_pages/chromium_page.py | 10 ++++-- DrissionPage/_units/network_listener.py | 25 +++++++++---- DrissionPage/_units/network_listener.pyi | 46 ++++++++++++++++++------ README.md | 12 +++---- 7 files changed, 71 insertions(+), 35 deletions(-) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 35d93c5..d8df4e9 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -93,14 +93,15 @@ class ChromiumBase(BasePage): self._get_document() self._first_run = False - def _driver_init(self, tab_id): + def _driver_init(self, tab_id, is_init=True): """新建页面、页面刷新、切换标签页后要进行的cdp参数初始化 :param tab_id: 要跳转到的标签页id + :param is_init: 是否初始化时执行本方法，用于判断是否to_tab()调用 :return: None """ self._is_loading = True - if hasattr(self, '_driver'): - return + if is_init and hasattr(self, '_driver'): + return # ChromiumPage接收ChromiumDriver方式启动时 self._driver = ChromiumDriver(tab_id=tab_id, tab_type='page', address=self.address) self._driver.call_method('DOM.enable') @@ -379,6 +380,7 @@ class ChromiumBase(BasePage): self.wait.load_complete() if self._scroll is None: self._scroll = ChromiumPageScroll(self) + self.set.scroll.smooth(False) return self._scroll @property diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 447e86c..f036256 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -50,7 +50,7 @@ class ChromiumBase(BasePage): def _connect_browser(self, tab_id: str = None) -> None: ... - def _driver_init(self, tab_id: str) -> None: ... + def _driver_init(self, tab_id: str, is_init:bool=True) -> None: ... def _get_document(self) -> None: ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index f83f322..d7fb0ed 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -81,9 +81,8 @@ class ChromiumFrame(ChromiumBase): self.retry_interval = self._target_page.retry_interval self._page_load_strategy = self._target_page.page_load_strategy self._download_path = self._target_page.download_path - # self._when_download_file_exists = self._target_page._when_download_file_exists - def _driver_init(self, tab_id): + def _driver_init(self, tab_id, is_init=True): """避免出现服务器500错误 :param tab_id: 要跳转到的标签页id :return: None diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 6c6ba37..6fd1026 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -17,6 +17,7 @@ from .._pages.chromium_tab import ChromiumTab from .._units.setter import ChromiumPageSetter from .._units.tab_rect import ChromiumTabRect from .._units.waiter import ChromiumPageWaiter +from ..errors import BrowserConnectError class ChromiumPage(ChromiumBase): @@ -65,8 +66,11 @@ class ChromiumPage(ChromiumBase): """连接浏览器""" connect_browser(self._driver_options) ws = get(f'http://{self._driver_options.debugger_address}/json/version', - headers={'Connection': 'close'}).json()['webSocketDebuggerUrl'] - self._browser = Browser(self._driver_options.debugger_address, ws.split('/')[-1], self) + headers={'Connection': 'close'}) + if not ws: + raise BrowserConnectError('\n浏览器连接失败，请检查是否启用全局代理。如有，须开放127.0.0.1地址。') + ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] + self._browser = Browser(self._driver_options.debugger_address, ws, self) def _d_set_runtime_settings(self): """设置运行时用到的属性""" @@ -230,7 +234,7 @@ class ChromiumPage(ChromiumBase): return self.driver.stop() - self._driver_init(tab_id) + self._driver_init(tab_id, False) if read_doc and self.ready_state in ('complete', None): self._get_document() diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index 6376ee1..e36954c 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -23,8 +23,7 @@ class NetworkListener(object): :param page: ChromiumBase对象 """ self._page = page - self._driver = ChromiumDriver(page.tab_id, 'page', page.address) - self._driver.call_method('Network.enable') + self._driver = None self._caught = None # 临存捕捉到的数据 self._request_ids = None # 暂存须要拦截的请求id @@ -75,6 +74,11 @@ class NetworkListener(object): """ if targets: self.set_targets(targets, is_regex, method) + if self.listening: + return + + self._driver = ChromiumDriver(self._page.tab_id, 'page', self._page.address) + self._driver.call_method('Network.enable') self.listening = True self._request_ids = {} @@ -118,10 +122,10 @@ class NetworkListener(object): return [self._caught.get_nowait() for _ in range(count)] def steps(self, count=None, timeout=None, gap=1): - """用于单步操作，可实现没收到若干个数据包执行一步操作（如翻页） + """用于单步操作，可实现每收到若干个数据包执行一步操作（如翻页） :param count: 需捕获的数据包总数，为None表示无限 :param timeout: 每个数据包等待时间，为None表示无限 - :param gap: 每接收到多少个数据包触发 + :param gap: 每接收到多少个数据包返回一次数据 :return: 用于在接收到监听目标时触发动作的可迭代对象 """ caught = 0 @@ -144,6 +148,8 @@ class NetworkListener(object): if self.listening: self.pause() self.clear() + self._driver.stop() + self._driver = None def pause(self, clear=True): """暂停监听 @@ -181,7 +187,7 @@ class NetworkListener(object): def _requestWillBeSent(self, **kwargs): """接收到请求时的回调函数""" if not self._targets: - self._request_ids[kwargs['requestId']] = DataPacket(self._driver.id, None, kwargs) + self._request_ids[kwargs['requestId']] = DataPacket(self._page.tab_id, None, kwargs) if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): self._request_ids[kwargs['requestId']]._raw_post_data = \ self._driver.call_method('Network.getRequestPostData', requestId=kwargs['requestId'])['postData'] @@ -192,7 +198,7 @@ class NetworkListener(object): if ((self._is_regex and search(target, kwargs['request']['url'])) or (not self._is_regex and target in kwargs['request']['url'])) and ( not self._method or kwargs['request']['method'] in self._method): - self._request_ids[kwargs['requestId']] = DataPacket(self._driver.id, target, kwargs) + self._request_ids[kwargs['requestId']] = DataPacket(self._page.tab_id, target, kwargs) if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): self._request_ids[kwargs['requestId']]._raw_post_data = \ @@ -253,7 +259,7 @@ class DataPacket(object): :param target: 监听目标 :param raw_request: 原始request数据，从cdp获得 """ - self.tab = tab_id + self.tab_id = tab_id self.target = target self._raw_request = raw_request @@ -353,6 +359,11 @@ class Response(object): self._headers = CaseInsensitiveDict(self._response['headers']) return self._headers + @property + def raw_body(self): + """返回未被处理的body文本""" + return self._raw_body + @property def body(self): """返回body内容，如果是json格式，自动进行转换，如果时图片格式，进行base64转换，其它格式直接返回文本""" diff --git a/DrissionPage/_units/network_listener.pyi b/DrissionPage/_units/network_listener.pyi index 3b85ad4..bb26f99 100644 --- a/DrissionPage/_units/network_listener.pyi +++ b/DrissionPage/_units/network_listener.pyi @@ -100,14 +100,15 @@ class Request(object): _headers: Union[CaseInsensitiveDict, None] = ... method: str = ... - # urlFragment: str = ... - # postDataEntries: list = ... - # mixedContentType: str = ... - # initialPriority: str = ... - # referrerPolicy: str = ... - # isLinkPreload: bool = ... - # trustTokenParams: dict = ... - # isSameSite: bool = ... + urlFragment = ... + hasPostData = ... + postDataEntries = ... + mixedContentType = ... + initialPriority = ... + referrerPolicy = ... + isLinkPreload = ... + trustTokenParams = ... + isSameSite = ... def __init__(self, raw_request: dict, post_data: str): self._request: dict = ... @@ -122,9 +123,29 @@ class Request(object): class Response(object): - status: str = ... - statusText: int = ... - mimeType: str = ... + url = ... + status = ... + statusText = ... + headersText = ... + mimeType = ... + requestHeaders = ... + requestHeadersText = ... + connectionReused = ... + connectionId = ... + remoteIPAddress = ... + remotePort = ... + fromDiskCache = ... + fromServiceWorker = ... + fromPrefetchCache = ... + encodedDataLength = ... + timing = ... + serviceWorkerResponseSource = ... + responseTime = ... + cacheStorageCacheName = ... + protocol = ... + alternateProtocolUsage = ... + securityState = ... + securityDetails = ... def __init__(self, raw_response: dict, raw_body: str, base64_body: bool): self._response: dict = ... @@ -136,5 +157,8 @@ class Response(object): @property def headers(self) -> CaseInsensitiveDict: ... + @property + def raw_body(self) -> str: ... + @property def body(self) -> Union[str, dict, bool]: ... diff --git a/README.md b/README.md index 0c1b297..8d5fe9e 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ python 版本：3.6 及以上 --- +# 🛠 如何使用 + **📖 使用文档：** [点击查看](http://g1879.gitee.io/drissionpagedocs) **交流 QQ 群：** 897838127[已满]、558778073 @@ -36,7 +38,7 @@ python 版本：3.6 及以上 # 🔥 新版预告 -查看下一步开发计划：[新版预告](http://g1879.gitee.io/drissionpagedocs/whatsnew/3_3/) +查看下一步开发计划：[新版预告](https://g1879.gitee.io/drissionpagedocs/whatsnew/3_3/) --- @@ -110,15 +112,9 @@ python 版本：3.6 及以上 --- -# 🛠 使用文档 - -[点击跳转到使用文档](http://g1879.gitee.io/drissionpage) - ---- - # 🔖 版本历史 -[点击查看版本历史](http://g1879.gitee.io/drissionpagedocs/history/3.x/) +[点击查看版本历史](https://g1879.gitee.io/drissionpagedocs/history/) --- From d587ca6095e80cadc83c28b4bddc369325c5b5aa Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 24 Oct 2023 17:45:20 +0800 Subject: [PATCH 050/182] =?UTF-8?q?=E4=BC=98=E5=8C=96ChromiumDriver?= =?UTF-8?q?=E8=B6=85=E6=97=B6=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/chromium_driver.py | 20 +++++++------------- DrissionPage/_elements/chromium_element.py | 4 ++-- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index 94a9664..4985a98 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -6,6 +6,7 @@ from json import dumps, loads from queue import Queue, Empty from threading import Thread, Event +from time import perf_counter from requests import get from websocket import WebSocketTimeoutException, WebSocketException, WebSocketConnectionClosedException, \ @@ -45,7 +46,7 @@ class ChromiumDriver(object): def _send(self, message, timeout=None): """发送信息到浏览器，并返回浏览器返回的信息 :param message: 发送给浏览器的数据 - :param timeout: 超时时间 + :param timeout: 超时时间，为None表示无限 :return: 浏览器返回的数据 """ if 'id' not in message: @@ -64,10 +65,8 @@ class ChromiumDriver(object): print(f'发> {message_json}') break - if not isinstance(timeout, (int, float)) or timeout > 1: - q_timeout = 1 - else: - q_timeout = timeout / 2.0 + if timeout is not None: + timeout = perf_counter() + timeout try: self.method_results[message['id']] = Queue() @@ -75,19 +74,14 @@ class ChromiumDriver(object): while not self._stopped.is_set(): try: - if isinstance(timeout, (int, float)): - if timeout < q_timeout: - q_timeout = timeout - timeout -= q_timeout - - return self.method_results[message['id']].get(timeout=q_timeout) + return self.method_results[message['id']].get_nowait() except Empty: if self.has_alert: return {'error': {'message': 'alert exists'}, 'type': 'alert_exists'} - if isinstance(timeout, (int, float)) and timeout <= 0: - raise TimeoutError(f"调用{message['method']}超时。") + if timeout is not None and perf_counter() > timeout: + return {'error': {'message': 'timeout'}} continue diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 07176bf..260eba3 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -1306,7 +1306,7 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): try: if as_expr: res = page.run_cdp('Runtime.evaluate', expression=script, returnByValue=False, - awaitPromise=True, userGesture=True, timeout=timeout * 1000) + awaitPromise=True, userGesture=True, _timeout=timeout) else: args = args or () @@ -1314,7 +1314,7 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): script = f'function(){{{script}}}' res = page.run_cdp('Runtime.callFunctionOn', functionDeclaration=script, objectId=obj_id, arguments=[convert_argument(arg) for arg in args], returnByValue=False, - awaitPromise=True, userGesture=True) + awaitPromise=True, userGesture=True, _timeout=timeout) except ContextLossError: if is_page: From 2939e4d42b33851e346908c9be020e52dc9ca5e6 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 24 Oct 2023 23:50:16 +0800 Subject: [PATCH 051/182] =?UTF-8?q?Tab=E5=8F=AF=E5=A4=84=E7=90=86=E8=87=AA?= =?UTF-8?q?=E5=B7=B1=E7=9A=84alert=EF=BC=9B=E9=87=8D=E6=9E=84=E5=A4=84?= =?UTF-8?q?=E7=90=86alert=E9=80=BB=E8=BE=91=EF=BC=8Calert=E5=AD=98?= =?UTF-8?q?=E5=9C=A8=E6=97=B6=E4=B9=9F=E5=8F=AF=E5=A4=84=E7=90=86=E9=9D=9E?= =?UTF-8?q?js=E5=91=BD=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/chromium_driver.py | 12 ++-- DrissionPage/_base/chromium_driver.pyi | 2 +- DrissionPage/_commons/constants.py | 1 - DrissionPage/_elements/chromium_element.py | 13 +++- DrissionPage/_elements/chromium_element.pyi | 2 + DrissionPage/_pages/chromium_base.py | 69 ++++++++++++++++++++- DrissionPage/_pages/chromium_base.pyi | 24 ++++++- DrissionPage/_pages/chromium_page.py | 61 +----------------- DrissionPage/_pages/chromium_page.pyi | 18 ------ README.md | 4 +- 10 files changed, 115 insertions(+), 91 deletions(-) diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index 4985a98..50dae8a 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -24,7 +24,7 @@ class ChromiumDriver(object): self.address = address self.type = tab_type self._debug = False - self.has_alert = False + self.alert_flag = False # 标记alert出现，跳过一条请求后复原 self._websocket_url = f'ws://{address}/devtools/{tab_type}/{tab_id}' self._cur_id = 0 @@ -77,8 +77,9 @@ class ChromiumDriver(object): return self.method_results[message['id']].get_nowait() except Empty: - if self.has_alert: - return {'error': {'message': 'alert exists'}, 'type': 'alert_exists'} + if self.alert_flag: + self.alert_flag = False + return {'result': []} if timeout is not None and perf_counter() > timeout: return {'error': {'message': 'timeout'}} @@ -114,7 +115,10 @@ class ChromiumDriver(object): print(f'<收 {msg_json}') break - if "method" in msg: + if 'method' in msg: + if msg['method'].startswith('Page.javascriptDialog'): + self.alert_flag = msg['method'].endswith('Opening') + self.event_queue.put(msg) elif msg.get('id') in self.method_results: diff --git a/DrissionPage/_base/chromium_driver.pyi b/DrissionPage/_base/chromium_driver.pyi index aeba4f9..378879b 100644 --- a/DrissionPage/_base/chromium_driver.pyi +++ b/DrissionPage/_base/chromium_driver.pyi @@ -23,7 +23,7 @@ class ChromiumDriver(object): address: str type: str _debug: bool - has_alert: bool + alert_flag: bool _websocket_url: str _cur_id: int _ws: Optional[WebSocket] diff --git a/DrissionPage/_commons/constants.py b/DrissionPage/_commons/constants.py index 9192448..a1c8054 100644 --- a/DrissionPage/_commons/constants.py +++ b/DrissionPage/_commons/constants.py @@ -5,7 +5,6 @@ """ from ..errors import ElementNotFoundError -HANDLE_ALERT_METHOD = 'Page.handleJavaScriptDialog' FRAME_ELEMENT = ('iframe', 'frame') ERROR = 'error' diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 260eba3..68ee961 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -18,7 +18,7 @@ from .._units.clicker import Clicker from .._units.setter import ChromiumElementSetter from .._units.waiter import ChromiumElementWaiter from ..errors import ContextLossError, ElementLossError, JavaScriptError, ElementNotFoundError, \ - CDPError, NoResourceError, NoRectError + CDPError, NoResourceError, NoRectError, AlertExistsError class ChromiumElement(DrissionElement): @@ -206,6 +206,14 @@ class ChromiumElement(DrissionElement): return self._select + def check(self, uncheck=False): + """选中或取消选中当前元素 + :param uncheck: 是否取消选中 + :return: None + """ + js = 'this.checked=false' if uncheck else 'this.checked=true' + self.run_js(js) + def parent(self, level_or_loc=1, index=1): """返回上面某一级父元素，可指定层数或用查询语法定位 :param level_or_loc: 第几级父元素，或定位符 @@ -1303,6 +1311,9 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): obj_id = page_or_ele._root_id is_page = True + if page.has_alert: + raise AlertExistsError + try: if as_expr: res = page.run_cdp('Runtime.evaluate', expression=script, returnByValue=False, diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index b10dd0a..3f584a3 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -150,6 +150,8 @@ class ChromiumElement(DrissionElement): @property def select(self) -> ChromiumSelect: ... + def check(self, uncheck: bool = False) -> None: ... + def attr(self, attr: str) -> Union[str, None]: ... def remove_attr(self, attr: str) -> None: ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index d8df4e9..5595dac 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -15,7 +15,7 @@ from requests import get from .._base.base import BasePage from .._base.chromium_driver import ChromiumDriver -from .._commons.constants import HANDLE_ALERT_METHOD, ERROR, NoneElement +from .._commons.constants import ERROR, NoneElement from .._commons.locator import get_loc from .._commons.tools import get_usable_path, clean_folder from .._commons.web import location_in_viewport @@ -47,6 +47,7 @@ class ChromiumBase(BasePage): self._screencast = None self._actions = None self._listener = None + self._has_alert = False self._download_path = str(Path('../..').absolute()) @@ -103,6 +104,9 @@ class ChromiumBase(BasePage): if is_init and hasattr(self, '_driver'): return # ChromiumPage接收ChromiumDriver方式启动时 self._driver = ChromiumDriver(tab_id=tab_id, tab_type='page', address=self.address) + self._alert = Alert() + self._driver.set_listener('Page.javascriptDialogOpening', self._on_alert_open) + self._driver.set_listener('Page.javascriptDialogClosed', self._on_alert_close) self._driver.call_method('DOM.enable') self._driver.call_method('Page.enable') @@ -429,14 +433,19 @@ class ChromiumBase(BasePage): self._listener = NetworkListener(self) return self._listener + @property + def has_alert(self): + """返回是否存在提示框""" + return self._has_alert + def run_cdp(self, cmd, **cmd_args): """执行Chrome DevTools Protocol语句 :param cmd: 协议项目 :param cmd_args: 参数 :return: 执行的结果 """ - if self.driver.has_alert and cmd != HANDLE_ALERT_METHOD: - raise AlertExistsError + # if self.driver.has_alert and cmd != HANDLE_ALERT_METHOD: + # raise AlertExistsError r = self.driver.call_method(cmd, **cmd_args) if ERROR not in r: @@ -824,6 +833,48 @@ class ChromiumBase(BasePage): if cookies: self.run_cdp_loaded('Network.clearBrowserCookies') + def handle_alert(self, accept=True, send=None, timeout=None): + """处理提示框，可以自动等待提示框出现 + :param accept: True表示确认，False表示取消，其它值不会按按钮但依然返回文本值 + :param send: 处理prompt提示框时可输入文本 + :param timeout: 等待提示框出现的超时时间，为None则使用self.timeout属性的值 + :return: 提示框内容文本，未等到提示框则返回False + """ + timeout = self.timeout if timeout is None else timeout + timeout = .1 if timeout <= 0 else timeout + end_time = perf_counter() + timeout + while not self._alert.activated and perf_counter() < end_time: + sleep(.1) + if not self._alert.activated: + return False + + res_text = self._alert.text + if self._alert.type == 'prompt': + self.driver.call_method('Page.handleJavaScriptDialog', accept=accept, promptText=send) + else: + self.driver.call_method('Page.handleJavaScriptDialog', accept=accept) + return res_text + + def _on_alert_close(self, **kwargs): + """alert关闭时触发的方法""" + self._alert.activated = False + self._alert.text = None + self._alert.type = None + self._alert.defaultPrompt = None + self._alert.response_accept = kwargs.get('result') + self._alert.response_text = kwargs['userInput'] + self._has_alert = False + + def _on_alert_open(self, **kwargs): + """alert出现时触发的方法""" + self._alert.activated = True + self._alert.text = kwargs['message'] + self._alert.type = kwargs['message'] + self._alert.defaultPrompt = kwargs.get('defaultPrompt', None) + self._alert.response_accept = None + self._alert.response_text = None + self._has_alert = True + def _d_connect(self, to_url, times=0, interval=1, show_errmsg=False, timeout=None): """尝试连接，重试若干次 :param to_url: 要访问的url @@ -1166,3 +1217,15 @@ class ScreencastMode(object): def imgs_mode(self): self._screencast._mode = 'imgs' + + +class Alert(object): + """用于保存alert信息的类""" + + def __init__(self): + self.activated = False + self.text = None + self.type = None + self.defaultPrompt = None + self.response_accept = None + self.response_text = None diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index f036256..edabfa1 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -47,10 +47,12 @@ class ChromiumBase(BasePage): self._screencast: Screencast = ... self._actions: ActionChains = ... self._listener: NetworkListener = ... + self._alert: Alert = ... + self._has_alert: bool = ... def _connect_browser(self, tab_id: str = None) -> None: ... - def _driver_init(self, tab_id: str, is_init:bool=True) -> None: ... + def _driver_init(self, tab_id: str, is_init: bool = True) -> None: ... def _get_document(self) -> None: ... @@ -156,6 +158,9 @@ class ChromiumBase(BasePage): @property def listen(self) -> NetworkListener: ... + @property + def has_alert(self) -> bool: ... + def run_js(self, script: str, *args: Any, as_expr: bool = False) -> Any: ... def run_js_loaded(self, script: str, *args: Any, as_expr: bool = False) -> Any: ... @@ -226,6 +231,12 @@ class ChromiumBase(BasePage): cache: bool = True, cookies: bool = True) -> None: ... + def handle_alert(self, accept: bool = True, send: str = None, timeout: float = None) -> Union[str, False]: ... + + def _on_alert_close(self, **kwargs): ... + + def _on_alert_open(self, **kwargs): ... + def _d_connect(self, to_url: str, times: int = 0, @@ -286,3 +297,14 @@ class ScreencastMode(object): def frugal_imgs_mode(self) -> None: ... def imgs_mode(self) -> None: ... + + +class Alert(object): + + def __init__(self): + self.activated: bool = ... + self.text: str = ... + self.type: str = ... + self.defaultPrompt: str = ... + self.response_accept: str = ... + self.response_text: str = ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 6fd1026..98e67ef 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from pathlib import Path -from time import perf_counter, sleep +from time import sleep from requests import get @@ -85,13 +85,8 @@ class ChromiumPage(ChromiumBase): def _page_init(self): """浏览器相关设置""" - self._alert = Alert() - self._driver.set_listener('Page.javascriptDialogOpening', self._on_alert_open) - self._driver.set_listener('Page.javascriptDialogClosed', self._on_alert_close) - self._rect = None self._main_tab = self.tab_id - self._browser.connect_to_page() @property @@ -284,64 +279,10 @@ class ChromiumPage(ChromiumBase): """ self.close_tabs(tabs_or_ids, True) - def handle_alert(self, accept=True, send=None, timeout=None): - """处理提示框，可以自动等待提示框出现 - :param accept: True表示确认，False表示取消，其它值不会按按钮但依然返回文本值 - :param send: 处理prompt提示框时可输入文本 - :param timeout: 等待提示框出现的超时时间，为None则使用self.timeout属性的值 - :return: 提示框内容文本，未等到提示框则返回False - """ - timeout = self.timeout if timeout is None else timeout - timeout = .1 if timeout <= 0 else timeout - end_time = perf_counter() + timeout - while not self._alert.activated and perf_counter() < end_time: - sleep(.1) - if not self._alert.activated: - return False - - res_text = self._alert.text - if self._alert.type == 'prompt': - self.driver.call_method('Page.handleJavaScriptDialog', accept=accept, promptText=send) - else: - self.driver.call_method('Page.handleJavaScriptDialog', accept=accept) - return res_text - def quit(self): """关闭浏览器""" self.browser.quit() - def _on_alert_close(self, **kwargs): - """alert关闭时触发的方法""" - self._alert.activated = False - self._alert.text = None - self._alert.type = None - self._alert.defaultPrompt = None - self._alert.response_accept = kwargs.get('result') - self._alert.response_text = kwargs['userInput'] - self._driver.has_alert = False - - def _on_alert_open(self, **kwargs): - """alert出现时触发的方法""" - self._alert.activated = True - self._alert.text = kwargs['message'] - self._alert.type = kwargs['message'] - self._alert.defaultPrompt = kwargs.get('defaultPrompt', None) - self._alert.response_accept = None - self._alert.response_text = None - self._driver.has_alert = True - - -class Alert(object): - """用于保存alert信息的类""" - - def __init__(self): - self.activated = False - self.text = None - self.type = None - self.defaultPrompt = None - self.response_accept = None - self.response_text = None - def get_rename(original, rename): if '.' in rename: diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index 15b6714..bf82ca6 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -23,7 +23,6 @@ class ChromiumPage(ChromiumBase): timeout: float = None): self._driver_options: ChromiumOptions = ... self._main_tab: str = ... - self._alert: Alert = ... self._browser: Browser = ... self._rect: Optional[ChromiumTabRect] = ... @@ -83,24 +82,7 @@ class ChromiumPage(ChromiumBase): def close_other_tabs(self, tabs_or_ids: Union[ str, ChromiumTab, List[Union[str, ChromiumTab]], Tuple[Union[str, ChromiumTab]]] = None) -> None: ... - def handle_alert(self, accept: bool = True, send: str = None, timeout: float = None) -> Union[str, False]: ... - def quit(self) -> None: ... - def _on_alert_close(self, **kwargs): ... - - def _on_alert_open(self, **kwargs): ... - - -class Alert(object): - - def __init__(self): - self.activated: bool = ... - self.text: str = ... - self.type: str = ... - self.defaultPrompt: str = ... - self.response_accept: str = ... - self.response_text: str = ... - def get_rename(original: str, rename: str) -> str: ... diff --git a/README.md b/README.md index 8d5fe9e..49e9d7f 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,9 @@ python 版本：3.6 及以上 # 🛠 如何使用 -**📖 使用文档：** [点击查看](http://g1879.gitee.io/drissionpagedocs) +**📖 使用文档：** [点击查看](https://g1879.gitee.io/drissionpagedocs) -**交流 QQ 群：** 897838127[已满]、558778073 +**交流 QQ 群：** 897838127[已满]、558778073[已满]、636361957 --- From ab1f85d192a2bf9abbe3eaa5c68a4145bc77f26b Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 25 Oct 2023 20:00:28 +0800 Subject: [PATCH 052/182] =?UTF-8?q?auto=5Fport()=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=A4=9A=E7=BA=BF=E7=A8=8B=EF=BC=9B=E6=94=AF=E6=8C=81=E6=9F=A5?= =?UTF-8?q?=E6=89=BE=E7=94=A8=E5=AE=8C=E7=AB=AF=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/chromium_driver.py | 3 ++- DrissionPage/_configs/chromium_options.py | 26 ++++++++++++++++------ DrissionPage/_configs/chromium_options.pyi | 2 +- DrissionPage/_pages/chromium_base.py | 1 - 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index 50dae8a..c593222 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -6,7 +6,7 @@ from json import dumps, loads from queue import Queue, Empty from threading import Thread, Event -from time import perf_counter +from time import perf_counter, sleep from requests import get from websocket import WebSocketTimeoutException, WebSocketException, WebSocketConnectionClosedException, \ @@ -84,6 +84,7 @@ class ChromiumDriver(object): if timeout is not None and perf_counter() > timeout: return {'error': {'message': 'timeout'}} + sleep(.02) continue except Exception: diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index 67dfe89..1ce072f 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -4,7 +4,9 @@ @Contact : g1879@qq.com """ from pathlib import Path +from shutil import rmtree from tempfile import gettempdir, TemporaryDirectory +from threading import Lock from .options_manage import OptionsManager from .._commons.tools import port_is_using, clean_folder @@ -471,24 +473,34 @@ class ChromiumOptions(object): class PortFinder(object): - used_port = [] + used_port = {} def __init__(self): self.tmp_dir = Path(gettempdir()) / 'DrissionPage' / 'TempFolder' self.tmp_dir.mkdir(parents=True, exist_ok=True) if not PortFinder.used_port: clean_folder(self.tmp_dir) + self._lock = Lock() def get_port(self): """查找一个可用端口 :return: 可以使用的端口和用户文件夹路径组成的元组 """ - for i in range(9600, 19800): - if i in PortFinder.used_port or port_is_using('127.0.0.1', i): - continue + with self._lock: + for i in range(9600, 19600): + if i in PortFinder.used_port: + continue + elif port_is_using('127.0.0.1', i): + PortFinder.used_port[i] = None + continue + path = TemporaryDirectory(dir=self.tmp_dir).name + PortFinder.used_port[i] = path + return i, path - path = TemporaryDirectory(dir=self.tmp_dir) - PortFinder.used_port.append(i) - return i, path.name + for i in range(9600, 19600): + if port_is_using('127.0.0.1', i): + continue + rmtree(PortFinder.used_port[i], ignore_errors=True) + return i, TemporaryDirectory(dir=self.tmp_dir).name raise OSError('未找到可用端口。') diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index 17d1906..161ff85 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -127,7 +127,7 @@ class ChromiumOptions(object): class PortFinder(object): - used_port: list = ... + used_port: dict = ... @staticmethod def get_port() -> Tuple[int, str]: ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 5595dac..0b3a5e2 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -384,7 +384,6 @@ class ChromiumBase(BasePage): self.wait.load_complete() if self._scroll is None: self._scroll = ChromiumPageScroll(self) - self.set.scroll.smooth(False) return self._scroll @property From cd1369e33a9239a4016026d866e1b50a288d4a4c Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 26 Oct 2023 21:40:06 +0800 Subject: [PATCH 053/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=9B=91=E5=90=AC?= =?UTF-8?q?=E5=99=A8=E5=B0=8Fbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 167 +---------------------- DrissionPage/_pages/chromium_base.pyi | 38 +----- DrissionPage/_units/network_listener.py | 18 +-- DrissionPage/_units/screencast.py | 172 ++++++++++++++++++++++++ DrissionPage/_units/screencast.pyi | 46 +++++++ 5 files changed, 229 insertions(+), 212 deletions(-) create mode 100644 DrissionPage/_units/screencast.py create mode 100644 DrissionPage/_units/screencast.pyi diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 0b3a5e2..1159b17 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -3,13 +3,11 @@ @Author : g1879 @Contact : g1879@qq.com """ -from base64 import b64decode from json import loads, JSONDecodeError from os.path import sep from pathlib import Path from re import findall -from threading import Thread -from time import perf_counter, sleep, time +from time import perf_counter, sleep from requests import get @@ -17,12 +15,13 @@ from .._base.base import BasePage from .._base.chromium_driver import ChromiumDriver from .._commons.constants import ERROR, NoneElement from .._commons.locator import get_loc -from .._commons.tools import get_usable_path, clean_folder +from .._commons.tools import get_usable_path from .._commons.web import location_in_viewport from .._elements.chromium_element import ChromiumScroll, ChromiumElement, run_js, make_chromium_ele from .._elements.session_element import make_session_ele from .._units.action_chains import ActionChains from .._units.network_listener import NetworkListener +from .._units.screencast import Screencast from .._units.setter import ChromiumBaseSetter from .._units.waiter import ChromiumBaseWaiter from ..errors import ContextLossError, ElementLossError, AlertExistsError, CDPError, TabClosedError, \ @@ -1058,166 +1057,6 @@ class Timeout(object): return str({'implicit': self.implicit, 'page_load': self.page_load, 'script': self.script}) -class Screencast(object): - def __init__(self, page): - self._page = page - self._path = None - self._running = False - self._enable = False - self._mode = 'video' - - @property - def set_mode(self): - """返回用于设置录屏幕式的对象""" - return ScreencastMode(self) - - def start(self, save_path=None): - """开始录屏 - :param save_path: 录屏保存位置 - :return: None - """ - self.set_save_path(save_path) - if self._path is None: - raise ValueError('save_path必须设置。') - clean_folder(self._path) - if self._mode.startswith('frugal'): - self._page.driver.set_listener('Page.screencastFrame', self._onScreencastFrame) - self._page.run_cdp('Page.startScreencast', everyNthFrame=1, quality=100) - - elif not self._mode.startswith('js'): - self._running = True - self._enable = True - Thread(target=self._run).start() - - else: - js = ''' - async function () { - stream = await navigator.mediaDevices.getDisplayMedia({video: true, audio: true}) - mime = MediaRecorder.isTypeSupported("video/webm; codecs=vp9") - ? "video/webm; codecs=vp9" - : "video/webm" - mediaRecorder = new MediaRecorder(stream, {mimeType: mime}) - DrissionPage_Screencast_chunks = [] - mediaRecorder.addEventListener('dataavailable', function(e) { - DrissionPage_Screencast_blob_ok = false; - DrissionPage_Screencast_chunks.push(e.data); - DrissionPage_Screencast_blob_ok = true; - }) - mediaRecorder.start() - - mediaRecorder.addEventListener('stop', function(){ - while(DrissionPage_Screencast_blob_ok==false){} - DrissionPage_Screencast_blob = new Blob(DrissionPage_Screencast_chunks, - {type: DrissionPage_Screencast_chunks[0].type}); - }) - } - ''' - print('请手动选择要录制的目标。') - self._page.run_js('var DrissionPage_Screencast_blob;var DrissionPage_Screencast_blob_ok=false;') - self._page.run_js(js) - - def stop(self, video_name=None): - """停止录屏 - :param video_name: 视频文件名，为None时以当前时间名命 - :return: 文件路径 - """ - if video_name and not video_name.endswith('mp4'): - video_name = f'{video_name}.mp4' - name = f'{time()}.mp4' if not video_name else video_name - path = f'{self._path}{sep}{name}' - - if self._mode.startswith('js'): - self._page.run_js('mediaRecorder.stop();', as_expr=True) - while not self._page.run_js('return DrissionPage_Screencast_blob_ok;'): - sleep(.1) - blob = self._page.run_js('return DrissionPage_Screencast_blob;') - uuid = self._page.run_cdp('IO.resolveBlob', objectId=blob['result']['objectId'])['uuid'] - data = self._page.run_cdp('IO.read', handle=f'blob:{uuid}')['data'] - with open(path, 'wb') as f: - f.write(b64decode(data)) - return path - - if self._mode.startswith('frugal'): - self._page.driver.set_listener('Page.screencastFrame', None) - self._page.run_cdp('Page.stopScreencast') - else: - self._enable = False - while self._running: - sleep(.1) - - if self._mode.endswith('imgs'): - return str(Path(self._path).absolute()) - - if not str(video_name).isascii() or not str(self._path).isascii(): - raise TypeError('转换成视频仅支持英文路径和文件名。') - - try: - from cv2 import VideoWriter, imread, VideoWriter_fourcc - from numpy import fromfile, uint8 - except ModuleNotFoundError: - raise ModuleNotFoundError('请先安装cv2，pip install opencv-python') - - pic_list = Path(self._path).glob('*.jpg') - img = imread(str(next(pic_list))) - imgInfo = img.shape - size = (imgInfo[1], imgInfo[0]) - - videoWrite = VideoWriter(path, VideoWriter_fourcc(*"mp4v"), 5, size) - - for i in pic_list: - img = imread(str(i)) - videoWrite.write(img) - - clean_folder(self._path, ignore=(name,)) - return f'{self._path}{sep}{name}' - - def set_save_path(self, save_path=None): - """设置保存路径 - :param save_path: 保存路径 - :return: None - """ - if save_path: - save_path = Path(save_path) - if save_path.exists() and save_path.is_file(): - raise TypeError('save_path必须指定文件夹。') - save_path.mkdir(parents=True, exist_ok=True) - self._path = save_path - - def _run(self): - """非节俭模式运行方法""" - self._running = True - while self._enable: - self._page.get_screenshot(path=self._path, name=f'{time()}.jpg') - sleep(.04) - self._running = False - - def _onScreencastFrame(self, **kwargs): - """节俭模式运行方法""" - with open(f'{self._path}\\{kwargs["metadata"]["timestamp"]}.jpg', 'wb') as f: - f.write(b64decode(kwargs['data'])) - self._page.run_cdp('Page.screencastFrameAck', sessionId=kwargs['sessionId']) - - -class ScreencastMode(object): - def __init__(self, screencast): - self._screencast = screencast - - def video_mode(self): - self._screencast._mode = 'video' - - def frugal_video_mode(self): - self._screencast._mode = 'frugal_video' - - def js_video_mode(self): - self._screencast._mode = 'js_video' - - def frugal_imgs_mode(self): - self._screencast._mode = 'frugal_imgs' - - def imgs_mode(self): - self._screencast._mode = 'imgs' - - class Alert(object): """用于保存alert信息的类""" diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index edabfa1..4ff2964 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -18,6 +18,7 @@ from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage from .._units.action_chains import ActionChains from .._units.network_listener import NetworkListener +from .._units.screencast import Screencast from .._units.setter import ChromiumBaseSetter from .._units.waiter import ChromiumBaseWaiter @@ -262,43 +263,6 @@ class Timeout(object): self.script: float = ... -class Screencast(object): - def __init__(self, page: ChromiumBase): - self._page: ChromiumBase = ... - self._path: Path = ... - self._running: bool = ... - self._enable: bool = ... - self._mode: str = ... - - @property - def set_mode(self) -> ScreencastMode: ... - - def start(self, save_path: Union[str, Path] = None) -> None: ... - - def stop(self, video_name: str = None) -> str: ... - - def set_save_path(self, save_path: Union[str, Path] = None) -> None: ... - - def _run(self) -> None: ... - - def _onScreencastFrame(self, **kwargs) -> None: ... - - -class ScreencastMode(object): - def __init__(self, screencast: Screencast): - self._screencast: Screencast = ... - - def video_mode(self) -> None: ... - - def frugal_video_mode(self) -> None: ... - - def js_video_mode(self) -> None: ... - - def frugal_imgs_mode(self) -> None: ... - - def imgs_mode(self) -> None: ... - - class Alert(object): def __init__(self): diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index e36954c..a82c5f9 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -12,7 +12,6 @@ from time import perf_counter, sleep from requests.structures import CaseInsensitiveDict from .._base.chromium_driver import ChromiumDriver -from ..errors import CDPError class NetworkListener(object): @@ -218,16 +217,13 @@ class NetworkListener(object): request_id = kwargs['requestId'] dp = self._request_ids.get(request_id) if dp: - try: - r = self._driver.call_method('Network.getResponseBody', requestId=request_id) - body = r['body'] - is_base64 = r['base64Encoded'] - except CDPError: - body = '' - is_base64 = False - - dp._raw_body = body - dp._base64_body = is_base64 + r = self._driver.call_method('Network.getResponseBody', requestId=request_id) + if 'body' in r: + dp._raw_body = r['body'] + dp._base64_body = r['base64Encoded'] + else: + dp._raw_body = '' + dp._base64_body = False self._caught.put(dp) try: diff --git a/DrissionPage/_units/screencast.py b/DrissionPage/_units/screencast.py new file mode 100644 index 0000000..1d25683 --- /dev/null +++ b/DrissionPage/_units/screencast.py @@ -0,0 +1,172 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" +from base64 import b64decode +from os.path import sep +from pathlib import Path +from threading import Thread +from time import sleep, time + +from .._commons.tools import clean_folder + + +class Screencast(object): + def __init__(self, page): + self._page = page + self._path = None + self._running = False + self._enable = False + self._mode = 'video' + + @property + def set_mode(self): + """返回用于设置录屏幕式的对象""" + return ScreencastMode(self) + + def start(self, save_path=None): + """开始录屏 + :param save_path: 录屏保存位置 + :return: None + """ + self.set_save_path(save_path) + if self._path is None: + raise ValueError('save_path必须设置。') + clean_folder(self._path) + if self._mode.startswith('frugal'): + self._page.driver.set_listener('Page.screencastFrame', self._onScreencastFrame) + self._page.run_cdp('Page.startScreencast', everyNthFrame=1, quality=100) + + elif not self._mode.startswith('js'): + self._running = True + self._enable = True + Thread(target=self._run).start() + + else: + js = ''' + async function () { + stream = await navigator.mediaDevices.getDisplayMedia({video: true, audio: true}) + mime = MediaRecorder.isTypeSupported("video/webm; codecs=vp9") + ? "video/webm; codecs=vp9" + : "video/webm" + mediaRecorder = new MediaRecorder(stream, {mimeType: mime}) + DrissionPage_Screencast_chunks = [] + mediaRecorder.addEventListener('dataavailable', function(e) { + DrissionPage_Screencast_blob_ok = false; + DrissionPage_Screencast_chunks.push(e.data); + DrissionPage_Screencast_blob_ok = true; + }) + mediaRecorder.start() + + mediaRecorder.addEventListener('stop', function(){ + while(DrissionPage_Screencast_blob_ok==false){} + DrissionPage_Screencast_blob = new Blob(DrissionPage_Screencast_chunks, + {type: DrissionPage_Screencast_chunks[0].type}); + }) + } + ''' + print('请手动选择要录制的目标。') + self._page.run_js('var DrissionPage_Screencast_blob;var DrissionPage_Screencast_blob_ok=false;') + self._page.run_js(js) + + def stop(self, video_name=None): + """停止录屏 + :param video_name: 视频文件名，为None时以当前时间名命 + :return: 文件路径 + """ + if video_name and not video_name.endswith('mp4'): + video_name = f'{video_name}.mp4' + name = f'{time()}.mp4' if not video_name else video_name + path = f'{self._path}{sep}{name}' + + if self._mode.startswith('js'): + self._page.run_js('mediaRecorder.stop();', as_expr=True) + while not self._page.run_js('return DrissionPage_Screencast_blob_ok;'): + sleep(.1) + blob = self._page.run_js('return DrissionPage_Screencast_blob;') + uuid = self._page.run_cdp('IO.resolveBlob', objectId=blob['result']['objectId'])['uuid'] + data = self._page.run_cdp('IO.read', handle=f'blob:{uuid}')['data'] + with open(path, 'wb') as f: + f.write(b64decode(data)) + return path + + if self._mode.startswith('frugal'): + self._page.driver.set_listener('Page.screencastFrame', None) + self._page.run_cdp('Page.stopScreencast') + else: + self._enable = False + while self._running: + sleep(.1) + + if self._mode.endswith('imgs'): + return str(Path(self._path).absolute()) + + if not str(video_name).isascii() or not str(self._path).isascii(): + raise TypeError('转换成视频仅支持英文路径和文件名。') + + try: + from cv2 import VideoWriter, imread, VideoWriter_fourcc + from numpy import fromfile, uint8 + except ModuleNotFoundError: + raise ModuleNotFoundError('请先安装cv2，pip install opencv-python') + + pic_list = Path(self._path).glob('*.jpg') + img = imread(str(next(pic_list))) + imgInfo = img.shape + size = (imgInfo[1], imgInfo[0]) + + videoWrite = VideoWriter(path, VideoWriter_fourcc(*"mp4v"), 5, size) + + for i in pic_list: + img = imread(str(i)) + videoWrite.write(img) + + clean_folder(self._path, ignore=(name,)) + return f'{self._path}{sep}{name}' + + def set_save_path(self, save_path=None): + """设置保存路径 + :param save_path: 保存路径 + :return: None + """ + if save_path: + save_path = Path(save_path) + if save_path.exists() and save_path.is_file(): + raise TypeError('save_path必须指定文件夹。') + save_path.mkdir(parents=True, exist_ok=True) + self._path = save_path + + def _run(self): + """非节俭模式运行方法""" + self._running = True + while self._enable: + self._page.get_screenshot(path=self._path, name=f'{time()}.jpg') + sleep(.04) + self._running = False + + def _onScreencastFrame(self, **kwargs): + """节俭模式运行方法""" + with open(f'{self._path}\\{kwargs["metadata"]["timestamp"]}.jpg', 'wb') as f: + f.write(b64decode(kwargs['data'])) + self._page.run_cdp('Page.screencastFrameAck', sessionId=kwargs['sessionId']) + + +class ScreencastMode(object): + def __init__(self, screencast): + self._screencast = screencast + + def video_mode(self): + self._screencast._mode = 'video' + + def frugal_video_mode(self): + self._screencast._mode = 'frugal_video' + + def js_video_mode(self): + self._screencast._mode = 'js_video' + + def frugal_imgs_mode(self): + self._screencast._mode = 'frugal_imgs' + + def imgs_mode(self): + self._screencast._mode = 'imgs' diff --git a/DrissionPage/_units/screencast.pyi b/DrissionPage/_units/screencast.pyi new file mode 100644 index 0000000..6c23592 --- /dev/null +++ b/DrissionPage/_units/screencast.pyi @@ -0,0 +1,46 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" +from pathlib import Path +from typing import Union + +from .._pages.chromium_base import ChromiumBase + + +class Screencast(object): + def __init__(self, page: ChromiumBase): + self._page: ChromiumBase = ... + self._path: Path = ... + self._running: bool = ... + self._enable: bool = ... + self._mode: str = ... + + @property + def set_mode(self) -> ScreencastMode: ... + + def start(self, save_path: Union[str, Path] = None) -> None: ... + + def stop(self, video_name: str = None) -> str: ... + + def set_save_path(self, save_path: Union[str, Path] = None) -> None: ... + + def _run(self) -> None: ... + + def _onScreencastFrame(self, **kwargs) -> None: ... + + +class ScreencastMode(object): + def __init__(self, screencast: Screencast): + self._screencast: Screencast = ... + + def video_mode(self) -> None: ... + + def frugal_video_mode(self) -> None: ... + + def js_video_mode(self) -> None: ... + + def frugal_imgs_mode(self) -> None: ... + + def imgs_mode(self) -> None: ... From 341591edc64a62bb73cd7c8e950432750d0146c1 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 26 Oct 2023 23:15:21 +0800 Subject: [PATCH 054/182] =?UTF-8?q?=E4=B8=8D=E5=86=8D=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E4=BC=A0=E9=80=92ChromiumDriver=E6=96=B9=E5=BC=8F=E5=88=9B?= =?UTF-8?q?=E5=BB=BA`ChromiumPage`=E5=92=8C`WebPage`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 5 +--- DrissionPage/_pages/chromium_base.pyi | 2 +- DrissionPage/_pages/chromium_page.py | 34 ++++++++++++--------------- DrissionPage/_pages/chromium_page.pyi | 5 ++-- DrissionPage/_pages/web_page.py | 14 ++++++----- DrissionPage/_pages/web_page.pyi | 2 +- setup.py | 2 +- 7 files changed, 29 insertions(+), 35 deletions(-) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 1159b17..0d6ceb6 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -93,15 +93,12 @@ class ChromiumBase(BasePage): self._get_document() self._first_run = False - def _driver_init(self, tab_id, is_init=True): + def _driver_init(self, tab_id): """新建页面、页面刷新、切换标签页后要进行的cdp参数初始化 :param tab_id: 要跳转到的标签页id - :param is_init: 是否初始化时执行本方法，用于判断是否to_tab()调用 :return: None """ self._is_loading = True - if is_init and hasattr(self, '_driver'): - return # ChromiumPage接收ChromiumDriver方式启动时 self._driver = ChromiumDriver(tab_id=tab_id, tab_type='page', address=self.address) self._alert = Alert() self._driver.set_listener('Page.javascriptDialogOpening', self._on_alert_open) diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 4ff2964..fb8e148 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -53,7 +53,7 @@ class ChromiumBase(BasePage): def _connect_browser(self, tab_id: str = None) -> None: ... - def _driver_init(self, tab_id: str, is_init: bool = True) -> None: ... + def _driver_init(self, tab_id: str) -> None: ... def _get_document(self) -> None: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 98e67ef..30d0e71 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -9,7 +9,6 @@ from time import sleep from requests import get from .._base.browser import Browser -from .._base.chromium_driver import ChromiumDriver from .._commons.browser import connect_browser from .._configs.chromium_options import ChromiumOptions from .._pages.chromium_base import ChromiumBase, Timeout @@ -23,42 +22,39 @@ from ..errors import BrowserConnectError class ChromiumPage(ChromiumBase): """用于管理浏览器的类""" - def __init__(self, addr_driver_opts=None, tab_id=None, timeout=None): + def __init__(self, addr_or_opts=None, tab_id=None, timeout=None, addr_driver_opts=None): """ - :param addr_driver_opts: 浏览器地址:端口或ChromiumOptions对象 + :param addr_or_opts: 浏览器地址:端口或ChromiumOptions对象 :param tab_id: 要控制的标签页id，不指定默认为激活的 :param timeout: 超时时间 """ + if not addr_or_opts and addr_driver_opts: + addr_or_opts = addr_driver_opts self._page = self - address = self._handle_options(addr_driver_opts) + address = self._handle_options(addr_or_opts) self._run_browser() super().__init__(address, tab_id) self.set.timeouts(implicit=timeout) self._page_init() - def _handle_options(self, addr_driver_opts): + def _handle_options(self, addr_or_opts): """设置浏览器启动属性 - :param addr_driver_opts: 'ip:port'、ChromiumOptions、ChromiumDriver + :param addr_or_opts: 'ip:port'、ChromiumOptions、ChromiumDriver :return: 返回浏览器地址 """ - if not addr_driver_opts: - self._driver_options = ChromiumOptions(addr_driver_opts) + if not addr_or_opts: + self._driver_options = ChromiumOptions(addr_or_opts) - elif isinstance(addr_driver_opts, ChromiumOptions): - self._driver_options = addr_driver_opts + elif isinstance(addr_or_opts, ChromiumOptions): + self._driver_options = addr_or_opts # 接收浏览器地址和端口 - elif isinstance(addr_driver_opts, str): + elif isinstance(addr_or_opts, str): self._driver_options = ChromiumOptions() - self._driver_options.set_debugger_address(addr_driver_opts) - - elif isinstance(addr_driver_opts, ChromiumDriver): - self._driver_options = ChromiumOptions(False) - self._driver_options.set_debugger_address(addr_driver_opts.address) - self._driver = addr_driver_opts + self._driver_options.set_debugger_address(addr_or_opts) else: - raise TypeError('只能接收ip:port格式、ChromiumOptions或ChromiumDriver类型参数。') + raise TypeError('只能接收ip:port格式或ChromiumOptions类型参数。') return self._driver_options.debugger_address @@ -229,7 +225,7 @@ class ChromiumPage(ChromiumBase): return self.driver.stop() - self._driver_init(tab_id, False) + self._driver_init(tab_id) if read_doc and self.ready_state in ('complete', None): self._get_document() diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index bf82ca6..5295ba0 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -6,7 +6,6 @@ from typing import Union, Tuple, List, Optional from .._base.browser import Browser -from .._base.chromium_driver import ChromiumDriver from .._configs.chromium_options import ChromiumOptions from .._pages.chromium_base import ChromiumBase from .._pages.chromium_tab import ChromiumTab @@ -18,7 +17,7 @@ from .._units.waiter import ChromiumPageWaiter class ChromiumPage(ChromiumBase): def __init__(self, - addr_driver_opts: Union[str, int, ChromiumOptions, ChromiumDriver] = None, + addr_or_opts: Union[str, int, ChromiumOptions] = None, tab_id: str = None, timeout: float = None): self._driver_options: ChromiumOptions = ... @@ -26,7 +25,7 @@ class ChromiumPage(ChromiumBase): self._browser: Browser = ... self._rect: Optional[ChromiumTabRect] = ... - def _handle_options(self, addr_driver_opts: Union[str, ChromiumDriver, ChromiumOptions]) -> str: ... + def _handle_options(self, addr_or_opts: Union[str, ChromiumOptions]) -> str: ... def _run_browser(self) -> None: ... diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index bc940b2..698193d 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -15,13 +15,15 @@ from .._units.setter import WebPageSetter class WebPage(SessionPage, ChromiumPage, BasePage): """整合浏览器和request的页面类""" - def __init__(self, mode='d', timeout=None, driver_or_options=None, session_or_options=None): + def __init__(self, mode='d', timeout=None, driver_options=None, session_or_options=None, driver_or_options=None): """初始化函数 :param mode: 'd' 或 's'，即driver模式和session模式 :param timeout: 超时时间，d模式时为寻找元素时间，s模式时为连接时间，默认10秒 - :param driver_or_options: ChromiumDriver对象，只使用s模式时应传入False + :param driver_options: ChromiumDriver对象，只使用s模式时应传入False :param session_or_options: Session对象或SessionOptions对象，只使用d模式时应传入False """ + if not driver_options and driver_or_options: + driver_options = driver_or_options self._mode = mode.lower() if self._mode not in ('s', 'd'): raise ValueError('mode参数只能是s或d。') @@ -29,10 +31,10 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self._has_session = True super().__init__(session_or_options=session_or_options) - if not driver_or_options: - driver_or_options = ChromiumOptions(read_file=driver_or_options) - driver_or_options.set_timeouts(implicit=self._timeout).set_paths(download_path=self.download_path) - super(SessionPage, self).__init__(addr_driver_opts=driver_or_options, timeout=timeout) + if not driver_options: + driver_options = ChromiumOptions(read_file=driver_options) + driver_options.set_timeouts(implicit=self._timeout).set_paths(download_path=self.download_path) + super(SessionPage, self).__init__(addr_or_opts=driver_options, timeout=timeout) self.change_mode(self._mode, go=False, copy_cookies=False) def __call__(self, loc_or_str, timeout=None): diff --git a/DrissionPage/_pages/web_page.pyi b/DrissionPage/_pages/web_page.pyi index 7939f5a..6509692 100644 --- a/DrissionPage/_pages/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -25,7 +25,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def __init__(self, mode: str = 'd', timeout: float = None, - driver_or_options: Union[ChromiumDriver, ChromiumOptions, bool] = None, + driver_options: Union[ChromiumOptions, bool] = None, session_or_options: Union[Session, SessionOptions, bool] = None) -> None: self._mode: str = ... self._has_driver: bool = ... diff --git a/setup.py b/setup.py index ab09e66..473d567 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="3.3.0", + version="4.0.0b0", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From 368665df576563657107128167f419fdc2cf0556 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 30 Oct 2023 01:57:18 +0800 Subject: [PATCH 055/182] =?UTF-8?q?4.0.0b1=E5=AF=B9=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E9=80=BB=E8=BE=91=E8=BF=9B=E8=A1=8C=E5=AE=8C=E5=85=A8=E9=87=8D?= =?UTF-8?q?=E6=9E=84=EF=BC=8C'none'=E6=A8=A1=E5=BC=8F=E4=B8=8D=E4=B8=BB?= =?UTF-8?q?=E5=8A=A8=E5=81=9C=E6=AD=A2=E7=BD=91=E9=A1=B5=E4=B8=94=E6=97=A0?= =?UTF-8?q?=E8=A7=86timeout=EF=BC=9Bwait.new=5Ftab()=E6=88=90=E5=8A=9F?= =?UTF-8?q?=E6=97=B6=E8=BF=94=E5=9B=9Eid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/base.py | 2 +- DrissionPage/_base/chromium_driver.py | 15 +- DrissionPage/_elements/chromium_element.py | 17 +- DrissionPage/_pages/chromium_base.py | 236 +++++++++------------ DrissionPage/_pages/chromium_base.pyi | 15 +- DrissionPage/_pages/chromium_frame.py | 173 ++++++++------- DrissionPage/_pages/chromium_frame.pyi | 4 +- DrissionPage/_units/download_manager.py | 4 +- DrissionPage/_units/waiter.py | 17 +- DrissionPage/_units/waiter.pyi | 4 +- setup.py | 2 +- 11 files changed, 227 insertions(+), 262 deletions(-) diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index fe6f9a8..521f982 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -367,7 +367,7 @@ class BasePage(BaseParser): self.retry_times = 3 self.retry_interval = 2 self._DownloadKit = None - self._download_path = str(Path('../..').absolute()) + self._download_path = str(Path('.').absolute()) @property def title(self): diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index c593222..b6781e2 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -6,7 +6,7 @@ from json import dumps, loads from queue import Queue, Empty from threading import Thread, Event -from time import perf_counter, sleep +from time import perf_counter from requests import get from websocket import WebSocketTimeoutException, WebSocketException, WebSocketConnectionClosedException, \ @@ -56,8 +56,8 @@ class ChromiumDriver(object): message_json = dumps(message) if self._debug: - if self._debug is True or ( - isinstance(self._debug, str) and message.get('method', '').startswith(self._debug)): + if self._debug is True or (isinstance(self._debug, str) and + message.get('method', '').startswith(self._debug)): print(f'发> {message_json}') elif isinstance(self._debug, (list, tuple, set)): for m in self._debug: @@ -74,17 +74,16 @@ class ChromiumDriver(object): while not self._stopped.is_set(): try: - return self.method_results[message['id']].get_nowait() + return self.method_results[message['id']].get(.2) except Empty: if self.alert_flag: self.alert_flag = False - return {'result': []} + return {'error': {'message': 'alert exists.'}} if timeout is not None and perf_counter() > timeout: return {'error': {'message': 'timeout'}} - sleep(.02) continue except Exception: @@ -138,7 +137,11 @@ class ChromiumDriver(object): function = self.event_handlers.get(event['method']) if function: + if self._debug: + print(f'开始执行 {function.__name__}') function(**event['params']) + if self._debug: + print(f'执行 {function.__name__}完毕') self.event_queue.task_done() diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 68ee961..e51a0d6 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -376,7 +376,7 @@ class ChromiumElement(DrissionElement): def run_js(self, script, *args, as_expr=False): """对本元素执行javascript代码 :param script: js文本 - :param args: 参数，按顺序在js文本中对应argument[0]、argument[1]... + :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 :return: 运行的结果 """ @@ -385,7 +385,7 @@ class ChromiumElement(DrissionElement): def run_async_js(self, script, *args, as_expr=False): """以异步方式对本元素执行javascript代码 :param script: js文本 - :param args: 参数，按顺序在js文本中对应argument[0]、argument[1]... + :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 :return: None """ @@ -841,7 +841,7 @@ class ChromiumShadowRoot(BaseElement): def run_js(self, script, *args, as_expr=False): """运行javascript代码 :param script: js文本 - :param args: 参数，按顺序在js文本中对应argument[0]、argument[1]... + :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 :return: 运行的结果 """ @@ -850,7 +850,7 @@ class ChromiumShadowRoot(BaseElement): def run_async_js(self, script, *args, as_expr=False): """以异步方式执行js代码 :param script: js文本 - :param args: 参数，按顺序在js文本中对应argument[0]、argument[1]... + :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 :return: None """ @@ -1042,9 +1042,9 @@ class ChromiumShadowRoot(BaseElement): loc = loc[0], loc[1][5:] timeout = timeout if timeout is not None else self.page.timeout - t1 = perf_counter() + end_time = perf_counter() + timeout eles = make_session_ele(self.html).eles(loc) - while not eles and perf_counter() - t1 <= timeout: + while not eles and perf_counter() <= end_time: eles = make_session_ele(self.html).eles(loc) if not eles: @@ -1299,7 +1299,7 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): :param script: js文本 :param as_expr: 是否作为表达式运行，为True时args无效 :param timeout: 超时时间 - :param args: 参数，按顺序在js文本中对应argument[0]、argument[1]... + :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :return: js执行结果 """ if isinstance(page_or_ele, (ChromiumElement, ChromiumShadowRoot)): @@ -1701,7 +1701,8 @@ class ChromiumScroll(object): x = r['layoutViewport']['pageX'] y = r['layoutViewport']['pageY'] - while True: + end_time = perf_counter() + self._driver.page.timeout + while perf_counter() < end_time: sleep(.1) r = page.run_cdp('Page.getLayoutMetrics') x1 = r['layoutViewport']['pageX'] diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 0d6ceb6..bc36b5d 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -24,8 +24,8 @@ from .._units.network_listener import NetworkListener from .._units.screencast import Screencast from .._units.setter import ChromiumBaseSetter from .._units.waiter import ChromiumBaseWaiter -from ..errors import ContextLossError, ElementLossError, AlertExistsError, CDPError, TabClosedError, \ - NoRectError, BrowserConnectError, GetDocumentError +from ..errors import (ContextLossError, ElementLossError, CDPError, TabClosedError, NoRectError, BrowserConnectError, + AlertExistsError) class ChromiumBase(BasePage): @@ -41,14 +41,14 @@ class ChromiumBase(BasePage): self._is_loading = None self._root_id = None # object id self._debug = False - self._debug_recorder = None self._set = None self._screencast = None self._actions = None self._listener = None self._has_alert = False + self._ready_state = None - self._download_path = str(Path('../..').absolute()) + self._download_path = str(Path('.').absolute()) if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): address = f'127.0.0.1:{address}' @@ -76,7 +76,6 @@ class ChromiumBase(BasePage): :param tab_id: 要控制的标签页id，不指定默认为激活的 :return: None """ - self._first_run = True self._is_reading = False self._upload_list = None self._wait = None @@ -89,9 +88,15 @@ class ChromiumBase(BasePage): if not tab_id: raise BrowserConnectError('浏览器连接失败，可能是浏览器版本原因。') tab_id = tab_id[0] + self._driver_init(tab_id) - self._get_document() - self._first_run = False + if self.ready_state == 'complete' and self._ready_state is None: + self._get_document() + self._ready_state = 'complete' + + r = self.run_cdp('Page.getFrameTree') + for i in findall(r"'id': '(.*?)'", str(r)): + self.browser._frames[i] = self.tab_id def _driver_init(self, tab_id): """新建页面、页面刷新、切换标签页后要进行的cdp参数初始化 @@ -99,6 +104,7 @@ class ChromiumBase(BasePage): :return: None """ self._is_loading = True + self._frame_id = tab_id self._driver = ChromiumDriver(tab_id=tab_id, tab_type='page', address=self.address) self._alert = Alert() self._driver.set_listener('Page.javascriptDialogOpening', self._on_alert_open) @@ -108,59 +114,20 @@ class ChromiumBase(BasePage): self._driver.call_method('Page.enable') self._driver.call_method('Emulation.setFocusEmulationEnabled', enabled=True) - self._driver.set_listener('Page.frameStoppedLoading', self._onFrameStoppedLoading) self._driver.set_listener('Page.frameStartedLoading', self._onFrameStartedLoading) - self._driver.set_listener('DOM.documentUpdated', self._onDocumentUpdated) - self._driver.set_listener('Page.loadEventFired', self._onLoadEventFired) self._driver.set_listener('Page.frameNavigated', self._onFrameNavigated) + self._driver.set_listener('Page.domContentEventFired', self._onDomContentEventFired) + self._driver.set_listener('Page.loadEventFired', self._onLoadEventFired) + self._driver.set_listener('Page.frameStoppedLoading', self._onFrameStoppedLoading) self._driver.set_listener('Page.frameAttached', self._onFrameAttached) self._driver.set_listener('Page.frameDetached', self._onFrameDetached) def _get_document(self): - """刷新cdp使用的document数据""" if self._is_reading: return - self._is_reading = True - - if self._debug: - print('获取document') - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '获取document', '开始')) - - try: # 遇到过网站在标签页关闭时触发读取文档导致错误，屏蔽掉 - self._wait_loaded() - except TabClosedError: - return - - end_time = perf_counter() + 10 - while perf_counter() < end_time: - try: - b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] - self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id)['object']['objectId'] - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '信息', f'root_id：{self._root_id}')) - break - - except CDPError as e: - err = e - if self._debug: - print('重试获取document') - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), 'err', '读取root_id出错')) - - sleep(.1) - - else: - txt = f'请检查是否创建了过多页面对象同时操作浏览器。\n如无法解决，请把以下信息报告作者。\n{err._info}\n' \ - f'报告网址：https://gitee.com/g1879/DrissionPage/issues' - raise GetDocumentError(txt) - - if self._debug: - print('获取document结束') - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '获取document', '结束')) - + b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] + self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id)['object']['objectId'] r = self.run_cdp('Page.getFrameTree') for i in findall(r"'id': '(.*?)'", str(r)): self.browser._frames[i] = self.tab_id @@ -173,25 +140,18 @@ class ChromiumBase(BasePage): :param timeout: 超时时间 :return: 是否成功，超时返回False """ - timeout = timeout if timeout is not None else self.timeouts.page_load + if self.page_load_strategy == 'none': + return True + timeout = timeout if timeout is not None else self.timeouts.page_load end_time = perf_counter() + timeout while perf_counter() < end_time: - state = self.ready_state - if state is None: # 存在alert的情况 - return None - - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), 'waiting', state)) - - if state == 'complete': + if self._ready_state == 'complete': return True - elif self.page_load_strategy == 'eager' and state in ('interactive', 'complete'): - self.stop_loading() - return True - elif self.page_load_strategy == 'none': + elif self.page_load_strategy == 'eager' and self._ready_state in ('interactive', 'complete'): self.stop_loading() return True + sleep(.1) self.stop_loading() @@ -209,50 +169,44 @@ class ChromiumBase(BasePage): def _onFrameStartedLoading(self, **kwargs): """页面开始加载时执行""" self.browser._frames[kwargs['frameId']] = self.tab_id - if kwargs['frameId'] == self._target_id: + if kwargs['frameId'] == self._frame_id: + self._ready_state = 'loading' self._is_loading = True - if self._debug: - print('页面开始加载 FrameStartedLoading') - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '加载流程', 'FrameStartedLoading')) + print(f'frameStartedLoading {kwargs}') + + def _onFrameNavigated(self, **kwargs): + """页面跳转时执行""" + if kwargs['frame']['id'] == self._frame_id: + self._ready_state = 'loading' + self._is_loading = True + if self._debug: + print(f'FrameNavigated {kwargs}') + + def _onDomContentEventFired(self, **kwargs): + """在页面刷新、变化后重新读取页面内容""" + self._ready_state = 'interactive' + if self.page_load_strategy == 'eager': + self.run_cdp('Page.stopLoading') + if self._debug: + print(f'DomContentEventFired {kwargs}') + + def _onLoadEventFired(self, **kwargs): + """在页面刷新、变化后重新读取页面内容""" + self._ready_state = 'complete' + if self._debug: + print(f'LoadEventFired {kwargs}') + # self._get_document() def _onFrameStoppedLoading(self, **kwargs): """页面加载完成后执行""" self.browser._frames[kwargs['frameId']] = self.tab_id - if kwargs['frameId'] == self._target_id and self._first_run is False and self._is_loading: + if kwargs['frameId'] == self._frame_id: + self._ready_state = 'complete' if self._debug: - print('页面停止加载 FrameStoppedLoading') - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '加载流程', 'FrameStoppedLoading')) - + print(f'FrameStoppedLoading {kwargs}') self._get_document() - def _onLoadEventFired(self, **kwargs): - """在页面刷新、变化后重新读取页面内容""" - if self._debug: - print('loadEventFired') - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '加载流程', 'loadEventFired')) - - self._get_document() - - def _onDocumentUpdated(self, **kwargs): - """页面跳转时执行""" - if self._debug: - print('documentUpdated') - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '加载流程', 'documentUpdated')) - - def _onFrameNavigated(self, **kwargs): - """页面跳转时执行""" - if kwargs['frame'].get('parentId', None) == self._target_id and self._first_run is False and self._is_loading: - self._is_loading = True - if self._debug: - print('navigated') - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '加载流程', 'navigated')) - def _onFileChooserOpened(self, **kwargs): """文件选择框打开时执行""" if self._upload_list: @@ -344,14 +298,13 @@ class ChromiumBase(BasePage): @property def ready_state(self): - """返回当前页面加载状态，'loading' 'interactive' 'complete'，有弹出框时返回None""" - while True: - try: - return self.run_cdp('Runtime.evaluate', expression='document.readyState;')['result']['value'] - except (AlertExistsError, TypeError): - return None - except ContextLossError: - continue + """返回当前页面加载状态，'loading' 'interactive' 'complete'，'timeout' 表示可能有弹出框""" + try: + return self.run_cdp('Runtime.evaluate', expression='document.readyState;', _timeout=3)['result']['value'] + except ContextLossError: + return None + except TimeoutError: + return 'timeout' @property def size(self): @@ -439,9 +392,6 @@ class ChromiumBase(BasePage): :param cmd_args: 参数 :return: 执行的结果 """ - # if self.driver.has_alert and cmd != HANDLE_ALERT_METHOD: - # raise AlertExistsError - r = self.driver.call_method(cmd, **cmd_args) if ERROR not in r: return r @@ -455,8 +405,10 @@ class ChromiumBase(BasePage): raise ElementLossError elif error == 'tab closed': raise TabClosedError - elif error == 'alert exists': - pass + elif error == 'timeout': + raise TimeoutError + elif error == 'alert exists.': + raise AlertExistsError elif error in ('Node does not have a layout object', 'Could not compute box model.'): raise NoRectError elif r['type'] == 'call_method_error': @@ -476,7 +428,7 @@ class ChromiumBase(BasePage): def run_js(self, script, *args, as_expr=False): """运行javascript代码 :param script: js文本 - :param args: 参数，按顺序在js文本中对应argument[0]、argument[1]... + :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 :return: 运行的结果 """ @@ -485,7 +437,7 @@ class ChromiumBase(BasePage): def run_js_loaded(self, script, *args, as_expr=False): """运行javascript代码，执行前等待页面加载完毕 :param script: js文本 - :param args: 参数，按顺序在js文本中对应argument[0]、argument[1]... + :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 :return: 运行的结果 """ @@ -495,7 +447,7 @@ class ChromiumBase(BasePage): def run_async_js(self, script, *args, as_expr=False): """以异步方式执行js代码 :param script: js文本 - :param args: 参数，按顺序在js文本中对应argument[0]、argument[1]... + :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 :return: None """ @@ -691,11 +643,12 @@ class ChromiumBase(BasePage): """页面停止加载""" if self._debug: print('停止页面加载') - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '操作', '停止页面加载')) - - self.run_cdp('Page.stopLoading') - while self.ready_state not in ('complete', None): + try: + self.run_cdp('Page.stopLoading') + except TabClosedError: + pass + end_time = perf_counter() + self.timeouts.page_load + while self._ready_state != 'complete' and perf_counter() < end_time: sleep(.1) def remove_ele(self, loc_or_ele): @@ -881,32 +834,39 @@ class ChromiumBase(BasePage): """ err = None timeout = timeout if timeout is not None else self.timeouts.page_load - for t in range(times + 1): err = None - result = self.run_cdp('Page.navigate', url=to_url) - - is_timeout = self._wait_loaded(timeout) - if is_timeout is None: - return None - is_timeout = not is_timeout - self.wait.load_complete() - - if is_timeout: + end_time = perf_counter() + timeout + result = self.run_cdp('Page.navigate', url=to_url, _timeout=timeout) + if result.get('error') == 'timeout': err = TimeoutError('页面连接超时。') - if 'errorText' in result: + + elif 'errorText' in result: err = ConnectionError(result['errorText']) + if err: + sleep(interval) + if self._debug or show_errmsg: + print(f'重试{t + 1} {to_url}') + self.stop_loading() + continue + + if self.page_load_strategy == 'none': + return True + + yu = end_time - perf_counter() + ok = self._wait_loaded(1 if yu <= 0 else yu) + if not ok: + err = TimeoutError('页面连接超时。') + sleep(interval) + if self._debug or show_errmsg: + print(f'重试{t + 1} {to_url}') + self.stop_loading() + continue + if not err: break - if t < times: - sleep(interval) - while self.ready_state not in ('complete', None): - sleep(.1) - if self._debug or show_errmsg: - print(f'重试{t + 1} {to_url}') - if err: if show_errmsg: raise err if err is not None else ConnectionError('连接异常。') diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index fb8e148..22812b3 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -4,9 +4,7 @@ @Contact : g1879@qq.com """ from pathlib import Path -from typing import Union, Tuple, List, Any - -from DataRecorder import Recorder +from typing import Union, Tuple, List, Any, Optional from .._base.base import BasePage from .._base.browser import Browser @@ -32,6 +30,7 @@ class ChromiumBase(BasePage): self._page: ChromiumPage = ... self.address: str = ... self._driver: ChromiumDriver = ... + self._frame_id: str = ... self._is_reading: bool = ... self._timeouts: Timeout = ... self._first_run: bool = ... @@ -41,7 +40,6 @@ class ChromiumBase(BasePage): self._url: str = ... self._root_id: str = ... self._debug: bool = ... - self._debug_recorder: Recorder = ... self._upload_list: list = ... self._wait: ChromiumBaseWaiter = ... self._set: ChromiumBaseSetter = ... @@ -50,6 +48,7 @@ class ChromiumBase(BasePage): self._listener: NetworkListener = ... self._alert: Alert = ... self._has_alert: bool = ... + self._ready_state: Optional[str] = ... def _connect_browser(self, tab_id: str = None) -> None: ... @@ -65,13 +64,13 @@ class ChromiumBase(BasePage): def _onFrameStartedLoading(self, **kwargs): ... - def _onFrameStoppedLoading(self, **kwargs): ... + def _onFrameNavigated(self, **kwargs): ... + + def _onDomContentEventFired(self, **kwargs): ... def _onLoadEventFired(self, **kwargs): ... - def _onDocumentUpdated(self, **kwargs): ... - - def _onFrameNavigated(self, **kwargs): ... + def _onFrameStoppedLoading(self, **kwargs): ... def _onFileChooserOpened(self, **kwargs): ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index d7fb0ed..34214c0 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from copy import copy -from re import search +from re import search, findall from threading import Thread from time import sleep, perf_counter @@ -14,7 +14,7 @@ from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase, ChromiumPageScroll from .._units.setter import ChromiumFrameSetter from .._units.waiter import FrameWaiter -from ..errors import ContextLossError +from ..errors import ContextLossError, ElementLossError, GetDocumentError class ChromiumFrame(ChromiumBase): @@ -40,6 +40,7 @@ class ChromiumFrame(ChromiumBase): self._backend_id = ele.ids.backend_id self._frame_ele = ele self._states = None + self._ids = ChromiumFrameIds(self) if self._is_inner_frame(): self._is_diff_domain = False @@ -50,9 +51,8 @@ class ChromiumFrame(ChromiumBase): super().__init__(page.address, self.frame_id, page.timeout) obj_id = super().run_js('document;', as_expr=True)['objectId'] self.doc_ele = ChromiumElement(self, obj_id=obj_id) - self._ids = ChromiumFrameIds(self) - end_time = perf_counter() + 2 + end_time = perf_counter() + 5 while perf_counter() < end_time and self.url == 'about:blank': sleep(.1) @@ -92,28 +92,46 @@ class ChromiumFrame(ChromiumBase): except: get(f'http://{self.address}/json', headers={'Connection': 'close'}) super()._driver_init(tab_id) + self.driver.set_listener('Inspector.detached', self._onInspectorDetached) def _reload(self): """重新获取document""" debug = self._debug + d_debug = self.driver._debug if debug: print('重新获取document') self._frame_ele = ChromiumElement(self._target_page, backend_id=self._backend_id) node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self._frame_ele.ids.backend_id)['node'] - if self._is_inner_frame(): - self._is_diff_domain = False - self.doc_ele = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) - super().__init__(self.address, self._target_page.tab_id, self._target_page.timeout) - self._debug = debug + end_time = perf_counter() + self.timeout + while perf_counter() < end_time: + try: + if self._is_inner_frame(): + self._is_diff_domain = False + self.doc_ele = ChromiumElement(self._target_page, + backend_id=node['contentDocument']['backendNodeId']) + super().__init__(self.address, self._target_page.tab_id, self._target_page.timeout) + self._debug = debug + self.driver._debug = d_debug + else: + self._is_diff_domain = True + self._driver.stop() + super().__init__(self.address, self.frame_id, self._target_page.timeout) + obj_id = super().run_js('document;', as_expr=True)['objectId'] + self.doc_ele = ChromiumElement(self, obj_id=obj_id) + self._debug = debug + self.driver._debug = d_debug + break + except: + pass + + sleep(.1) + else: - self._is_diff_domain = True - self._driver.stop() - super().__init__(self.address, self.frame_id, self._target_page.timeout) - obj_id = super().run_js('document;', as_expr=True)['objectId'] - self.doc_ele = ChromiumElement(self, obj_id=obj_id) - self._debug = debug + raise GetDocumentError + + self.wait.load_complete() def _check_ok(self): """用于应付同域异域之间跳转导致元素丢失问题""" @@ -122,7 +140,7 @@ class ChromiumFrame(ChromiumBase): try: self._target_page.run_cdp('DOM.describeNode', nodeId=self.ids.node_id) - except Exception: + except ElementLossError: self._reload() # sleep(2) @@ -130,72 +148,42 @@ class ChromiumFrame(ChromiumBase): """刷新cdp使用的document数据""" if self._is_reading: return - self._is_reading = True + if self._is_diff_domain is False: + node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self.ids.backend_id)['node'] + self.doc_ele = ChromiumElement(self._target_page, + backend_id=node['contentDocument']['backendNodeId']) - if self._debug: - print('---获取document') + else: + b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] + self.doc_ele = ChromiumElement(self, backend_id=b_id) - end_time = perf_counter() + 3 - while self.is_alive and perf_counter() < end_time: - try: - if self._is_diff_domain is False: - node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self.ids.backend_id)['node'] - self.doc_ele = ChromiumElement(self._target_page, - backend_id=node['contentDocument']['backendNodeId']) - - else: - b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] - self.doc_ele = ChromiumElement(self, backend_id=b_id) - - break - - except Exception: - sleep(.1) - - # else: - # raise RuntimeError('获取document失败。') - - if self._debug: - print('---获取document结束') + r = self.run_cdp('Page.getFrameTree') + for i in findall(r"'id': '(.*?)'", str(r)): + self.browser._frames[i] = self.tab_id self._is_loading = False self._is_reading = False - def _onFrameNavigated(self, **kwargs): - """页面跳转时触发""" - if kwargs['frame']['id'] == self.frame_id and self._first_run is False and self._is_loading: - self._is_loading = True - - if self._debug: - print('navigated') - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '加载流程', 'navigated')) - - def _onLoadEventFired(self, **kwargs): - """在页面刷新、变化后重新读取页面内容""" - # 用于覆盖父类方法，不能删 - self._get_new_document() - - if self._debug: - print('loadEventFired') - if self._debug_recorder: - self._debug_recorder.add_data((perf_counter(), '加载流程', 'loadEventFired')) - - def _onFrameStartedLoading(self, **kwargs): - """页面开始加载时触发""" - if kwargs['frameId'] == self.frame_id: - self._is_loading = True - if self._debug: - print('页面开始加载 FrameStartedLoading') - def _onFrameStoppedLoading(self, **kwargs): """页面加载完成后触发""" - if kwargs['frameId'] == self.frame_id and self._first_run is False and self._is_loading: + self.browser._frames[kwargs['frameId']] = self.tab_id + if kwargs['frameId'] == self.frame_id: + self._ready_state = 'complete' if self._debug: - print('页面停止加载 FrameStoppedLoading') + print(f'FrameStoppedLoading {kwargs}') self._get_new_document() + def _onInspectorDetached(self, **kwargs): + self._is_loading = True + # print('reload') + self._reload() + + # def _onFrameDetached(self, **kwargs): + # if kwargs['frameId'] == self.frame_id: + # self._is_loading = True + # self._reload() + @property def page(self): return self._page @@ -387,7 +375,7 @@ class ChromiumFrame(ChromiumBase): def run_js(self, script, *args, as_expr=False): """运行javascript代码 :param script: js文本 - :param args: 参数，按顺序在js文本中对应argument[0]、argument[1]... + :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 :return: 运行的结果 """ @@ -614,34 +602,43 @@ class ChromiumFrame(ChromiumBase): for t in range(times + 1): err = None - result = self.driver.call_method('Page.navigate', url=to_url, frameId=self.frame_id) - - is_timeout = not self._wait_loaded(timeout) - sleep(.5) - self.wait.load_complete() - - if is_timeout: + end_time = perf_counter() + timeout + result = self.driver.call_method('Page.navigate', url=to_url, frameId=self.frame_id, _timeout=timeout) + if result.get('error') == 'timeout': err = TimeoutError('页面连接超时。') - if 'errorText' in result: + + elif 'errorText' in result: err = ConnectionError(result['errorText']) + if err: + sleep(interval) + if self._debug or show_errmsg: + print(f'重试{t + 1} {to_url}') + self.stop_loading() + continue + + if self.page_load_strategy == 'none': + return True + + yu = end_time - perf_counter() + ok = self._wait_loaded(1 if yu <= 0 else yu) + if not ok: + err = TimeoutError('页面连接超时。') + sleep(interval) + if self._debug or show_errmsg: + print(f'重试{t + 1} {to_url}') + self.stop_loading() + continue + if not err: break - if t < times: - sleep(interval) - while self.ready_state not in ('complete', None): - sleep(.1) - if self._debug: - print('重试') - if show_errmsg: - print(f'重试 {to_url}') - if err: if show_errmsg: raise err if err is not None else ConnectionError('连接异常。') return False + self._check_ok() return True def _is_inner_frame(self): diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 8b4e0bf..4121917 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -49,9 +49,9 @@ class ChromiumFrame(ChromiumBase): def _get_new_document(self) -> None: ... - def _onFrameAttached(self, **kwargs): ... + def _onFrameStoppedLoading(self, **kwargs): ... - def _onFrameDetached(self, **kwargs): ... + def _onInspectorDetached(self, **kwargs): ... @property def page(self) -> Union[ChromiumPage, WebPage]: ... diff --git a/DrissionPage/_units/download_manager.py b/DrissionPage/_units/download_manager.py index 029ccb9..be55e9d 100644 --- a/DrissionPage/_units/download_manager.py +++ b/DrissionPage/_units/download_manager.py @@ -266,8 +266,8 @@ class DownloadMission(object): """ if show: print(f'url：{self.url}') - t2 = perf_counter() - while self.name is None and perf_counter() - t2 < 4: + end_time = perf_counter() + while self.name is None and perf_counter() < end_time: sleep(0.01) print(f'文件名：{self.name}') print(f'目标路径：{self.path}') diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 4baca34..8236b30 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -82,8 +82,12 @@ class ChromiumBaseWaiter(object): def upload_paths_inputted(self): """等待自动填写上传文件路径""" - while self._driver._upload_list: + end_time = perf_counter() + self._driver.timeout + while perf_counter() < end_time: + if not self._driver._upload_list: + return True sleep(.01) + return False def download_begin(self, timeout=None, cancel_it=False): """等待浏览器下载开始，可将其拦截 @@ -201,7 +205,7 @@ class ChromiumTabWaiter(ChromiumBaseWaiter): else: end_time = perf_counter() + timeout - while end_time > perf_counter(): + while perf_counter() < end_time: if not self._driver.browser._dl_mgr.get_tab_missions(self._driver.tab_id): return True sleep(.5) @@ -224,13 +228,14 @@ class ChromiumPageWaiter(ChromiumTabWaiter): """等待新标签页出现 :param timeout: 等待超时时间，为None则使用页面对象timeout属性 :param raise_err: 等待失败时是否报错，为None时根据Settings设置 - :return: 是否等到新标签页出现 + :return: 等到新标签页返回其id，否则返回False """ timeout = timeout if timeout is not None else self._driver.timeout end_time = perf_counter() + timeout while perf_counter() < end_time: - if self._driver.tab_id != self._driver.latest_tab: - return True + latest_tab = self._driver.latest_tab + if self._driver.tab_id != latest_tab: + return latest_tab sleep(.01) if raise_err is True or Settings.raise_when_wait_failed is True: @@ -251,7 +256,7 @@ class ChromiumPageWaiter(ChromiumTabWaiter): else: end_time = perf_counter() + timeout - while end_time > perf_counter(): + while perf_counter() < end_time: if not self._driver.browser._dl_mgr._missions: return True sleep(.5) diff --git a/DrissionPage/_units/waiter.pyi b/DrissionPage/_units/waiter.pyi index 3bbd19c..1f4da31 100644 --- a/DrissionPage/_units/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -37,7 +37,7 @@ class ChromiumBaseWaiter(object): def load_complete(self, timeout: float = None, raise_err: bool = None) -> bool: ... - def upload_paths_inputted(self) -> None: ... + def upload_paths_inputted(self) -> bool: ... def download_begin(self, timeout: float = None, cancel_it: bool = False) -> Union[DownloadMission, bool]: ... @@ -62,7 +62,7 @@ class ChromiumTabWaiter(ChromiumBaseWaiter): class ChromiumPageWaiter(ChromiumTabWaiter): _driver: ChromiumPage = ... - def new_tab(self, timeout: float = None, raise_err: bool = None) -> bool: ... + def new_tab(self, timeout: float = None, raise_err: bool = None) -> Union[str, bool]: ... def all_downloads_done(self, timeout: float = None, cancel_if_timeout: bool = True) -> bool: ... diff --git a/setup.py b/setup.py index 473d567..5007555 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b0", + version="4.0.0b1", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From 989a92adb79dee207a3ff37db5b33b7d2cd7b731 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 30 Oct 2023 18:09:14 +0800 Subject: [PATCH 056/182] =?UTF-8?q?=E4=B8=BB=E5=8A=A8=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E4=B9=9F=E9=81=B5=E5=BE=AA=E5=8A=A0=E8=BD=BD=E7=AD=96=E7=95=A5?= =?UTF-8?q?=E8=B6=85=E6=97=B6=EF=BC=9B=E4=BF=AE=E5=A4=8D=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E8=AE=BE=E7=BD=AE=E9=97=AE=E9=A2=98=EF=BC=9B?= =?UTF-8?q?=E5=A2=9E=E5=BC=BA=E8=AF=BB=E5=8F=96doc=E7=A8=B3=E5=AE=9A?= =?UTF-8?q?=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 28 ++++++++++++++++++++++--- DrissionPage/_pages/chromium_base.pyi | 2 +- DrissionPage/_pages/chromium_frame.py | 22 +++++++++++++------ DrissionPage/_units/download_manager.py | 2 +- MANIFEST.in | 2 +- setup.py | 2 +- 6 files changed, 45 insertions(+), 13 deletions(-) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index bc36b5d..1adf588 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -7,6 +7,7 @@ from json import loads, JSONDecodeError from os.path import sep from pathlib import Path from re import findall +from threading import Thread from time import perf_counter, sleep from requests import get @@ -25,7 +26,7 @@ from .._units.screencast import Screencast from .._units.setter import ChromiumBaseSetter from .._units.waiter import ChromiumBaseWaiter from ..errors import (ContextLossError, ElementLossError, CDPError, TabClosedError, NoRectError, BrowserConnectError, - AlertExistsError) + AlertExistsError, GetDocumentError) class ChromiumBase(BasePage): @@ -126,8 +127,17 @@ class ChromiumBase(BasePage): if self._is_reading: return self._is_reading = True - b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] - self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id)['object']['objectId'] + end_time = perf_counter() + 10 + while perf_counter() < end_time: + try: + b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] + self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id)['object']['objectId'] + break + except: + continue + else: + raise GetDocumentError + r = self.run_cdp('Page.getFrameTree') for i in findall(r"'id': '(.*?)'", str(r)): self.browser._frames[i] = self.tab_id @@ -172,6 +182,10 @@ class ChromiumBase(BasePage): if kwargs['frameId'] == self._frame_id: self._ready_state = 'loading' self._is_loading = True + if self.page_load_strategy == 'eager': + t = Thread(target=self._wait_to_stop) + t.daemon = True + t.start() if self._debug: print(f'frameStartedLoading {kwargs}') @@ -228,6 +242,14 @@ class ChromiumBase(BasePage): """ return self.ele(loc_or_str, timeout) + def _wait_to_stop(self): + """eager策略超时时使页面停止加载""" + end_time = perf_counter() + self.timeouts.page_load + while perf_counter() < end_time: + sleep(.1) + if self._ready_state in ('interactive', 'complete') and self._is_loading: + self.stop_loading() + @property def main(self): return self._page diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 22812b3..7c34135 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -74,7 +74,7 @@ class ChromiumBase(BasePage): def _onFileChooserOpened(self, **kwargs): ... - # def _onDownloadWillBegin(self, **kwargs): ... + def _wait_to_stop(self): ... def _d_set_start_options(self, address, none) -> None: ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 34214c0..fb8a047 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -149,14 +149,24 @@ class ChromiumFrame(ChromiumBase): if self._is_reading: return self._is_reading = True - if self._is_diff_domain is False: - node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self.ids.backend_id)['node'] - self.doc_ele = ChromiumElement(self._target_page, - backend_id=node['contentDocument']['backendNodeId']) + end_time = perf_counter() + 10 + while perf_counter() < end_time: + try: + if self._is_diff_domain is False: + node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self.ids.backend_id)['node'] + self.doc_ele = ChromiumElement(self._target_page, + backend_id=node['contentDocument']['backendNodeId']) + + else: + b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] + self.doc_ele = ChromiumElement(self, backend_id=b_id) + + break + except: + continue else: - b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] - self.doc_ele = ChromiumElement(self, backend_id=b_id) + raise GetDocumentError r = self.run_cdp('Page.getFrameTree') for i in findall(r"'id': '(.*?)'", str(r)): diff --git a/DrissionPage/_units/download_manager.py b/DrissionPage/_units/download_manager.py index be55e9d..09b8afb 100644 --- a/DrissionPage/_units/download_manager.py +++ b/DrissionPage/_units/download_manager.py @@ -181,7 +181,7 @@ class BrowserDownloadManager(object): return mission.received_bytes = kwargs['receivedBytes'] mission.total_bytes = kwargs['totalBytes'] - form_path = f'{mission.save_path}{sep}{mission.id}' + form_path = f'{mission.path}{sep}{mission.id}' to_path = str(get_usable_path(f'{mission.path}{sep}{mission.name}')) move(form_path, to_path) self.set_done(mission, 'completed', final_path=to_path) diff --git a/MANIFEST.in b/MANIFEST.in index 4c619d1..962ba91 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include DrissionPage/configs/configs.ini +include DrissionPage/_configs/configs.ini include DrissionPage/*.pyi include DrissionPage/*/*.py include DrissionPage/*/*.pyi \ No newline at end of file diff --git a/setup.py b/setup.py index 5007555..76574fa 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b1", + version="4.0.0b3", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From 722e2991506442cc37ed691cf61af81de8e8b864 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 30 Oct 2023 22:05:52 +0800 Subject: [PATCH 057/182] =?UTF-8?q?`rect.borwser=5Fsize`=E6=94=B9=E4=B8=BA?= =?UTF-8?q?`window=5Fsize`=EF=BC=9B`rect.borwser=5Flocation`=E6=94=B9?= =?UTF-8?q?=E4=B8=BA`window=5Flocation`=EF=BC=9Bcdp=E8=AE=BE=E7=BD=AE30?= =?UTF-8?q?=E7=A7=92=E8=B6=85=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/chromium_driver.py | 2 +- DrissionPage/_pages/chromium_base.py | 44 ++++++++++++------------ DrissionPage/_pages/chromium_base.pyi | 1 + DrissionPage/_units/network_listener.pyi | 3 +- DrissionPage/_units/setter.py | 10 +++--- DrissionPage/_units/setter.pyi | 10 +++--- DrissionPage/_units/tab_rect.py | 38 ++++++++++---------- DrissionPage/_units/tab_rect.pyi | 6 ++-- 8 files changed, 58 insertions(+), 56 deletions(-) diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index b6781e2..948cdc7 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -155,7 +155,7 @@ class ChromiumDriver(object): if self._stopped.is_set(): return {'error': 'tab closed', 'type': 'tab_closed'} - timeout = kwargs.pop("_timeout", None) + timeout = kwargs.pop("_timeout", 30) result = self._send({"method": _method, "params": kwargs}, timeout=timeout) if result is None: return {'error': 'tab closed', 'type': 'tab_closed'} diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 1adf588..2b0d462 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -145,28 +145,6 @@ class ChromiumBase(BasePage): self._is_loading = False self._is_reading = False - def _wait_loaded(self, timeout=None): - """等待页面加载完成，超时触发停止加载 - :param timeout: 超时时间 - :return: 是否成功，超时返回False - """ - if self.page_load_strategy == 'none': - return True - - timeout = timeout if timeout is not None else self.timeouts.page_load - end_time = perf_counter() + timeout - while perf_counter() < end_time: - if self._ready_state == 'complete': - return True - elif self.page_load_strategy == 'eager' and self._ready_state in ('interactive', 'complete'): - self.stop_loading() - return True - - sleep(.1) - - self.stop_loading() - return False - def _onFrameDetached(self, **kwargs): try: self.browser._frames.pop(kwargs['frameId']) @@ -845,6 +823,28 @@ class ChromiumBase(BasePage): self._alert.response_text = None self._has_alert = True + def _wait_loaded(self, timeout=None): + """等待页面加载完成，超时触发停止加载 + :param timeout: 超时时间 + :return: 是否成功，超时返回False + """ + if self.page_load_strategy == 'none': + return True + + timeout = timeout if timeout is not None else self.timeouts.page_load + end_time = perf_counter() + timeout + while perf_counter() < end_time: + if self._ready_state == 'complete': + return True + elif self.page_load_strategy == 'eager' and self._ready_state in ('interactive', + 'complete') and not self._is_loading: + return True + + sleep(.1) + + self.stop_loading() + return False + def _d_connect(self, to_url, times=0, interval=1, show_errmsg=False, timeout=None): """尝试连接，重试若干次 :param to_url: 要访问的url diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 7c34135..8c35695 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -32,6 +32,7 @@ class ChromiumBase(BasePage): self._driver: ChromiumDriver = ... self._frame_id: str = ... self._is_reading: bool = ... + self._is_timeout: bool = ... self._timeouts: Timeout = ... self._first_run: bool = ... self._is_loading: bool = ... diff --git a/DrissionPage/_units/network_listener.pyi b/DrissionPage/_units/network_listener.pyi index bb26f99..75aab24 100644 --- a/DrissionPage/_units/network_listener.pyi +++ b/DrissionPage/_units/network_listener.pyi @@ -35,7 +35,8 @@ class NetworkListener(object): def go_on(self) -> None: ... - def wait(self, count: int = 1, timeout: float = None, fix_count: bool = True): ... + def wait(self, count: int = 1, timeout: float = None, + fix_count: bool = True) -> Union[List[DataPacket], DataPacket, None]: ... @property def results(self) -> Union[DataPacket, Dict[str, List[DataPacket]], False]: ... diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index f0de74a..841aea3 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -122,6 +122,11 @@ class TabSetter(ChromiumBaseSetter): def __init__(self, page): super().__init__(page) + @property + def window(self): + """返回用于设置浏览器窗口的对象""" + return WindowSetter(self._page) + def download_path(self, path): """设置下载路径 :param path: 下载路径 @@ -165,11 +170,6 @@ class ChromiumPageSetter(TabSetter): """ self._page._main_tab = tab_id or self._page.tab_id - @property - def window(self): - """返回用于设置浏览器窗口的对象""" - return WindowSetter(self._page) - def tab_to_front(self, tab_or_id=None): """激活标签页使其处于最前面 :param tab_or_id: 标签页对象或id，为None表示当前标签页 diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index f44413e..e98b7de 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -56,6 +56,9 @@ class ChromiumBaseSetter(object): class TabSetter(ChromiumBaseSetter): def __init__(self, page): ... + @property + def window(self) -> WindowSetter: ... + def download_path(self, path: Union[str, Path]) -> None: ... def download_file_name(self, name: str) -> None: ... @@ -70,9 +73,6 @@ class ChromiumPageSetter(TabSetter): def main_tab(self, tab_id: str = None) -> None: ... - @property - def window(self) -> WindowSetter: ... - def tab_to_front(self, tab_or_id: Union[str, ChromiumTab] = None) -> None: ... @@ -184,8 +184,8 @@ class PageScrollSetter(object): class WindowSetter(object): - def __init__(self, page: ChromiumPage): - self._page: ChromiumPage = ... + def __init__(self, page: ChromiumBase): + self._page: ChromiumBase = ... self._window_id: str = ... def maximized(self) -> None: ... diff --git a/DrissionPage/_units/tab_rect.py b/DrissionPage/_units/tab_rect.py index 6655932..df27036 100644 --- a/DrissionPage/_units/tab_rect.py +++ b/DrissionPage/_units/tab_rect.py @@ -12,16 +12,27 @@ class ChromiumTabRect(object): @property def window_state(self): """返回窗口状态：normal、fullscreen、maximized、 minimized""" - return self._get_browser_rect()['windowState'] + return self._get_window_rect()['windowState'] @property - def browser_location(self): - """返回浏览器在屏幕上的坐标，左上角为(0, 0)""" - r = self._get_browser_rect() + def window_location(self): + """返回窗口在屏幕上的坐标，左上角为(0, 0)""" + r = self._get_window_rect() if r['windowState'] in ('maximized', 'fullscreen'): return 0, 0 return r['left'] + 7, r['top'] + @property + def window_size(self): + """返回窗口大小""" + r = self._get_window_rect() + if r['windowState'] == 'fullscreen': + return r['width'], r['height'] + elif r['windowState'] == 'maximized': + return r['width'] - 16, r['height'] - 16 + else: + return r['width'] - 16, r['height'] - 7 + @property def page_location(self): """返回页面左上角在屏幕中坐标，左上角为(0, 0)""" @@ -32,22 +43,11 @@ class ChromiumTabRect(object): @property def viewport_location(self): """返回视口在屏幕中坐标，左上角为(0, 0)""" - w_bl, h_bl = self.browser_location - w_bs, h_bs = self.browser_size + w_bl, h_bl = self.window_location + w_bs, h_bs = self.window_size w_vs, h_vs = self.viewport_size_with_scrollbar return w_bl + w_bs - w_vs, h_bl + h_bs - h_vs - @property - def browser_size(self): - """返回浏览器大小""" - r = self._get_browser_rect() - if r['windowState'] == 'fullscreen': - return r['width'], r['height'] - elif r['windowState'] == 'maximized': - return r['width'] - 16, r['height'] - 16 - else: - return r['width'] - 16, r['height'] - 7 - @property def page_size(self): """返回页面总宽高，格式：(宽, 高)""" @@ -71,6 +71,6 @@ class ChromiumTabRect(object): """获取页面范围信息""" return self._page.run_cdp_loaded('Page.getLayoutMetrics') - def _get_browser_rect(self): - """获取浏览器范围信息""" + def _get_window_rect(self): + """获取窗口范围信息""" return self._page.browser.get_window_bounds(self._page.tab_id) diff --git a/DrissionPage/_units/tab_rect.pyi b/DrissionPage/_units/tab_rect.pyi index 95ebadc..d21aa1e 100644 --- a/DrissionPage/_units/tab_rect.pyi +++ b/DrissionPage/_units/tab_rect.pyi @@ -17,7 +17,7 @@ class ChromiumTabRect(object): def window_state(self) -> str: ... @property - def browser_location(self) -> Tuple[int, int]: ... + def window_location(self) -> Tuple[int, int]: ... @property def page_location(self) -> Tuple[int, int]: ... @@ -26,7 +26,7 @@ class ChromiumTabRect(object): def viewport_location(self) -> Tuple[int, int]: ... @property - def browser_size(self) -> Tuple[int, int]: ... + def window_size(self) -> Tuple[int, int]: ... @property def page_size(self) -> Tuple[int, int]: ... @@ -39,4 +39,4 @@ class ChromiumTabRect(object): def _get_page_rect(self) -> dict: ... - def _get_browser_rect(self) -> dict: ... + def _get_window_rect(self) -> dict: ... From b5f2e28e32a901e91c0d9bb00c3b0a8f8a367455 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 31 Oct 2023 00:25:29 +0800 Subject: [PATCH 058/182] =?UTF-8?q?4.0.0b3=E6=8A=93=E5=8C=85=E5=8F=AF?= =?UTF-8?q?=E5=8C=85=E5=90=AB=E6=89=A9=E5=B1=95=E4=BF=A1=E6=81=AF=EF=BC=9B?= =?UTF-8?q?url=E4=B8=8D=E5=AF=B9=E6=89=80=E6=9C=89=E5=85=81=E8=AE=B8?= =?UTF-8?q?=E7=9A=84=E5=AD=97=E7=AC=A6=E8=BD=AC=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/base.py | 2 +- DrissionPage/_pages/chromium_base.py | 7 ++- DrissionPage/_pages/chromium_base.pyi | 1 + DrissionPage/_units/network_listener.py | 78 ++++++++++++++++++------ DrissionPage/_units/network_listener.pyi | 43 +++++++++++-- 5 files changed, 105 insertions(+), 26 deletions(-) diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index 521f982..fbf4f67 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -414,7 +414,7 @@ class BasePage(BaseParser): :param interval: 重试间隔 :return: 重试次数和间隔组成的tuple """ - self._url = quote(url, safe='/:&?=%;#@+![]') + self._url = quote(url, safe='-_.~!*\'();:@&=+$,/?#[]') retry = retry if retry is not None else self.retry_times interval = interval if interval is not None else self.retry_interval return retry, interval diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 2b0d462..6327634 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -48,6 +48,7 @@ class ChromiumBase(BasePage): self._listener = None self._has_alert = False self._ready_state = None + self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc self._download_path = str(Path('.').absolute()) @@ -188,16 +189,18 @@ class ChromiumBase(BasePage): self._ready_state = 'complete' if self._debug: print(f'LoadEventFired {kwargs}') - # self._get_document() + self._get_document() + self._doc_got = True def _onFrameStoppedLoading(self, **kwargs): """页面加载完成后执行""" self.browser._frames[kwargs['frameId']] = self.tab_id - if kwargs['frameId'] == self._frame_id: + if kwargs['frameId'] == self._frame_id and self._doc_got is False: self._ready_state = 'complete' if self._debug: print(f'FrameStoppedLoading {kwargs}') self._get_document() + self._doc_got = False def _onFileChooserOpened(self, **kwargs): """文件选择框打开时执行""" diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 8c35695..9a8aed2 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -49,6 +49,7 @@ class ChromiumBase(BasePage): self._listener: NetworkListener = ... self._alert: Alert = ... self._has_alert: bool = ... + self._doc_got: bool = ... self._ready_state: Optional[str] = ... def _connect_browser(self, tab_id: str = None) -> None: ... diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index a82c5f9..36f99f7 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -179,17 +179,20 @@ class NetworkListener(object): def _set_callback(self): """设置监听请求的回调函数""" self._driver.set_listener('Network.requestWillBeSent', self._requestWillBeSent) + self._driver.set_listener('Network.requestWillBeSentExtraInfo', self._requestWillBeSentExtraInfo) self._driver.set_listener('Network.responseReceived', self._response_received) + self._driver.set_listener('Network.responseReceivedExtraInfo', self._responseReceivedExtraInfo) self._driver.set_listener('Network.loadingFinished', self._loading_finished) self._driver.set_listener('Network.loadingFailed', self._loading_failed) def _requestWillBeSent(self, **kwargs): """接收到请求时的回调函数""" if not self._targets: - self._request_ids[kwargs['requestId']] = DataPacket(self._page.tab_id, None, kwargs) + packet = self._request_ids.setdefault(kwargs['requestId'], DataPacket(self._page.tab_id, None)) + packet._raw_request = kwargs if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): - self._request_ids[kwargs['requestId']]._raw_post_data = \ - self._driver.call_method('Network.getRequestPostData', requestId=kwargs['requestId'])['postData'] + packet._raw_post_data = self._driver.call_method('Network.getRequestPostData', + requestId=kwargs['requestId'])['postData'] return @@ -197,20 +200,40 @@ class NetworkListener(object): if ((self._is_regex and search(target, kwargs['request']['url'])) or (not self._is_regex and target in kwargs['request']['url'])) and ( not self._method or kwargs['request']['method'] in self._method): - self._request_ids[kwargs['requestId']] = DataPacket(self._page.tab_id, target, kwargs) - + packet = self._request_ids.setdefault(kwargs['requestId'], DataPacket(self._page.tab_id, None)) + packet._raw__request = kwargs if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): - self._request_ids[kwargs['requestId']]._raw_post_data = \ - self._driver.call_method('Network.getRequestPostData', requestId=kwargs['requestId'])['postData'] + packet._raw_post_data = self._driver.call_method('Network.getRequestPostData', + requestId=kwargs['requestId'])['postData'] + + break + + def _requestWillBeSentExtraInfo(self, **kwargs): + if not self._targets: + packet = self._request_ids.setdefault(kwargs['requestId'], DataPacket(self._page.tab_id, None)) + packet._requestExtraInfo = kwargs + return + + for target in self._targets: + if ((self._is_regex and search(target, kwargs['request']['url'])) or + (not self._is_regex and target in kwargs['request']['url'])) and ( + not self._method or kwargs['request']['method'] in self._method): + packet = self._request_ids.setdefault(kwargs['requestId'], DataPacket(self._page.tab_id, None)) + packet._requestExtraInfo = kwargs break def _response_received(self, **kwargs): """接收到返回信息时处理方法""" - request_id = kwargs['requestId'] - if request_id in self._request_ids: - self._request_ids[request_id]._raw_response = kwargs['response'] - self._request_ids[request_id]._resource_type = kwargs['type'] + request = self._request_ids.get(kwargs['requestId']) + if request: + request._raw_response = kwargs['response'] + request._resource_type = kwargs['type'] + + def _responseReceivedExtraInfo(self, **kwargs): + request = self._request_ids.get(kwargs['requestId']) + if request: + request._responseExtraInfo = kwargs def _loading_finished(self, **kwargs): """请求完成时处理方法""" @@ -249,21 +272,22 @@ class NetworkListener(object): class DataPacket(object): """返回的数据包管理类""" - def __init__(self, tab_id, target, raw_request): + def __init__(self, tab_id, target): """ :param tab_id: 产生这个数据包的tab的id :param target: 监听目标 - :param raw_request: 原始request数据，从cdp获得 """ self.tab_id = tab_id self.target = target - self._raw_request = raw_request + self._raw_request = None self._raw_post_data = None self._raw_response = None self._raw_body = None self._base64_body = False + self._requestExtraInfo = None + self._responseExtraInfo = None self._request = None self._response = None @@ -293,22 +317,23 @@ class DataPacket(object): @property def request(self): if self._request is None: - self._request = Request(self._raw_request['request'], self._raw_post_data) + self._request = Request(self._raw_request['request'], self._raw_post_data, self._requestExtraInfo) return self._request @property def response(self): if self._response is None: - self._response = Response(self._raw_response, self._raw_body, self._base64_body) + self._response = Response(self._raw_response, self._raw_body, self._base64_body, self._responseExtraInfo) return self._response class Request(object): - def __init__(self, raw_request, post_data): + def __init__(self, raw_request, post_data, extra_info): self._request = raw_request self._raw_post_data = post_data self._postData = None self._headers = None + self.extra_info = RequestExtraInfo(extra_info or {}) def __getattr__(self, item): return self._request.get(item, None) @@ -338,12 +363,13 @@ class Request(object): class Response(object): - def __init__(self, raw_response, raw_body, base64_body): + def __init__(self, raw_response, raw_body, base64_body, extra_info): self._response = raw_response self._raw_body = raw_body self._is_base64_body = base64_body self._body = None self._headers = None + self.extra_info = ResponseExtraInfo(extra_info or {}) def __getattr__(self, item): return self._response.get(item, None) @@ -374,3 +400,19 @@ class Response(object): self._body = self._raw_body return self._body + + +class ExtraInfo(object): + def __init__(self, extra_info): + self._extra_info = extra_info + + def __getattr__(self, item): + return self._extra_info.get(item, None) + + +class RequestExtraInfo(ExtraInfo): + pass + + +class ResponseExtraInfo(ExtraInfo): + pass diff --git a/DrissionPage/_units/network_listener.pyi b/DrissionPage/_units/network_listener.pyi index 75aab24..47cdb07 100644 --- a/DrissionPage/_units/network_listener.pyi +++ b/DrissionPage/_units/network_listener.pyi @@ -49,8 +49,12 @@ class NetworkListener(object): def _requestWillBeSent(self, **kwargs) -> None: ... + def _requestWillBeSentExtraInfo(self, **kwargs) -> None: ... + def _response_received(self, **kwargs) -> None: ... + def _responseReceivedExtraInfo(self, **kwargs) -> None: ... + def _loading_finished(self, **kwargs) -> None: ... def _loading_failed(self, **kwargs) -> None: ... @@ -64,11 +68,11 @@ class NetworkListener(object): class DataPacket(object): """返回的数据包管理类""" - def __init__(self, tab_id: str, target: Optional[str], raw_info: dict): + def __init__(self, tab_id: str, target: Optional[str]): self.tab_id: str = ... self.target: str = ... - self._raw_request: dict = ... - self._raw_response: dict = ... + self._raw_request: Optional[dict] = ... + self._raw_response: Optional[dict] = ... self._raw_post_data: str = ... self._raw_body: str = ... self._base64_body: bool = ... @@ -76,6 +80,8 @@ class DataPacket(object): self._response: Response = ... self.errorText: str = ... self._resource_type: str = ... + self._requestExtraInfo: Optional[dict] = ... + self._responseExtraInfo: Optional[dict] = ... @property def url(self) -> str: ... @@ -98,6 +104,7 @@ class DataPacket(object): class Request(object): url: str = ... + extra_info: Optional[RequestExtraInfo] = ... _headers: Union[CaseInsensitiveDict, None] = ... method: str = ... @@ -111,7 +118,7 @@ class Request(object): trustTokenParams = ... isSameSite = ... - def __init__(self, raw_request: dict, post_data: str): + def __init__(self, raw_request: dict, post_data: str, extra_info: Optional[dict]): self._request: dict = ... self._raw_post_data: str = ... self._postData: str = ... @@ -124,6 +131,7 @@ class Request(object): class Response(object): + extra_info: Optional[ResponseExtraInfo] = ... url = ... status = ... statusText = ... @@ -148,7 +156,7 @@ class Response(object): securityState = ... securityDetails = ... - def __init__(self, raw_response: dict, raw_body: str, base64_body: bool): + def __init__(self, raw_response: dict, raw_body: str, base64_body: bool, extra_info: Optional[dict]): self._response: dict = ... self._raw_body: str = ... self._is_base64_body: bool = ... @@ -163,3 +171,28 @@ class Response(object): @property def body(self) -> Union[str, dict, bool]: ... + + +class ExtraInfo(object): + def __init__(self, extra_info: dict): + self._extra_info: dict = ... + + +class RequestExtraInfo(ExtraInfo): + requestId: str = ... + associatedCookies: List[dict] = ... + headers: dict = ... + connectTiming: dict = ... + clientSecurityState: dict = ... + siteHasCookieInOtherPartition: bool = ... + + +class ResponseExtraInfo(ExtraInfo): + requestId: str = ... + blockedCookies: List[dict] = ... + headers: dict = ... + resourceIPAddressSpace: str = ... + statusCode: int = ... + headersText: str = ... + cookiePartitionKey: str = ... + cookiePartitionKeyOpaque: bool = ... From b8382e3e5fc49879c5b35c4463a02a52f9caa2d8 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 31 Oct 2023 15:20:45 +0800 Subject: [PATCH 059/182] =?UTF-8?q?4.0.0b4=E4=BF=AE=E5=A4=8D=E6=8A=93?= =?UTF-8?q?=E5=8C=85=E9=97=AE=E9=A2=98=EF=BC=9B=E5=88=9B=E5=BB=BAChromiumP?= =?UTF-8?q?age=E5=8F=AF=E6=8E=A5=E6=94=B6int=E8=A1=A8=E7=A4=BA=E7=AB=AF?= =?UTF-8?q?=E5=8F=A3=E5=8F=B7=EF=BC=9BFrame=20=5Freload()=E6=97=B6?= =?UTF-8?q?=E5=85=B3=E9=97=AD=E6=97=A7=E8=BF=9E=E6=8E=A5=EF=BC=9B=E4=BF=AE?= =?UTF-8?q?=E5=A4=8Ddriver=E9=98=BB=E6=96=AD=E9=97=AE=E9=A2=98=EF=BC=9B?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=BF=9E=E6=8E=A5=E8=AF=BB=E5=8F=96doc?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/chromium_driver.py | 6 +- DrissionPage/_pages/chromium_base.py | 4 +- DrissionPage/_pages/chromium_frame.py | 8 +-- DrissionPage/_pages/chromium_page.py | 10 ++- DrissionPage/_units/network_listener.py | 83 ++++++++++++------------ DrissionPage/_units/network_listener.pyi | 1 + setup.py | 2 +- 7 files changed, 57 insertions(+), 57 deletions(-) diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index 948cdc7..6732dc6 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -74,12 +74,12 @@ class ChromiumDriver(object): while not self._stopped.is_set(): try: - return self.method_results[message['id']].get(.2) + return self.method_results[message['id']].get(timeout=.2) except Empty: if self.alert_flag: self.alert_flag = False - return {'error': {'message': 'alert exists.'}} + return {'result': {'message': 'alert exists.'}} if timeout is not None and perf_counter() > timeout: return {'error': {'message': 'timeout'}} @@ -155,7 +155,7 @@ class ChromiumDriver(object): if self._stopped.is_set(): return {'error': 'tab closed', 'type': 'tab_closed'} - timeout = kwargs.pop("_timeout", 30) + timeout = kwargs.pop("_timeout", 20) result = self._send({"method": _method, "params": kwargs}, timeout=timeout) if result is None: return {'error': 'tab closed', 'type': 'tab_closed'} diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 6327634..2afb42f 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -85,7 +85,6 @@ class ChromiumBase(BasePage): if not tab_id: json = get(f'http://{self.address}/json', headers={'Connection': 'close'}).json() - tab_id = [i['id'] for i in json if i['type'] == 'page'] if not tab_id: raise BrowserConnectError('浏览器连接失败，可能是浏览器版本原因。') @@ -159,6 +158,7 @@ class ChromiumBase(BasePage): """页面开始加载时执行""" self.browser._frames[kwargs['frameId']] = self.tab_id if kwargs['frameId'] == self._frame_id: + self._doc_got = False self._ready_state = 'loading' self._is_loading = True if self.page_load_strategy == 'eager': @@ -171,6 +171,7 @@ class ChromiumBase(BasePage): def _onFrameNavigated(self, **kwargs): """页面跳转时执行""" if kwargs['frame']['id'] == self._frame_id: + self._doc_got = False self._ready_state = 'loading' self._is_loading = True if self._debug: @@ -200,7 +201,6 @@ class ChromiumBase(BasePage): if self._debug: print(f'FrameStoppedLoading {kwargs}') self._get_document() - self._doc_got = False def _onFileChooserOpened(self, **kwargs): """文件选择框打开时执行""" diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index fb8a047..5613f58 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -98,6 +98,7 @@ class ChromiumFrame(ChromiumBase): """重新获取document""" debug = self._debug d_debug = self.driver._debug + old_driver = self.driver if debug: print('重新获取document') @@ -131,6 +132,7 @@ class ChromiumFrame(ChromiumBase): else: raise GetDocumentError + old_driver.stop() self.wait.load_complete() def _check_ok(self): @@ -186,14 +188,8 @@ class ChromiumFrame(ChromiumBase): def _onInspectorDetached(self, **kwargs): self._is_loading = True - # print('reload') self._reload() - # def _onFrameDetached(self, **kwargs): - # if kwargs['frameId'] == self.frame_id: - # self._is_loading = True - # self._reload() - @property def page(self): return self._page diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 30d0e71..467ae1c 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -24,7 +24,7 @@ class ChromiumPage(ChromiumBase): def __init__(self, addr_or_opts=None, tab_id=None, timeout=None, addr_driver_opts=None): """ - :param addr_or_opts: 浏览器地址:端口或ChromiumOptions对象 + :param addr_or_opts: 浏览器地址:端口、ChromiumOptions对象或端口数字（int） :param tab_id: 要控制的标签页id，不指定默认为激活的 :param timeout: 超时时间 """ @@ -48,11 +48,14 @@ class ChromiumPage(ChromiumBase): elif isinstance(addr_or_opts, ChromiumOptions): self._driver_options = addr_or_opts - # 接收浏览器地址和端口 elif isinstance(addr_or_opts, str): self._driver_options = ChromiumOptions() self._driver_options.set_debugger_address(addr_or_opts) + elif isinstance(addr_or_opts, int): + self._driver_options = ChromiumOptions() + self._driver_options.set_local_port(addr_or_opts) + else: raise TypeError('只能接收ip:port格式或ChromiumOptions类型参数。') @@ -186,7 +189,8 @@ class ChromiumPage(ChromiumBase): :param switch_to: 新建标签页后是否把焦点移过去 :return: switch_to为False时返回新标签页对象，否则返回当前对象， """ - return self if switch_to else ChromiumTab(self, self._new_tab(url, switch_to)) + tid = self._new_tab(url, switch_to) + return self if switch_to else ChromiumTab(self, tid) def to_main_tab(self): """跳转到主标签页""" diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index 36f99f7..cd87112 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -26,6 +26,7 @@ class NetworkListener(object): self._caught = None # 临存捕捉到的数据 self._request_ids = None # 暂存须要拦截的请求id + self._extra_info_ids = None self.listening = False self._targets = None # 默认监听所有 @@ -81,6 +82,7 @@ class NetworkListener(object): self.listening = True self._request_ids = {} + self._extra_info_ids = {} self._caught = Queue(maxsize=0) self._set_callback() @@ -174,6 +176,7 @@ class NetworkListener(object): def clear(self): """清空结果""" self._request_ids = {} + self._extra_info_ids = {} self._caught.queue.clear() def _set_callback(self): @@ -188,59 +191,43 @@ class NetworkListener(object): def _requestWillBeSent(self, **kwargs): """接收到请求时的回调函数""" if not self._targets: - packet = self._request_ids.setdefault(kwargs['requestId'], DataPacket(self._page.tab_id, None)) - packet._raw_request = kwargs + rid = kwargs['requestId'] + p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, None)) + p._raw_request = kwargs if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): - packet._raw_post_data = self._driver.call_method('Network.getRequestPostData', - requestId=kwargs['requestId'])['postData'] - + p._raw_post_data = self._driver.call_method('Network.getRequestPostData', requestId=rid)['postData'] return + rid = kwargs['requestId'] for target in self._targets: if ((self._is_regex and search(target, kwargs['request']['url'])) or (not self._is_regex and target in kwargs['request']['url'])) and ( not self._method or kwargs['request']['method'] in self._method): - packet = self._request_ids.setdefault(kwargs['requestId'], DataPacket(self._page.tab_id, None)) - packet._raw__request = kwargs + p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, None)) + p._raw_request = kwargs if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): - packet._raw_post_data = self._driver.call_method('Network.getRequestPostData', - requestId=kwargs['requestId'])['postData'] - + p._raw_post_data = self._driver.call_method('Network.getRequestPostData', requestId=rid)['postData'] break def _requestWillBeSentExtraInfo(self, **kwargs): - if not self._targets: - packet = self._request_ids.setdefault(kwargs['requestId'], DataPacket(self._page.tab_id, None)) - packet._requestExtraInfo = kwargs - return - - for target in self._targets: - if ((self._is_regex and search(target, kwargs['request']['url'])) or - (not self._is_regex and target in kwargs['request']['url'])) and ( - not self._method or kwargs['request']['method'] in self._method): - packet = self._request_ids.setdefault(kwargs['requestId'], DataPacket(self._page.tab_id, None)) - packet._requestExtraInfo = kwargs - - break + self._extra_info_ids.setdefault(kwargs['requestId'], {})['request'] = kwargs def _response_received(self, **kwargs): """接收到返回信息时处理方法""" - request = self._request_ids.get(kwargs['requestId']) + request = self._request_ids.get(kwargs['requestId'], None) if request: request._raw_response = kwargs['response'] request._resource_type = kwargs['type'] def _responseReceivedExtraInfo(self, **kwargs): - request = self._request_ids.get(kwargs['requestId']) - if request: - request._responseExtraInfo = kwargs + self._extra_info_ids[kwargs['requestId']]['response'] = kwargs def _loading_finished(self, **kwargs): """请求完成时处理方法""" - request_id = kwargs['requestId'] - dp = self._request_ids.get(request_id) + r_id = kwargs['requestId'] + dp = self._request_ids.get(r_id) if dp: - r = self._driver.call_method('Network.getResponseBody', requestId=request_id) + r = self._driver.call_method('Network.getResponseBody', requestId=r_id) if 'body' in r: dp._raw_body = r['body'] dp._base64_body = r['base64Encoded'] @@ -248,25 +235,37 @@ class NetworkListener(object): dp._raw_body = '' dp._base64_body = False + ei = self._extra_info_ids.get(r_id, None) + if ei: + dp._requestExtraInfo = ei.get('request', None) + dp._responseExtraInfo = ei.get('response', None) + self._caught.put(dp) - try: - self._request_ids.pop(request_id) - except: - pass + + try: + self._request_ids.pop(r_id) + self._extra_info_ids.pop(r_id) + except: + pass def _loading_failed(self, **kwargs): """请求失败时的回调方法""" - request_id = kwargs['requestId'] - if request_id in self._request_ids: - dp = self._request_ids[request_id] + r_id = kwargs['requestId'] + dp = self._request_ids.get(r_id, None) + if dp: dp.errorText = kwargs['errorText'] dp._resource_type = kwargs['type'] - + ei = self._extra_info_ids.get(r_id, None) + if ei: + dp._requestExtraInfo = ei.get('request', None) + dp._responseExtraInfo = ei.get('response', None) self._caught.put(dp) - try: - self._request_ids.pop(request_id) - except: - pass + + try: + self._request_ids.pop(r_id) + self._extra_info_ids.pop(r_id) + except: + pass class DataPacket(object): diff --git a/DrissionPage/_units/network_listener.pyi b/DrissionPage/_units/network_listener.pyi index 47cdb07..e261483 100644 --- a/DrissionPage/_units/network_listener.pyi +++ b/DrissionPage/_units/network_listener.pyi @@ -21,6 +21,7 @@ class NetworkListener(object): self._is_regex: bool = ... self._driver: ChromiumDriver = ... self._request_ids: dict = ... + self._extra_info_ids: dict = ... self.listening: bool = ... @property diff --git a/setup.py b/setup.py index 76574fa..24c2142 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b3", + version="4.0.0b4", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From 5f0ef46b93b46bd4fb80dd66462e7619510dc89e Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 31 Oct 2023 19:57:37 +0800 Subject: [PATCH 060/182] =?UTF-8?q?4.0.0b5=E4=BF=AE=E5=A4=8D=E5=87=A0?= =?UTF-8?q?=E4=B8=AA=E9=9A=90=E8=97=8F=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 5 +++-- DrissionPage/_base/chromium_driver.pyi | 3 +++ DrissionPage/_commons/web.py | 2 +- DrissionPage/_pages/chromium_page.py | 1 + DrissionPage/_units/network_listener.py | 6 ++++-- setup.py | 2 +- 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 4c7aee0..1d814cb 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -84,8 +84,9 @@ class Browser(object): @property def tabs(self): """返回所有标签页id组成的列表""" - j = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp - return [i['id'] for i in j if i['type'] == 'page'] + # j = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp + j = self.run_cdp('Target.getTargets')['targetInfos'] + return [i['targetId'] for i in j if i['type'] == 'page'] @property def process_id(self): diff --git a/DrissionPage/_base/chromium_driver.pyi b/DrissionPage/_base/chromium_driver.pyi index 378879b..fac8ab5 100644 --- a/DrissionPage/_base/chromium_driver.pyi +++ b/DrissionPage/_base/chromium_driver.pyi @@ -7,6 +7,7 @@ from queue import Queue from threading import Thread, Event from typing import Union, Callable, Dict, Optional +from requests import Response from websocket import WebSocket @@ -57,3 +58,5 @@ class ChromiumDriver(object): class BrowserDriver(ChromiumDriver): BROWSERS: Dict[str, ChromiumDriver] = ... + + def get(self, url) -> Response: ... diff --git a/DrissionPage/_commons/web.py b/DrissionPage/_commons/web.py index 5a438f3..543d222 100644 --- a/DrissionPage/_commons/web.py +++ b/DrissionPage/_commons/web.py @@ -54,7 +54,7 @@ def get_ele_txt(e): if sub('[ \n\t\r]', '', el) != '': # 字符除了回车和空格还有其它内容 txt = el if not pre: - txt = txt.replace('\n', ' ').strip(' ') + txt = txt.replace('\r\n', ' ').replace('\n', ' ').strip(' ') txt = sub(r' {2,}', ' ', txt) str_list.append(txt) diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 467ae1c..2017eee 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -264,6 +264,7 @@ class ChromiumPage(ChromiumBase): for tab in tabs: self.browser.close_tab(tab) + sleep(.2) while len(self.tabs) != end_len: sleep(.1) diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index cd87112..9732330 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -203,7 +203,7 @@ class NetworkListener(object): if ((self._is_regex and search(target, kwargs['request']['url'])) or (not self._is_regex and target in kwargs['request']['url'])) and ( not self._method or kwargs['request']['method'] in self._method): - p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, None)) + p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, target)) p._raw_request = kwargs if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): p._raw_post_data = self._driver.call_method('Network.getRequestPostData', requestId=rid)['postData'] @@ -220,7 +220,9 @@ class NetworkListener(object): request._resource_type = kwargs['type'] def _responseReceivedExtraInfo(self, **kwargs): - self._extra_info_ids[kwargs['requestId']]['response'] = kwargs + r = self._extra_info_ids.get(kwargs['requestId']) + if r: + r['response'] = kwargs def _loading_finished(self, **kwargs): """请求完成时处理方法""" diff --git a/setup.py b/setup.py index 24c2142..780d1d4 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b4", + version="4.0.0b5", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From f278a32ede0a9f2c5225cb4f30db8c3d101cf4cf Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 1 Nov 2023 02:02:59 +0800 Subject: [PATCH 061/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dframe=5Fid=E4=B8=8D?= =?UTF-8?q?=E5=AF=B9=E5=BA=94=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 ++ DrissionPage/_base/browser.py | 5 ++--- DrissionPage/_base/chromium_driver.py | 12 +++++++----- DrissionPage/_pages/chromium_base.py | 10 +++++----- DrissionPage/_pages/chromium_frame.py | 2 ++ 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 8a2eaf3..ad912d6 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -11,3 +11,5 @@ from ._pages.web_page import WebPage # 启动配置类 from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions + +__all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage'] diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 1d814cb..4c7aee0 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -84,9 +84,8 @@ class Browser(object): @property def tabs(self): """返回所有标签页id组成的列表""" - # j = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp - j = self.run_cdp('Target.getTargets')['targetInfos'] - return [i['targetId'] for i in j if i['type'] == 'page'] + j = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp + return [i['id'] for i in j if i['type'] == 'page'] @property def process_id(self): diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index 6732dc6..a8181e6 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -137,11 +137,11 @@ class ChromiumDriver(object): function = self.event_handlers.get(event['method']) if function: - if self._debug: - print(f'开始执行 {function.__name__}') + # if self._debug: + # print(f'开始执行 {function.__name__}') function(**event['params']) - if self._debug: - print(f'执行 {function.__name__}完毕') + # if self._debug: + # print(f'执行 {function.__name__}完毕') self.event_queue.task_done() @@ -225,4 +225,6 @@ class BrowserDriver(ChromiumDriver): return f"" def get(self, url): - return get(url, headers={'Connection': 'close'}) + r = get(url, headers={'Connection': 'close'}) + r.close() + return r diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 2afb42f..2225a44 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -95,17 +95,12 @@ class ChromiumBase(BasePage): self._get_document() self._ready_state = 'complete' - r = self.run_cdp('Page.getFrameTree') - for i in findall(r"'id': '(.*?)'", str(r)): - self.browser._frames[i] = self.tab_id - def _driver_init(self, tab_id): """新建页面、页面刷新、切换标签页后要进行的cdp参数初始化 :param tab_id: 要跳转到的标签页id :return: None """ self._is_loading = True - self._frame_id = tab_id self._driver = ChromiumDriver(tab_id=tab_id, tab_type='page', address=self.address) self._alert = Alert() self._driver.set_listener('Page.javascriptDialogOpening', self._on_alert_open) @@ -123,6 +118,11 @@ class ChromiumBase(BasePage): self._driver.set_listener('Page.frameAttached', self._onFrameAttached) self._driver.set_listener('Page.frameDetached', self._onFrameDetached) + r = self.run_cdp('Page.getFrameTree') + for i in findall(r"'id': '(.*?)'", str(r)): + self.browser._frames[i] = self.tab_id + self._frame_id = r['frameTree']['frame']['id'] + def _get_document(self): if self._is_reading: return diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 5613f58..aeba255 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -46,6 +46,7 @@ class ChromiumFrame(ChromiumBase): self._is_diff_domain = False self.doc_ele = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) super().__init__(page.address, page.tab_id, page.timeout) + self._frame_id = self.frame_id else: self._is_diff_domain = True super().__init__(page.address, self.frame_id, page.timeout) @@ -113,6 +114,7 @@ class ChromiumFrame(ChromiumBase): self.doc_ele = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) super().__init__(self.address, self._target_page.tab_id, self._target_page.timeout) + self._frame_id = self.frame_id self._debug = debug self.driver._debug = d_debug else: From e8f3cf8df34c203a141c779153f9edfcc5124762 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 1 Nov 2023 23:32:14 +0800 Subject: [PATCH 062/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dget()=E8=B6=85?= =?UTF-8?q?=E6=97=B6=E6=B2=A1try=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 10 +++++----- DrissionPage/_pages/chromium_frame.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 2225a44..4a3da72 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -862,13 +862,13 @@ class ChromiumBase(BasePage): for t in range(times + 1): err = None end_time = perf_counter() + timeout - result = self.run_cdp('Page.navigate', url=to_url, _timeout=timeout) - if result.get('error') == 'timeout': + try: + result = self.run_cdp('Page.navigate', url=to_url, _timeout=timeout) + if 'errorText' in result: + err = ConnectionError(result['errorText']) + except TimeoutError: err = TimeoutError('页面连接超时。') - elif 'errorText' in result: - err = ConnectionError(result['errorText']) - if err: sleep(interval) if self._debug or show_errmsg: diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index aeba255..ac6f03d 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -611,13 +611,13 @@ class ChromiumFrame(ChromiumBase): for t in range(times + 1): err = None end_time = perf_counter() + timeout - result = self.driver.call_method('Page.navigate', url=to_url, frameId=self.frame_id, _timeout=timeout) - if result.get('error') == 'timeout': + try: + result = self.run_cdp('Page.navigate', url=to_url, _timeout=timeout) + if 'errorText' in result: + err = ConnectionError(result['errorText']) + except TimeoutError: err = TimeoutError('页面连接超时。') - elif 'errorText' in result: - err = ConnectionError(result['errorText']) - if err: sleep(interval) if self._debug or show_errmsg: From 2a0fa73f9d427bb966972e94449a7adc2d561519 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 2 Nov 2023 23:06:29 +0800 Subject: [PATCH 063/182] =?UTF-8?q?=E6=8E=A7=E5=88=B6tabs=E5=87=8F?= =?UTF-8?q?=E5=B0=91=E7=94=A8get()=E6=96=B9=E5=BC=8F=EF=BC=8C=E6=8F=90?= =?UTF-8?q?=E9=AB=98=E7=A8=B3=E5=AE=9A=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 9 +++++---- DrissionPage/_pages/chromium_frame.py | 2 +- DrissionPage/_pages/chromium_page.py | 13 ++----------- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 4c7aee0..7c8f0e9 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -79,12 +79,13 @@ class Browser(object): @property def tabs_count(self): """返回标签页数量""" - return len(self.tabs) + j = self.run_cdp('Target.getTargets')['targetInfos'] # 不要改用get，避免卡死 + return len([i for i in j if i['type'] == 'page']) @property def tabs(self): """返回所有标签页id组成的列表""" - j = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp + j = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp，因为顺序不对 return [i['id'] for i in j if i['type'] == 'page'] @property @@ -118,14 +119,14 @@ class Browser(object): :param tab_id: 标签页id :return: None """ - self._driver.get(f'http://{self.address}/json/close/{tab_id}') + self.run_cdp('Target.closeTarget', targetId=tab_id) def activate_tab(self, tab_id): """使标签页变为活动状态 :param tab_id: 标签页id :return: None """ - self._driver.get(f'http://{self.address}/json/activate/{tab_id}') + self.run_cdp('Target.activateTarget', targetId=tab_id) def get_window_bounds(self, tab_id=None): """返回浏览器窗口位置和大小信息 diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index ac6f03d..3ca9b25 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -91,7 +91,7 @@ class ChromiumFrame(ChromiumBase): try: super()._driver_init(tab_id) except: - get(f'http://{self.address}/json', headers={'Connection': 'close'}) + self.browser.driver.get(f'http://{self.address}/json') super()._driver_init(tab_id) self.driver.set_listener('Inspector.detached', self._onInspectorDetached) diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 2017eee..e19fbb6 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -161,17 +161,8 @@ class ChromiumPage(ChromiumBase): :return: 新标签页的id """ if switch_to: - begin_tabs = set(self.tabs) - len_tabs = len(begin_tabs) tid = self.run_cdp('Target.createTarget', url='')['targetId'] - - tabs = self.tabs - while len(tabs) == len_tabs: - tabs = self.tabs - sleep(.005) - - new_tab = set(tabs) - begin_tabs - self._to_tab(new_tab.pop(), read_doc=False) + self._to_tab(tid, read_doc=False) if url: self.get(url) @@ -265,7 +256,7 @@ class ChromiumPage(ChromiumBase): for tab in tabs: self.browser.close_tab(tab) sleep(.2) - while len(self.tabs) != end_len: + while self.tabs_count != end_len: sleep(.1) if self._main_tab in tabs: From 2ddb7aff952c841204522b4625d51ebb7a0438f7 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sat, 4 Nov 2023 17:18:28 +0800 Subject: [PATCH 064/182] =?UTF-8?q?=E9=80=82=E5=BA=94=E6=9C=80=E6=96=B0?= =?UTF-8?q?=E7=89=88=E6=B5=8F=E8=A7=88=E5=99=A8=E7=9A=84=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=EF=BC=8Ctabs=E4=B8=8D=E5=8C=85=E5=90=ABF12=E7=9A=84=E7=AA=97?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 4 ++-- DrissionPage/_pages/chromium_base.py | 12 +++--------- DrissionPage/_units/clicker.py | 3 +-- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 7c8f0e9..35ed1f0 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -80,13 +80,13 @@ class Browser(object): def tabs_count(self): """返回标签页数量""" j = self.run_cdp('Target.getTargets')['targetInfos'] # 不要改用get，避免卡死 - return len([i for i in j if i['type'] == 'page']) + return len([i for i in j if i['type'] == 'page' and not i['url'].startswith('devtools://')]) @property def tabs(self): """返回所有标签页id组成的列表""" j = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp，因为顺序不对 - return [i['id'] for i in j if i['type'] == 'page'] + return [i['id'] for i in j if i['type'] == 'page' and not i['url'].startswith('devtools://')] @property def process_id(self): diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 4a3da72..79a421d 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -10,8 +10,6 @@ from re import findall from threading import Thread from time import perf_counter, sleep -from requests import get - from .._base.base import BasePage from .._base.chromium_driver import ChromiumDriver from .._commons.constants import ERROR, NoneElement @@ -25,8 +23,8 @@ from .._units.network_listener import NetworkListener from .._units.screencast import Screencast from .._units.setter import ChromiumBaseSetter from .._units.waiter import ChromiumBaseWaiter -from ..errors import (ContextLossError, ElementLossError, CDPError, TabClosedError, NoRectError, BrowserConnectError, - AlertExistsError, GetDocumentError) +from ..errors import (ContextLossError, ElementLossError, CDPError, TabClosedError, NoRectError, AlertExistsError, + GetDocumentError) class ChromiumBase(BasePage): @@ -84,11 +82,7 @@ class ChromiumBase(BasePage): self._scroll = None if not tab_id: - json = get(f'http://{self.address}/json', headers={'Connection': 'close'}).json() - tab_id = [i['id'] for i in json if i['type'] == 'page'] - if not tab_id: - raise BrowserConnectError('浏览器连接失败，可能是浏览器版本原因。') - tab_id = tab_id[0] + tab_id = self.browser.tabs[0] self._driver_init(tab_id) if self.ready_state == 'complete' and self._ready_state is None: diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index c075d35..9a8659a 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -32,7 +32,7 @@ class Clicker(object): :param timeout: 模拟点击的超时时间，等待元素可见、不被遮挡、进入视口 :return: 是否点击成功 """ - if not by_js: + if not by_js: # 模拟点击 try: self._ele.scroll.to_see() can_click = False @@ -65,7 +65,6 @@ class Clicker(object): return True if Settings.raise_when_click_failed: raise CanNotClickError - return False def right(self): From 897a9cd9c8a3bc179e0b46a5d6de4cfaaf648197 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 5 Nov 2023 16:57:10 +0800 Subject: [PATCH 065/182] =?UTF-8?q?set.download=5Ffile=5Fname()=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0suffix=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_configs/options_manage.py | 3 ++- DrissionPage/_units/download_manager.py | 30 +++++++++++++++++------- DrissionPage/_units/download_manager.pyi | 3 ++- DrissionPage/_units/setter.py | 30 +++++++++++++++--------- DrissionPage/_units/setter.pyi | 18 +++++++++----- 5 files changed, 57 insertions(+), 27 deletions(-) diff --git a/DrissionPage/_configs/options_manage.py b/DrissionPage/_configs/options_manage.py index 5f2b3e5..0be5b8f 100644 --- a/DrissionPage/_configs/options_manage.py +++ b/DrissionPage/_configs/options_manage.py @@ -26,7 +26,8 @@ class OptionsManager(object): self.ini_path = str(path) if not Path(self.ini_path).exists(): - raise FileNotFoundError('ini文件不存在。') + raise FileNotFoundError('\nini文件不存在。\n如果是打包使用，请查看打包注意事项\nhttps://g1879.gitee.io/drission' + 'pagedocs/advance/packaging/') self._conf = RawConfigParser() self._conf.read(self.ini_path, encoding='utf-8') diff --git a/DrissionPage/_units/download_manager.py b/DrissionPage/_units/download_manager.py index 09b8afb..5cd27a8 100644 --- a/DrissionPage/_units/download_manager.py +++ b/DrissionPage/_units/download_manager.py @@ -48,13 +48,16 @@ class BrowserDownloadManager(object): self._browser.run_cdp('Browser.setDownloadBehavior', downloadPath=str(Path(path).absolute()), behavior='allowAndName', eventsEnabled=True) - def set_rename(self, tab_id, rename): + def set_rename(self, tab_id, rename=None, suffix=None): """设置某个tab的重命名文件名 :param tab_id: tab id - :param rename: 文件名 + :param rename: 文件名，可不含后缀，会自动使用远程文件后缀 + :param suffix: 后缀名，显式设置后缀名，不使用远程文件后缀 :return: None """ - TabDownloadSettings(tab_id).rename = rename + ts = TabDownloadSettings(tab_id) + ts.rename = rename + ts.suffix = suffix def set_file_exists(self, tab_id, mode): """设置某个tab下载文件重名时执行的策略 @@ -135,12 +138,22 @@ class BrowserDownloadManager(object): settings = TabDownloadSettings(tab_id if tab_id in TabDownloadSettings.TABS else self._page.tab_id) if settings.rename: - tmp = kwargs['suggestedFilename'].rsplit('.', 1) - ext_name = tmp[-1] if len(tmp) > 1 else '' - tmp = settings.rename.rsplit('.', 1) - ext_rename = tmp[-1] if len(tmp) > 1 else '' - name = settings.rename if ext_rename == ext_name else f'{settings.rename}.{ext_name}' + if settings.suffix is not None: + name = f'{settings.rename}.{settings.suffix}' + + else: + tmp = kwargs['suggestedFilename'].rsplit('.', 1) + ext_name = tmp[-1] if len(tmp) > 1 else '' + tmp = settings.rename.rsplit('.', 1) + ext_rename = tmp[-1] if len(tmp) > 1 else '' + name = settings.rename if ext_rename == ext_name else f'{settings.rename}.{ext_name}' + settings.rename = None + settings.suffix = None + + elif settings.suffix is not None: + name = f'{kwargs["suggestedFilename"].rsplit(".", 1)[0]}.{settings.suffix}' + settings.suffix = None else: name = kwargs['suggestedFilename'] @@ -210,6 +223,7 @@ class TabDownloadSettings(object): self._created = True self.tab_id = tab_id self.rename = None + self.suffix = None self.path = '' self.when_file_exists = 'rename' diff --git a/DrissionPage/_units/download_manager.pyi b/DrissionPage/_units/download_manager.pyi index 5153aa6..c77be03 100644 --- a/DrissionPage/_units/download_manager.pyi +++ b/DrissionPage/_units/download_manager.pyi @@ -19,7 +19,7 @@ class BrowserDownloadManager(object): def set_path(self, tab_id: str, path: Union[Path, str]) -> None: ... - def set_rename(self, tab_id: str, rename: str) -> None: ... + def set_rename(self, tab_id: str, rename: str = None, suffix: str = None) -> None: ... def set_file_exists(self, tab_id: str, mode: Literal['rename', 'skip', 'overwrite']) -> None: ... @@ -47,6 +47,7 @@ class TabDownloadSettings(object): tab_id: str = ... waiting_flag: Optional[bool, dict] = ... rename: Optional[str] = ... + suffix: Optional[str] = ... path: Optional[str] = ... when_file_exists: str = ... diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index 841aea3..d8bf2af 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -137,12 +137,13 @@ class TabSetter(ChromiumBaseSetter): if self._page._DownloadKit: self._page._DownloadKit.set.goal_path(path) - def download_file_name(self, name): + def download_file_name(self, name=None, suffix=None): """设置下一个被下载文件的名称 - :param name: 文件名，可不含后缀 + :param name: 文件名，可不含后缀，会自动使用远程文件后缀 + :param suffix: 后缀名，显式设置后缀名，不使用远程文件后缀 :return: None """ - self._page.browser._dl_mgr.set_rename(self._page.tab_id, name) + self._page.browser._dl_mgr.set_rename(self._page.tab_id, name, suffix) def when_download_file_exists(self, mode): """设置当存在同名文件时的处理方式 @@ -181,6 +182,11 @@ class ChromiumPageSetter(TabSetter): tab_or_id = tab_or_id.tab_id self._page.browser.activate_tab(tab_or_id) + @property + def window(self): + """返回用于设置浏览器窗口的对象""" + return PageWindowSetter(self._page) + class SessionPageSetter(object): def __init__(self, page): @@ -558,14 +564,6 @@ class WindowSetter(object): y = y if y is not None else info['top'] self._perform({'left': x - 8, 'top': y}) - def hide(self): - """隐藏浏览器窗口，只在Windows系统可用""" - show_or_hide_browser(self._page, hide=True) - - def show(self): - """显示浏览器窗口，只在Windows系统可用""" - show_or_hide_browser(self._page, hide=False) - def _get_info(self): """获取窗口位置及大小信息""" return self._page.run_cdp('Browser.getWindowForTarget') @@ -576,3 +574,13 @@ class WindowSetter(object): :return: None """ self._page.run_cdp('Browser.setWindowBounds', windowId=self._window_id, bounds=bounds) + + +class PageWindowSetter(WindowSetter): + def hide(self): + """隐藏浏览器窗口，只在Windows系统可用""" + show_or_hide_browser(self._page, hide=True) + + def show(self): + """显示浏览器窗口，只在Windows系统可用""" + show_or_hide_browser(self._page, hide=False) diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index e98b7de..e337d3b 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -61,7 +61,7 @@ class TabSetter(ChromiumBaseSetter): def download_path(self, path: Union[str, Path]) -> None: ... - def download_file_name(self, name: str) -> None: ... + def download_file_name(self, name: str = None, suffix: str = None) -> None: ... def when_download_file_exists(self, mode: FILE_EXISTS) -> None: ... @@ -71,6 +71,9 @@ class TabSetter(ChromiumBaseSetter): class ChromiumPageSetter(TabSetter): _page: ChromiumPage = ... + @property + def window(self) -> PageWindowSetter: ... + def main_tab(self, tab_id: str = None) -> None: ... def tab_to_front(self, tab_or_id: Union[str, ChromiumTab] = None) -> None: ... @@ -183,7 +186,6 @@ class PageScrollSetter(object): class WindowSetter(object): - def __init__(self, page: ChromiumBase): self._page: ChromiumBase = ... self._window_id: str = ... @@ -200,10 +202,14 @@ class WindowSetter(object): def location(self, x: int = None, y: int = None) -> None: ... - def hide(self) -> None: ... - - def show(self) -> None: ... - def _get_info(self) -> dict: ... def _perform(self, bounds: dict) -> None: ... + + +class PageWindowSetter(WindowSetter): + _page: ChromiumPage = ... + + def hide(self) -> None: ... + + def show(self) -> None: ... From 572f0d51c4bc0de3516bc7b72f34bf0f8b07f28b Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 5 Nov 2023 21:17:20 +0800 Subject: [PATCH 066/182] =?UTF-8?q?set.download=5Ffile=5Fname()=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0suffix=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_units/download_manager.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DrissionPage/_units/download_manager.py b/DrissionPage/_units/download_manager.py index 5cd27a8..0010119 100644 --- a/DrissionPage/_units/download_manager.py +++ b/DrissionPage/_units/download_manager.py @@ -139,7 +139,7 @@ class BrowserDownloadManager(object): settings = TabDownloadSettings(tab_id if tab_id in TabDownloadSettings.TABS else self._page.tab_id) if settings.rename: if settings.suffix is not None: - name = f'{settings.rename}.{settings.suffix}' + name = f'{settings.rename}.{settings.suffix}' if settings.suffix else settings.rename else: tmp = kwargs['suggestedFilename'].rsplit('.', 1) @@ -152,7 +152,9 @@ class BrowserDownloadManager(object): settings.suffix = None elif settings.suffix is not None: - name = f'{kwargs["suggestedFilename"].rsplit(".", 1)[0]}.{settings.suffix}' + name = kwargs["suggestedFilename"].rsplit(".", 1)[0] + if settings.suffix: + name = f'{name}.{settings.suffix}' settings.suffix = None else: From 9331721b44c828f58ec3ab82451864b75bb51c12 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 5 Nov 2023 22:33:28 +0800 Subject: [PATCH 067/182] =?UTF-8?q?=E8=B0=83=E6=95=B4ChromiumDriver?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/chromium_driver.py | 47 +++++++++++++-------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index a8181e6..83309a4 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -49,10 +49,9 @@ class ChromiumDriver(object): :param timeout: 超时时间，为None表示无限 :return: 浏览器返回的数据 """ - if 'id' not in message: - self._cur_id += 1 - message['id'] = self._cur_id - + self._cur_id += 1 + ws_id = self._cur_id + message['id'] = ws_id message_json = dumps(message) if self._debug: @@ -68,35 +67,35 @@ class ChromiumDriver(object): if timeout is not None: timeout = perf_counter() + timeout + self.method_results[ws_id] = Queue() try: - self.method_results[message['id']] = Queue() self._ws.send(message_json) - - while not self._stopped.is_set(): - try: - return self.method_results[message['id']].get(timeout=.2) - - except Empty: - if self.alert_flag: - self.alert_flag = False - return {'result': {'message': 'alert exists.'}} - - if timeout is not None and perf_counter() > timeout: - return {'error': {'message': 'timeout'}} - - continue - - except Exception: + except OSError: + self.method_results.pop(ws_id) return None - finally: - self.method_results.pop(message['id'], None) + while not self._stopped.is_set(): + try: + return self.method_results[ws_id].get(timeout=.2) + + except Empty: + if self.alert_flag: + self.alert_flag = False + return {'result': {'message': 'alert exists.'}} + + elif timeout is not None and perf_counter() > timeout: + return {'error': {'message': 'timeout'}} + + continue + + finally: + self.method_results.pop(ws_id) def _recv_loop(self): """接收浏览器信息的守护线程方法""" while not self._stopped.is_set(): try: - self._ws.settimeout(1) + # self._ws.settimeout(1) msg_json = self._ws.recv() msg = loads(msg_json) except WebSocketTimeoutException: From 99dfaf91da5443f1c3a0237971370ba19810fb8f Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 5 Nov 2023 23:05:27 +0800 Subject: [PATCH 068/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dbrowser=E7=9A=84?= =?UTF-8?q?=E4=B8=80=E4=BA=9B=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 7 +++---- DrissionPage/_base/browser.pyi | 2 +- DrissionPage/_units/download_manager.py | 15 ++++++++++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 35ed1f0..5593260 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -47,16 +47,15 @@ class Browser(object): self._process_id = i['id'] break - self.run_cdp('Target.setDiscoverTargets') + self.run_cdp('Target.setDiscoverTargets', discover=True) self._driver.set_listener('Target.targetDestroyed', self._onTargetDestroyed) def _onTargetDestroyed(self, **kwargs): """标签页关闭时执行""" tab_id = kwargs['targetId'] self._dl_mgr.clear_tab_info(tab_id) - for k, i in self._frames.items(): - if i == tab_id: - self._frames.pop(k) + for item in [(k, i) for k, i in self._frames.items() if i == tab_id]: + self._frames.pop(item[0]) def connect_to_page(self): """执行与page相关的逻辑""" diff --git a/DrissionPage/_base/browser.pyi b/DrissionPage/_base/browser.pyi index 8146d35..ee5edad 100644 --- a/DrissionPage/_base/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -50,6 +50,6 @@ class Browser(object): def connect_to_page(self) -> None: ... - def _onTargetDestroyed(self, **kwargs): ... + def _onTargetDestroyed(self, **kwargs) -> None: ... def quit(self) -> None: ... diff --git a/DrissionPage/_units/download_manager.py b/DrissionPage/_units/download_manager.py index 0010119..0fbd75f 100644 --- a/DrissionPage/_units/download_manager.py +++ b/DrissionPage/_units/download_manager.py @@ -127,9 +127,18 @@ class BrowserDownloadManager(object): :param tab_id: 标签页id :return: None """ - self._tab_missions.pop(tab_id) - self._flags.pop(tab_id) - TabDownloadSettings.TABS.pop(tab_id) + try: + self._tab_missions.pop(tab_id) + except KeyError: + pass + try: + self._flags.pop(tab_id) + except KeyError: + pass + try: + TabDownloadSettings.TABS.pop(tab_id) + except KeyError: + pass def _onDownloadWillBegin(self, **kwargs): """用于获取弹出新标签页触发的下载任务""" From f4e549a41411f10a26e16e61b4d8b0777ab1926b Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 6 Nov 2023 20:17:41 +0800 Subject: [PATCH 069/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=9B=91=E5=90=AC?= =?UTF-8?q?=E5=99=A8=E8=AE=BE=E7=BD=AEmethod=E6=97=B6=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98=EF=BC=9B=E4=BF=AE=E5=A4=8DDriver=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=9Bpop()=E5=85=A8=E9=83=A8=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0None?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/chromium_driver.py | 17 ++++---- DrissionPage/_commons/web.py | 4 +- DrissionPage/_configs/chromium_options.py | 2 +- DrissionPage/_configs/session_options.py | 3 +- DrissionPage/_units/download_manager.py | 17 ++------ DrissionPage/_units/network_listener.py | 47 +++++++++++------------ 6 files changed, 40 insertions(+), 50 deletions(-) diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index 83309a4..b9f1c9e 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -71,26 +71,29 @@ class ChromiumDriver(object): try: self._ws.send(message_json) except OSError: - self.method_results.pop(ws_id) + self.method_results.pop(ws_id, None) return None while not self._stopped.is_set(): try: - return self.method_results[ws_id].get(timeout=.2) + result = self.method_results[ws_id].get(timeout=.2) + self.method_results.pop(ws_id, None) + return result except Empty: if self.alert_flag: self.alert_flag = False - return {'result': {'message': 'alert exists.'}} + result = {'result': {'message': 'alert exists.'}} + self.method_results.pop(ws_id, None) + return result elif timeout is not None and perf_counter() > timeout: - return {'error': {'message': 'timeout'}} + result = {'error': {'message': 'timeout'}} + self.method_results.pop(ws_id, None) + return result continue - finally: - self.method_results.pop(ws_id) - def _recv_loop(self): """接收浏览器信息的守护线程方法""" while not self._stopped.is_set(): diff --git a/DrissionPage/_commons/web.py b/DrissionPage/_commons/web.py index 543d222..9d9c7b5 100644 --- a/DrissionPage/_commons/web.py +++ b/DrissionPage/_commons/web.py @@ -169,8 +169,8 @@ def cookie_to_dict(cookie): """ if isinstance(cookie, Cookie): cookie_dict = cookie.__dict__.copy() - cookie_dict.pop('rfc2109') - cookie_dict.pop('_rest') + cookie_dict.pop('rfc2109', None) + cookie_dict.pop('_rest', None) return cookie_dict elif isinstance(cookie, dict): diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index 1ce072f..214ccd0 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -209,7 +209,7 @@ class ChromiumOptions(object): :param arg: 设置项名称 :return: 当前对象 """ - self._prefs.pop(arg) + self._prefs.pop(arg, None) return self def remove_pref_from_file(self, arg): diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index 4339b08..d918f63 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -161,8 +161,7 @@ class SessionOptions(object): return self attr = attr.lower() - if attr in self._headers: - self._headers.pop(attr) + self._headers.pop(attr, None) return self diff --git a/DrissionPage/_units/download_manager.py b/DrissionPage/_units/download_manager.py index 0fbd75f..a36d362 100644 --- a/DrissionPage/_units/download_manager.py +++ b/DrissionPage/_units/download_manager.py @@ -101,7 +101,7 @@ class BrowserDownloadManager(object): mission.final_path = final_path if mission.tab_id in self._tab_missions and mission.id in self._tab_missions[mission.tab_id]: self._tab_missions[mission.tab_id].remove(mission.id) - self._missions.pop(mission.id) + self._missions.pop(mission.id, None) mission._is_done = True def cancel(self, mission): @@ -127,18 +127,9 @@ class BrowserDownloadManager(object): :param tab_id: 标签页id :return: None """ - try: - self._tab_missions.pop(tab_id) - except KeyError: - pass - try: - self._flags.pop(tab_id) - except KeyError: - pass - try: - TabDownloadSettings.TABS.pop(tab_id) - except KeyError: - pass + self._tab_missions.pop(tab_id, None) + self._flags.pop(tab_id, None) + TabDownloadSettings.TABS.pop(tab_id, None) def _onDownloadWillBegin(self, **kwargs): """用于获取弹出新标签页触发的下载任务""" diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index 9732330..2b2214a 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -72,7 +72,7 @@ class NetworkListener(object): :param method: 设置监听的请求类型，可用list等指定多个，为None时监听全部 :return: None """ - if targets: + if targets or method: self.set_targets(targets, is_regex, method) if self.listening: return @@ -191,23 +191,26 @@ class NetworkListener(object): def _requestWillBeSent(self, **kwargs): """接收到请求时的回调函数""" if not self._targets: - rid = kwargs['requestId'] - p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, None)) - p._raw_request = kwargs - if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): - p._raw_post_data = self._driver.call_method('Network.getRequestPostData', requestId=rid)['postData'] - return - - rid = kwargs['requestId'] - for target in self._targets: - if ((self._is_regex and search(target, kwargs['request']['url'])) or - (not self._is_regex and target in kwargs['request']['url'])) and ( - not self._method or kwargs['request']['method'] in self._method): - p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, target)) + if not self._method or kwargs['request']['method'] in self._method: + rid = kwargs['requestId'] + p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, None)) p._raw_request = kwargs if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): p._raw_post_data = self._driver.call_method('Network.getRequestPostData', requestId=rid)['postData'] - break + return + + else: + rid = kwargs['requestId'] + for target in self._targets: + if ((self._is_regex and search(target, kwargs['request']['url'])) or + (not self._is_regex and target in kwargs['request']['url'])) and ( + not self._method or kwargs['request']['method'] in self._method): + p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, target)) + p._raw_request = kwargs + if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): + p._raw_post_data = self._driver.call_method('Network.getRequestPostData', requestId=rid)[ + 'postData'] + break def _requestWillBeSentExtraInfo(self, **kwargs): self._extra_info_ids.setdefault(kwargs['requestId'], {})['request'] = kwargs @@ -244,11 +247,8 @@ class NetworkListener(object): self._caught.put(dp) - try: - self._request_ids.pop(r_id) - self._extra_info_ids.pop(r_id) - except: - pass + self._request_ids.pop(r_id, None) + self._extra_info_ids.pop(r_id, None) def _loading_failed(self, **kwargs): """请求失败时的回调方法""" @@ -263,11 +263,8 @@ class NetworkListener(object): dp._responseExtraInfo = ei.get('response', None) self._caught.put(dp) - try: - self._request_ids.pop(r_id) - self._extra_info_ids.pop(r_id) - except: - pass + self._request_ids.pop(r_id, None) + self._extra_info_ids.pop(r_id, None) class DataPacket(object): From ed8c53d7384abc501b1a7caa57a30a35556fc7ff Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 7 Nov 2023 16:15:01 +0800 Subject: [PATCH 070/182] =?UTF-8?q?=E5=85=83=E7=B4=A0=E5=A2=9E=E5=8A=A0loc?= =?UTF-8?q?ations.rect=E5=92=8Clocations.viewport=5Frect=E5=B1=9E=E6=80=A7?= =?UTF-8?q?=EF=BC=9B=E4=BC=98=E5=8C=96is=5Fcovered=E9=80=BB=E8=BE=91?= =?UTF-8?q?=EF=BC=9B=E4=BC=98=E5=8C=96Driver=20=5Fsend()=E9=80=BB=E8=BE=91?= =?UTF-8?q?=EF=BC=9B=E4=BF=AE=E5=A4=8DListener=20method=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=9B=E4=BC=98=E5=8C=96=E7=82=B9=E5=87=BB?= =?UTF-8?q?=E9=80=BB=E8=BE=91=EF=BC=9BDriver=E7=BB=9F=E4=B8=80=E5=9C=A8bro?= =?UTF-8?q?wser=E5=88=9B=E5=BB=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 24 +- DrissionPage/_base/browser.pyi | 7 +- DrissionPage/_base/chromium_driver.py | 17 +- DrissionPage/_commons/web.py | 4 +- DrissionPage/_configs/chromium_options.py | 2 +- DrissionPage/_configs/session_options.py | 3 +- DrissionPage/_elements/chromium_element.py | 367 ++------------------ DrissionPage/_elements/chromium_element.pyi | 117 +------ DrissionPage/_pages/chromium_base.py | 8 +- DrissionPage/_units/clicker.py | 54 +-- DrissionPage/_units/download_manager.py | 17 +- DrissionPage/_units/element_states.py | 100 ++++++ DrissionPage/_units/element_states.pyi | 50 +++ DrissionPage/_units/network_listener.py | 47 ++- DrissionPage/_units/select_element.py | 244 +++++++++++++ DrissionPage/_units/select_element.pyi | 63 ++++ 16 files changed, 593 insertions(+), 531 deletions(-) create mode 100644 DrissionPage/_units/element_states.py create mode 100644 DrissionPage/_units/element_states.pyi create mode 100644 DrissionPage/_units/select_element.py create mode 100644 DrissionPage/_units/select_element.pyi diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 5593260..46ad7c1 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -5,7 +5,7 @@ """ from time import sleep -from .chromium_driver import BrowserDriver +from .chromium_driver import BrowserDriver, ChromiumDriver from .._units.download_manager import BrowserDownloadManager @@ -38,6 +38,8 @@ class Browser(object): self._driver = BrowserDriver(browser_id, 'browser', address) self.id = browser_id self._frames = {} + self._drivers = {} + # self._drivers = {t: ChromiumDriver(t, 'page', address) for t in self.tabs} self._connected = False self._process_id = None @@ -49,13 +51,27 @@ class Browser(object): self.run_cdp('Target.setDiscoverTargets', discover=True) self._driver.set_listener('Target.targetDestroyed', self._onTargetDestroyed) + self._driver.set_listener('Target.targetCreated', self._onTargetCreated) + + def _get_driver(self, tab_id): + """获取对应tab id的ChromiumDriver + :param tab_id: 标签页id + :return: ChromiumDriver对象 + """ + return self._drivers.pop(tab_id, ChromiumDriver(tab_id, 'page', self.address)) + + def _onTargetCreated(self, **kwargs): + """标签页创建时执行""" + if kwargs['targetInfo']['type'] == 'page' and not kwargs['targetInfo']['url'].startswith('devtools://'): + self._drivers[kwargs['targetInfo']['targetId']] = ChromiumDriver(kwargs['targetInfo']['targetId'], 'page', self.address) def _onTargetDestroyed(self, **kwargs): """标签页关闭时执行""" tab_id = kwargs['targetId'] self._dl_mgr.clear_tab_info(tab_id) - for item in [(k, i) for k, i in self._frames.items() if i == tab_id]: - self._frames.pop(item[0]) + for key in [k for k, i in self._frames.items() if i == tab_id]: + self._frames.pop(key, None) + self._drivers.pop(tab_id, None) def connect_to_page(self): """执行与page相关的逻辑""" @@ -150,4 +166,4 @@ class Browser(object): break sleep(.2) - Browser.BROWSERS.pop(self.id) + Browser.BROWSERS.pop(self.id, None) diff --git a/DrissionPage/_base/browser.pyi b/DrissionPage/_base/browser.pyi index ee5edad..cb1a13a 100644 --- a/DrissionPage/_base/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -5,7 +5,7 @@ """ from typing import List, Optional, Union -from .chromium_driver import BrowserDriver +from .chromium_driver import BrowserDriver, ChromiumDriver from .._pages.chromium_page import ChromiumPage from .._units.download_manager import BrowserDownloadManager @@ -17,6 +17,7 @@ class Browser(object): id: str = ... address: str = ... _frames: dict = ... + _drivers: dict = ... _process_id: Optional[int] = ... _dl_mgr: BrowserDownloadManager = ... _connected: bool = ... @@ -25,6 +26,8 @@ class Browser(object): def __init__(self, address: str, browser_id: str, page: ChromiumPage): ... + def _get_driver(self, tab_id: str)->ChromiumDriver: ... + def run_cdp(self, cmd, **cmd_args) -> dict: ... @property @@ -50,6 +53,8 @@ class Browser(object): def connect_to_page(self) -> None: ... + def _onTargetCreated(self, **kwargs) -> None: ... + def _onTargetDestroyed(self, **kwargs) -> None: ... def quit(self) -> None: ... diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index 83309a4..b9f1c9e 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -71,26 +71,29 @@ class ChromiumDriver(object): try: self._ws.send(message_json) except OSError: - self.method_results.pop(ws_id) + self.method_results.pop(ws_id, None) return None while not self._stopped.is_set(): try: - return self.method_results[ws_id].get(timeout=.2) + result = self.method_results[ws_id].get(timeout=.2) + self.method_results.pop(ws_id, None) + return result except Empty: if self.alert_flag: self.alert_flag = False - return {'result': {'message': 'alert exists.'}} + result = {'result': {'message': 'alert exists.'}} + self.method_results.pop(ws_id, None) + return result elif timeout is not None and perf_counter() > timeout: - return {'error': {'message': 'timeout'}} + result = {'error': {'message': 'timeout'}} + self.method_results.pop(ws_id, None) + return result continue - finally: - self.method_results.pop(ws_id) - def _recv_loop(self): """接收浏览器信息的守护线程方法""" while not self._stopped.is_set(): diff --git a/DrissionPage/_commons/web.py b/DrissionPage/_commons/web.py index 543d222..9d9c7b5 100644 --- a/DrissionPage/_commons/web.py +++ b/DrissionPage/_commons/web.py @@ -169,8 +169,8 @@ def cookie_to_dict(cookie): """ if isinstance(cookie, Cookie): cookie_dict = cookie.__dict__.copy() - cookie_dict.pop('rfc2109') - cookie_dict.pop('_rest') + cookie_dict.pop('rfc2109', None) + cookie_dict.pop('_rest', None) return cookie_dict elif isinstance(cookie, dict): diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index 1ce072f..214ccd0 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -209,7 +209,7 @@ class ChromiumOptions(object): :param arg: 设置项名称 :return: 当前对象 """ - self._prefs.pop(arg) + self._prefs.pop(arg, None) return self def remove_pref_from_file(self, arg): diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index 4339b08..d918f63 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -161,8 +161,7 @@ class SessionOptions(object): return self attr = attr.lower() - if attr in self._headers: - self._headers.pop(attr) + self._headers.pop(attr, None) return self diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index e51a0d6..704c5d6 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -13,12 +13,14 @@ from .._commons.constants import FRAME_ELEMENT, NoneElement, Settings from .._commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions from .._commons.locator import get_loc from .._commons.tools import get_usable_path -from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, location_in_viewport, offset_scroll +from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, offset_scroll from .._units.clicker import Clicker +from .._units.element_states import ChromiumElementStates, ShadowRootStates +from .._units.select_element import SelectElement from .._units.setter import ChromiumElementSetter from .._units.waiter import ChromiumElementWaiter -from ..errors import ContextLossError, ElementLossError, JavaScriptError, ElementNotFoundError, \ - CDPError, NoResourceError, NoRectError, AlertExistsError +from ..errors import (ContextLossError, ElementLossError, JavaScriptError, ElementNotFoundError, + CDPError, NoResourceError, AlertExistsError) class ChromiumElement(DrissionElement): @@ -202,7 +204,7 @@ class ChromiumElement(DrissionElement): if self.tag != 'select': self._select = False else: - self._select = ChromiumSelect(self) + self._select = SelectElement(self) return self._select @@ -1436,103 +1438,6 @@ def send_key(ele, modifier, key): ele.page.run_cdp('Input.dispatchKeyEvent', **data) -class ChromiumElementStates(object): - def __init__(self, ele): - """ - :param ele: ChromiumElement - """ - self._ele = ele - - @property - def is_selected(self): - """返回元素是否被选择""" - return self._ele.run_js('return this.selected;') - - @property - def is_checked(self): - """返回元素是否被选择""" - return self._ele.run_js('return this.checked;') - - @property - def is_displayed(self): - """返回元素是否显示""" - return not (self._ele.style('visibility') == 'hidden' - or self._ele.run_js('return this.offsetParent === null;') - or self._ele.style('display') == 'none') - - @property - def is_enabled(self): - """返回元素是否可用""" - return not self._ele.run_js('return this.disabled;') - - @property - def is_alive(self): - """返回元素是否仍在DOM中""" - try: - d = self._ele.attrs - return True - except Exception: - return False - - @property - def is_in_viewport(self): - """返回元素是否出现在视口中，以元素click_point为判断""" - x, y = self._ele.locations.click_point - return location_in_viewport(self._ele.page, x, y) if x else False - - @property - def is_whole_in_viewport(self): - """返回元素是否整个都在视口内""" - x1, y1 = self._ele.location - w, h = self._ele.size - x2, y2 = x1 + w, y1 + h - return location_in_viewport(self._ele.page, x1, y1) and location_in_viewport(self._ele.page, x2, y2) - - @property - def is_covered(self): - """返回元素是否被覆盖，与是否在视口中无关""" - lx, ly = self._ele.locations.click_point - try: - r = self._ele.page.run_cdp('DOM.getNodeForLocation', x=lx, y=ly) - except CDPError: - return False - - if r.get('backendNodeId') != self._ele.ids.backend_id: - return True - - return False - - @property - def has_rect(self): - """返回元素是否拥有位置和大小，没有返回False，有返回大小元组""" - try: - return self._ele.size - except NoRectError: - return False - - -class ShadowRootStates(object): - def __init__(self, ele): - """ - :param ele: ChromiumElement - """ - self._ele = ele - - @property - def is_enabled(self): - """返回元素是否可用""" - return not self._ele.run_js('return this.disabled;') - - @property - def is_alive(self): - """返回元素是否仍在DOM中""" - try: - self._ele.page.run_cdp('DOM.describeNode', backendNodeId=self._ele.ids.backend_id) - return True - except Exception: - return False - - class Locations(object): def __init__(self, ele): """ @@ -1574,7 +1479,7 @@ class Locations(object): def viewport_click_point(self): """返回元素接受点击的点视口坐标""" m = self._get_viewport_rect('padding') - return int(self.viewport_midpoint[0]), int(m[1]) + 1 + return int(self.viewport_midpoint[0]), int(m[1]) + 3 @property def screen_location(self): @@ -1600,18 +1505,30 @@ class Locations(object): pr = self._ele.page.run_js('return window.devicePixelRatio;') return int((vx + ex) * pr), int((ey + vy) * pr) + @property + def rect(self): + """返回元素四个角坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" + vr = self._get_viewport_rect('border') + r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] + sx = r['pageX'] + sy = r['pageY'] + return [(vr[0] + sx, vr[1] + sy), (vr[2] + sx, vr[3] + sy), (vr[4] + sx, vr[5] + sy), (vr[6] + sx, vr[7] + sy)] + + @property + def viewport_rect(self): + """返回元素四个角视口坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" + r = self._get_viewport_rect('border') + return [(r[0], r[1]), (r[2], r[3]), (r[4], r[5]), (r[6], r[7])] + def _get_viewport_rect(self, quad): """按照类型返回在可视窗口中的范围 :param quad: 方框类型，margin border padding - :return: 四个角坐标，大小为0时返回None + :return: 四个角坐标 """ return self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele.ids.backend_id)['model'][quad] def _get_page_coord(self, x, y): """根据视口坐标获取绝对坐标""" - # js = 'return document.documentElement.scrollLeft+" "+document.documentElement.scrollTop;' - # xy = self._ele.run_js(js) - # sx, sy = xy.split(' ') r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] sx = r['pageX'] sy = r['pageY'] @@ -1728,244 +1645,6 @@ class ChromiumElementScroll(ChromiumScroll): self._driver.page.scroll.to_see(self._driver, center=True) -class ChromiumSelect(object): - """ChromiumSelect 类专门用于处理 d 模式下 select 标签""" - - def __init__(self, ele): - """ - :param ele: select 元素对象 - """ - if ele.tag != 'select': - raise TypeError("select方法只能在元素使用。") + + self._ele = ele + + def __call__(self, text_or_index, timeout=None): + """选定下拉列表中子元素 + :param text_or_index: 根据文本、值选或序号择选项，若允许多选，传入list或tuple可多选 + :param timeout: 超时时间，不输入默认实用页面超时时间 + :return: None + """ + para_type = 'index' if isinstance(text_or_index, int) else 'text' + timeout = timeout if timeout is not None else self._ele.page.timeout + return self._select(text_or_index, para_type, timeout=timeout) + + @property + def is_multi(self): + """返回是否多选表单""" + return self._ele.attr('multiple') is not None + + @property + def options(self): + """返回所有选项元素组成的列表""" + return self._ele.eles('xpath://option') + + @property + def selected_option(self): + """返回第一个被选中的option元素 + :return: ChromiumElement对象或None + """ + ele = self._ele.run_js('return this.options[this.selectedIndex];') + return ele + + @property + def selected_options(self): + """返回所有被选中的option元素列表 + :return: ChromiumElement对象组成的列表 + """ + return [x for x in self.options if x.states.is_selected] + + def all(self): + """全选""" + if not self.is_multi: + raise TypeError("只能在多选菜单执行此操作。") + return self._by_loc('tag:option', 1, False) + + def invert(self): + """反选""" + if not self.is_multi: + raise TypeError("只能对多项选框执行反选。") + change = False + for i in self.options: + change = True + mode = 'false' if i.states.is_selected else 'true' + i.run_js(f'this.selected={mode};') + if change: + self._dispatch_change() + + def clear(self): + """清除所有已选项""" + if not self.is_multi: + raise TypeError("只能在多选菜单执行此操作。") + return self._by_loc('tag:option', 1, True) + + def by_text(self, text, timeout=None): + """此方法用于根据text值选择项。当元素是多选列表时，可以接收list或tuple + :param text: text属性值，传入list或tuple可选择多项 + :param timeout: 超时时间，为None默认使用页面超时时间 + :return: 是否选择成功 + """ + return self._select(text, 'text', False, timeout) + + def by_value(self, value, timeout=None): + """此方法用于根据value值选择项。当元素是多选列表时，可以接收list或tuple + :param value: value属性值，传入list或tuple可选择多项 + :param timeout: 超时时间，为None默认使用页面超时时间 + :return: 是否选择成功 + """ + return self._select(value, 'value', False, timeout) + + def by_index(self, index, timeout=None): + """此方法用于根据index值选择项。当元素是多选列表时，可以接收list或tuple + :param index: 序号，0开始，传入list或tuple可选择多项 + :param timeout: 超时时间，为None默认使用页面超时时间 + :return: 是否选择成功 + """ + return self._select(index, 'index', False, timeout) + + def by_loc(self, loc, timeout=None): + """用定位符选择指定的项 + :param loc: 定位符 + :param timeout: 超时时间 + :return: 是否选择成功 + """ + return self._by_loc(loc, timeout) + + def cancel_by_text(self, text, timeout=None): + """此方法用于根据text值取消选择项。当元素是多选列表时，可以接收list或tuple + :param text: 文本，传入list或tuple可取消多项 + :param timeout: 超时时间，不输入默认实用页面超时时间 + :return: 是否取消成功 + """ + return self._select(text, 'text', True, timeout) + + def cancel_by_value(self, value, timeout=None): + """此方法用于根据value值取消选择项。当元素是多选列表时，可以接收list或tuple + :param value: value属性值，传入list或tuple可取消多项 + :param timeout: 超时时间，不输入默认实用页面超时时间 + :return: 是否取消成功 + """ + return self._select(value, 'value', True, timeout) + + def cancel_by_index(self, index, timeout=None): + """此方法用于根据index值取消选择项。当元素是多选列表时，可以接收list或tuple + :param index: 序号，0开始，传入list或tuple可取消多项 + :param timeout: 超时时间，不输入默认实用页面超时时间 + :return: 是否取消成功 + """ + return self._select(index, 'index', True, timeout) + + def cancel_by_loc(self, loc, timeout=None): + """用定位符取消选择指定的项 + :param loc: 定位符 + :param timeout: 超时时间 + :return: 是否选择成功 + """ + return self._by_loc(loc, timeout, True) + + def _by_loc(self, loc, timeout=None, cancel=False): + """用定位符取消选择指定的项 + :param loc: 定位符 + :param timeout: 超时时间 + :param cancel: 是否取消选择 + :return: 是否选择成功 + """ + eles = self._ele.eles(loc, timeout) + if not eles: + return False + + mode = 'false' if cancel else 'true' + if self.is_multi: + for ele in eles: + ele.run_js(f'this.selected={mode};') + self._dispatch_change() + return True + + eles[0].run_js(f'this.selected={mode};') + self._dispatch_change() + return True + + def _select(self, condition, para_type='text', cancel=False, timeout=None): + """选定或取消选定下拉列表中子元素 + :param condition: 根据文本、值选或序号择选项，若允许多选，传入list或tuple可多选 + :param para_type: 参数类型，可选 'text'、'value'、'index' + :param cancel: 是否取消选择 + :return: 是否选择成功 + """ + if not self.is_multi and isinstance(condition, (list, tuple)): + raise TypeError('单选列表只能传入str格式。') + + mode = 'false' if cancel else 'true' + timeout = timeout if timeout is not None else self._ele.page.timeout + condition = set(condition) if isinstance(condition, (list, tuple)) else {condition} + + if para_type in ('text', 'value'): + return self._text_value([str(i) for i in condition], para_type, mode, timeout) + elif para_type == 'index': + return self._index(condition, mode, timeout) + + def _text_value(self, condition, para_type, mode, timeout): + """执行text和value搜索 + :param condition: 条件set + :param para_type: 参数类型，可选 'text'、'value' + :param mode: 'true' 或 'false' + :param timeout: 超时时间 + :return: 是否选择成功 + """ + ok = False + text_len = len(condition) + eles = [] + end_time = perf_counter() + timeout + while perf_counter() < end_time: + if para_type == 'text': + eles = [i for i in self.options if i.text in condition] + elif para_type == 'value': + eles = [i for i in self.options if i.attr('value') in condition] + + if len(eles) >= text_len: + ok = True + break + + if ok: + for i in eles: + i.run_js(f'this.selected={mode};') + + self._dispatch_change() + return True + + return False + + def _index(self, condition, mode, timeout): + """执行index搜索 + :param condition: 条件set + :param mode: 'true' 或 'false' + :param timeout: 超时时间 + :return: 是否选择成功 + """ + ok = False + condition = [int(i) for i in condition] + text_len = max(condition) + end_time = perf_counter() + timeout + while perf_counter() < end_time: + if len(self.options) >= text_len: + ok = True + break + + if ok: + eles = self.options + for i in condition: + eles[i - 1].run_js(f'this.selected={mode};') + + self._dispatch_change() + return True + + return False + + def _dispatch_change(self): + """触发修改动作""" + self._ele.run_js('this.dispatchEvent(new UIEvent("change"));') diff --git a/DrissionPage/_units/select_element.pyi b/DrissionPage/_units/select_element.pyi new file mode 100644 index 0000000..55d07b6 --- /dev/null +++ b/DrissionPage/_units/select_element.pyi @@ -0,0 +1,63 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" +from typing import Union, Tuple, List + +from .._elements.chromium_element import ChromiumElement + + +class SelectElement(object): + def __init__(self, ele: ChromiumElement): + self._ele: ChromiumElement = ... + + def __call__(self, text_or_index: Union[str, int, list, tuple], timeout: float = None) -> bool: ... + + @property + def is_multi(self) -> bool: ... + + @property + def options(self) -> List[ChromiumElement]: ... + + @property + def selected_option(self) -> Union[ChromiumElement, None]: ... + + @property + def selected_options(self) -> List[ChromiumElement]: ... + + def clear(self) -> None: ... + + def all(self) -> None: ... + + def by_text(self, text: Union[str, list, tuple], timeout: float = None) -> bool: ... + + def by_value(self, value: Union[str, list, tuple], timeout: float = None) -> bool: ... + + def by_index(self, index: Union[int, list, tuple], timeout: float = None) -> bool: ... + + def by_loc(self, loc: Union[str, Tuple[str, str]], timeout: float = None) -> bool: ... + + def cancel_by_text(self, text: Union[str, list, tuple], timeout: float = None) -> bool: ... + + def cancel_by_value(self, value: Union[str, list, tuple], timeout: float = None) -> bool: ... + + def cancel_by_index(self, index: Union[int, list, tuple], timeout: float = None) -> bool: ... + + def cancel_by_loc(self, loc: Union[str, Tuple[str, str]], timeout: float = None) -> bool: ... + + def invert(self) -> None: ... + + def _by_loc(self, loc: Union[str, Tuple[str, str]], timeout: float = None, cancel: bool = False) -> bool: ... + + def _select(self, + condition: Union[str, int, list, tuple] = None, + para_type: str = 'text', + cancel: bool = False, + timeout: float = None) -> bool: ... + + def _text_value(self, condition: Union[list, set], para_type: str, mode: str, timeout: float) -> bool: ... + + def _index(self, condition: set, mode: str, timeout: float) -> bool: ... + + def _dispatch_change(self) -> None: ... \ No newline at end of file From 3dfbfb957f05a7d60acd03ccf71d57ba7e8c0dc7 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 7 Nov 2023 18:04:18 +0800 Subject: [PATCH 071/182] =?UTF-8?q?=E6=B5=8F=E8=A7=88=E5=99=A8=E5=85=B3?= =?UTF-8?q?=E9=97=AD=E6=97=B6=E5=88=A0=E9=99=A4=E8=AE=B0=E5=BD=95=E7=9A=84?= =?UTF-8?q?id?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 6 ++++-- DrissionPage/_base/browser.pyi | 2 ++ DrissionPage/_base/chromium_driver.py | 9 +++++++-- DrissionPage/_base/chromium_driver.pyi | 7 +++++++ 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 46ad7c1..1a0d932 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -35,7 +35,7 @@ class Browser(object): self.page = page self.address = address - self._driver = BrowserDriver(browser_id, 'browser', address) + self._driver = BrowserDriver(browser_id, 'browser', address, self) self.id = browser_id self._frames = {} self._drivers = {} @@ -63,7 +63,8 @@ class Browser(object): def _onTargetCreated(self, **kwargs): """标签页创建时执行""" if kwargs['targetInfo']['type'] == 'page' and not kwargs['targetInfo']['url'].startswith('devtools://'): - self._drivers[kwargs['targetInfo']['targetId']] = ChromiumDriver(kwargs['targetInfo']['targetId'], 'page', self.address) + self._drivers[kwargs['targetInfo']['targetId']] = ChromiumDriver(kwargs['targetInfo']['targetId'], 'page', + self.address) def _onTargetDestroyed(self, **kwargs): """标签页关闭时执行""" @@ -166,4 +167,5 @@ class Browser(object): break sleep(.2) + def _on_quit(self): Browser.BROWSERS.pop(self.id, None) diff --git a/DrissionPage/_base/browser.pyi b/DrissionPage/_base/browser.pyi index cb1a13a..fed394c 100644 --- a/DrissionPage/_base/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -58,3 +58,5 @@ class Browser(object): def _onTargetDestroyed(self, **kwargs) -> None: ... def quit(self) -> None: ... + + def _on_quit(self) -> None: ... diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index b9f1c9e..9d63a0d 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -211,17 +211,18 @@ class ChromiumDriver(object): class BrowserDriver(ChromiumDriver): BROWSERS = {} - def __new__(cls, tab_id, tab_type, address): + def __new__(cls, tab_id, tab_type, address, browser): if tab_id in cls.BROWSERS: return cls.BROWSERS[tab_id] return object.__new__(cls) - def __init__(self, tab_id, tab_type, address): + def __init__(self, tab_id, tab_type, address, browser): if hasattr(self, '_created'): return self._created = True BrowserDriver.BROWSERS[tab_id] = self super().__init__(tab_id, tab_type, address) + self.browser = browser def __repr__(self): return f"" @@ -230,3 +231,7 @@ class BrowserDriver(ChromiumDriver): r = get(url, headers={'Connection': 'close'}) r.close() return r + + def stop(self): + super().stop() + self.browser._on_quit() diff --git a/DrissionPage/_base/chromium_driver.pyi b/DrissionPage/_base/chromium_driver.pyi index fac8ab5..8977874 100644 --- a/DrissionPage/_base/chromium_driver.pyi +++ b/DrissionPage/_base/chromium_driver.pyi @@ -10,6 +10,8 @@ from typing import Union, Callable, Dict, Optional from requests import Response from websocket import WebSocket +from .browser import Browser + class GenericAttr(object): def __init__(self, name: str, tab: ChromiumDriver): ... @@ -58,5 +60,10 @@ class ChromiumDriver(object): class BrowserDriver(ChromiumDriver): BROWSERS: Dict[str, ChromiumDriver] = ... + browser: Browser = ... + + def __new__(cls, tab_id: str, tab_type: str, address: str, browser: Browser): ... + + def __init__(self, tab_id: str, tab_type: str, address: str, browser: Browser): ... def get(self, url) -> Response: ... From c6273d9bf25db807d232aa5a1086214e914756f6 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 8 Nov 2023 11:21:11 +0800 Subject: [PATCH 072/182] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=82=B9=E5=87=BB?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 98 +------------------ DrissionPage/_elements/chromium_element.pyi | 45 +-------- DrissionPage/_pages/chromium_base.py | 9 +- DrissionPage/_units/clicker.py | 42 +++++--- DrissionPage/_units/element_states.py | 8 +- DrissionPage/_units/element_states.pyi | 8 +- DrissionPage/_units/locations.py | 103 ++++++++++++++++++++ DrissionPage/_units/locations.pyi | 50 ++++++++++ 8 files changed, 201 insertions(+), 162 deletions(-) create mode 100644 DrissionPage/_units/locations.py create mode 100644 DrissionPage/_units/locations.pyi diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 704c5d6..3b5c185 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -16,6 +16,7 @@ from .._commons.tools import get_usable_path from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, offset_scroll from .._units.clicker import Clicker from .._units.element_states import ChromiumElementStates, ShadowRootStates +from .._units.locations import Locations from .._units.select_element import SelectElement from .._units.setter import ChromiumElementSetter from .._units.waiter import ChromiumElementWaiter @@ -1438,103 +1439,6 @@ def send_key(ele, modifier, key): ele.page.run_cdp('Input.dispatchKeyEvent', **data) -class Locations(object): - def __init__(self, ele): - """ - :param ele: ChromiumElement - """ - self._ele = ele - - @property - def location(self): - """返回元素左上角的绝对坐标""" - cl = self.viewport_location - return self._get_page_coord(cl[0], cl[1]) - - @property - def midpoint(self): - """返回元素中间点的绝对坐标""" - cl = self.viewport_midpoint - return self._get_page_coord(cl[0], cl[1]) - - @property - def click_point(self): - """返回元素接受点击的点的绝对坐标""" - cl = self.viewport_click_point - return self._get_page_coord(cl[0], cl[1]) - - @property - def viewport_location(self): - """返回元素左上角在视口中的坐标""" - m = self._get_viewport_rect('border') - return int(m[0]), int(m[1]) - - @property - def viewport_midpoint(self): - """返回元素中间点在视口中的坐标""" - m = self._get_viewport_rect('border') - return int(m[0] + (m[2] - m[0]) // 2), int(m[3] + (m[5] - m[3]) // 2) - - @property - def viewport_click_point(self): - """返回元素接受点击的点视口坐标""" - m = self._get_viewport_rect('padding') - return int(self.viewport_midpoint[0]), int(m[1]) + 3 - - @property - def screen_location(self): - """返回元素左上角在屏幕上坐标，左上角为(0, 0)""" - vx, vy = self._ele.page.rect.viewport_location - ex, ey = self.viewport_location - pr = self._ele.page.run_js('return window.devicePixelRatio;') - return int((vx + ex) * pr), int((ey + vy) * pr) - - @property - def screen_midpoint(self): - """返回元素中点在屏幕上坐标，左上角为(0, 0)""" - vx, vy = self._ele.page.rect.viewport_location - ex, ey = self.viewport_midpoint - pr = self._ele.page.run_js('return window.devicePixelRatio;') - return int((vx + ex) * pr), int((ey + vy) * pr) - - @property - def screen_click_point(self): - """返回元素中点在屏幕上坐标，左上角为(0, 0)""" - vx, vy = self._ele.page.rect.viewport_location - ex, ey = self.viewport_click_point - pr = self._ele.page.run_js('return window.devicePixelRatio;') - return int((vx + ex) * pr), int((ey + vy) * pr) - - @property - def rect(self): - """返回元素四个角坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" - vr = self._get_viewport_rect('border') - r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] - sx = r['pageX'] - sy = r['pageY'] - return [(vr[0] + sx, vr[1] + sy), (vr[2] + sx, vr[3] + sy), (vr[4] + sx, vr[5] + sy), (vr[6] + sx, vr[7] + sy)] - - @property - def viewport_rect(self): - """返回元素四个角视口坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" - r = self._get_viewport_rect('border') - return [(r[0], r[1]), (r[2], r[3]), (r[4], r[5]), (r[6], r[7])] - - def _get_viewport_rect(self, quad): - """按照类型返回在可视窗口中的范围 - :param quad: 方框类型，margin border padding - :return: 四个角坐标 - """ - return self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele.ids.backend_id)['model'][quad] - - def _get_page_coord(self, x, y): - """根据视口坐标获取绝对坐标""" - r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] - sx = r['pageX'] - sy = r['pageY'] - return x + sx, y + sy - - class ChromiumScroll(object): """用于滚动的对象""" diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index f65b149..c41464f 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from pathlib import Path -from typing import Union, Tuple, List, Any, Optional +from typing import Union, Tuple, List, Any from .._base.base import DrissionElement, BaseElement from .._commons.constants import NoneElement @@ -15,6 +15,7 @@ from .._pages.chromium_page import ChromiumPage from .._pages.web_page import WebPage from .._units.clicker import Clicker from .._units.element_states import ShadowRootStates, ChromiumElementStates +from .._units.locations import Locations from .._units.select_element import SelectElement from .._units.setter import ChromiumElementSetter from .._units.waiter import ChromiumElementWaiter @@ -363,48 +364,6 @@ def send_enter(ele: ChromiumElement) -> None: ... def send_key(ele: ChromiumElement, modifier: int, key: str) -> None: ... -class Locations(object): - def __init__(self, ele: ChromiumElement): - self._ele: ChromiumElement = ... - - @property - def location(self) -> Tuple[int, int]: ... - - @property - def midpoint(self) -> Tuple[int, int]: ... - - @property - def click_point(self) -> Tuple[int, int]: ... - - @property - def viewport_location(self) -> Tuple[int, int]: ... - - @property - def viewport_midpoint(self) -> Tuple[int, int]: ... - - @property - def viewport_click_point(self) -> Tuple[int, int]: ... - - @property - def screen_location(self) -> Tuple[int, int]: ... - - @property - def screen_midpoint(self) -> Tuple[int, int]: ... - - @property - def screen_click_point(self) -> Tuple[int, int]: ... - - @property - def rect(self) -> list: ... - - @property - def viewport_rect(self) -> list: ... - - def _get_viewport_rect(self, quad: str) -> Union[list, None]: ... - - def _get_page_coord(self, x: int, y: int) -> Tuple[int, int]: ... - - class ChromiumScroll(object): def __init__(self, page_or_ele: Union[ChromiumBase, ChromiumElement, ChromiumFrame]): self.t1: str = ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index a6726e8..e2c5b37 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -103,6 +103,11 @@ class ChromiumBase(BasePage): self._driver.call_method('Page.enable') self._driver.call_method('Emulation.setFocusEmulationEnabled', enabled=True) + r = self.run_cdp('Page.getFrameTree') + for i in findall(r"'id': '(.*?)'", str(r)): + self.browser._frames[i] = self.tab_id + self._frame_id = r['frameTree']['frame']['id'] + self._driver.set_listener('Page.frameStartedLoading', self._onFrameStartedLoading) self._driver.set_listener('Page.frameNavigated', self._onFrameNavigated) self._driver.set_listener('Page.domContentEventFired', self._onDomContentEventFired) @@ -111,10 +116,6 @@ class ChromiumBase(BasePage): self._driver.set_listener('Page.frameAttached', self._onFrameAttached) self._driver.set_listener('Page.frameDetached', self._onFrameDetached) - r = self.run_cdp('Page.getFrameTree') - for i in findall(r"'id': '(.*?)'", str(r)): - self.browser._frames[i] = self.tab_id - self._frame_id = r['frameTree']['frame']['id'] def _get_document(self): if self._is_reading: diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index eefc5ab..b71688d 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -7,7 +7,7 @@ from time import perf_counter, sleep from .._commons.constants import Settings from .._commons.web import offset_scroll -from ..errors import CanNotClickError, CDPError +from ..errors import CanNotClickError, CDPError, NoRectError class Clicker(object): @@ -26,7 +26,7 @@ class Clicker(object): """ return self.left(by_js, timeout) - def left(self, by_js=False, timeout=1): + def left(self, by_js=False, timeout=2): """点击元素，可选择是否用js点击 :param by_js: 是否用js点击，为None时先用模拟点击，遇到遮挡改用js，为True时直接用js点击，为False时只用模拟点击 :param timeout: 模拟点击的超时时间，等待元素可见、不被遮挡、进入视口 @@ -35,34 +35,48 @@ class Clicker(object): if not by_js: # 模拟点击 can_click = False timeout = self._ele.page.timeout if timeout is None else timeout - if timeout == 0 and self._ele.states.has_rect: - self._ele.scroll.to_see() - if self._ele.states.is_in_viewport and self._ele.states.is_enabled and self._ele.states.is_displayed: - can_click = True + rect = None + if timeout == 0: + try: + self._ele.scroll.to_see() + if self._ele.states.is_enabled and self._ele.states.is_displayed: + rect = self._ele.locations.viewport_rect + can_click = True + except NoRectError: + if by_js is False: + raise else: + rect = self._ele.states.has_rect end_time = perf_counter() + timeout - while not self._ele.states.has_rect and perf_counter() < end_time: + while not rect and perf_counter() < end_time: + rect = self._ele.states.has_rect sleep(.001) - if self._ele.states.has_rect: + + self._ele.wait.stop_moving(timeout=end_time - perf_counter()) + if rect: self._ele.scroll.to_see() + rect = self._ele.locations.rect while perf_counter() < end_time: - if (self._ele.states.is_in_viewport and self._ele.states.is_enabled - and self._ele.states.is_displayed): + if self._ele.states.is_enabled and self._ele.states.is_displayed: can_click = True break sleep(.001) - if not self._ele.states.has_rect or not self._ele.states.is_in_viewport: + elif by_js is False: + raise NoRectError + + if can_click and not self._ele.states.is_in_viewport: by_js = True elif can_click and (by_js is False or not self._ele.states.is_covered): - vx, vy = self._ele.locations.click_point + x = int(rect[1][0] - (rect[1][0] - rect[0][0]) / 2) + y = rect[0][0] + 3 try: - r = self._ele.page.run_cdp('DOM.getNodeForLocation', x=vx, y=vy, includeUserAgentShadowDOM=True, + r = self._ele.page.run_cdp('DOM.getNodeForLocation', x=x, y=y, includeUserAgentShadowDOM=True, ignorePointerEventsNone=True) if r['backendNodeId'] != self._ele.ids.backend_id: - vx, vy = self._ele.locations.viewport_click_point + vx, vy = self._ele.locations.viewport_midpoint else: vx, vy = self._ele.locations.viewport_click_point diff --git a/DrissionPage/_units/element_states.py b/DrissionPage/_units/element_states.py index efc2b97..73d724d 100644 --- a/DrissionPage/_units/element_states.py +++ b/DrissionPage/_units/element_states.py @@ -1,4 +1,8 @@ # -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" from .._commons.web import location_in_viewport from ..errors import CDPError, NoRectError @@ -71,9 +75,9 @@ class ChromiumElementStates(object): @property def has_rect(self): - """返回元素是否拥有位置和大小，没有返回False，有返回大小元组""" + """返回元素是否拥有位置和大小，没有返回False，有返回四个角在页面中坐标组成的列表""" try: - return self._ele.size + return self._ele.locations.rect except NoRectError: return False diff --git a/DrissionPage/_units/element_states.pyi b/DrissionPage/_units/element_states.pyi index d512f7f..81ebad5 100644 --- a/DrissionPage/_units/element_states.pyi +++ b/DrissionPage/_units/element_states.pyi @@ -1,5 +1,9 @@ # -*- coding:utf-8 -*- -from typing import Union, Tuple +""" +@Author : g1879 +@Contact : g1879@qq.com +""" +from typing import Union, Tuple, List from .._elements.chromium_element import ChromiumShadowRoot, ChromiumElement @@ -33,7 +37,7 @@ class ChromiumElementStates(object): def is_covered(self) -> bool: ... @property - def has_rect(self) -> Union[bool, Tuple[int, int]]: ... + def has_rect(self) -> Union[bool, List[Tuple[float, float]]]: ... class ShadowRootStates(object): diff --git a/DrissionPage/_units/locations.py b/DrissionPage/_units/locations.py new file mode 100644 index 0000000..a402681 --- /dev/null +++ b/DrissionPage/_units/locations.py @@ -0,0 +1,103 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" + + +class Locations(object): + def __init__(self, ele): + """ + :param ele: ChromiumElement + """ + self._ele = ele + + @property + def location(self): + """返回元素左上角的绝对坐标""" + cl = self.viewport_location + return self._get_page_coord(cl[0], cl[1]) + + @property + def midpoint(self): + """返回元素中间点的绝对坐标""" + cl = self.viewport_midpoint + return self._get_page_coord(cl[0], cl[1]) + + @property + def click_point(self): + """返回元素接受点击的点的绝对坐标""" + cl = self.viewport_click_point + return self._get_page_coord(cl[0], cl[1]) + + @property + def viewport_location(self): + """返回元素左上角在视口中的坐标""" + m = self._get_viewport_rect('border') + return int(m[0]), int(m[1]) + + @property + def viewport_midpoint(self): + """返回元素中间点在视口中的坐标""" + m = self._get_viewport_rect('border') + return int(m[0] + (m[2] - m[0]) // 2), int(m[3] + (m[5] - m[3]) // 2) + + @property + def viewport_click_point(self): + """返回元素接受点击的点视口坐标""" + m = self._get_viewport_rect('padding') + return int(self.viewport_midpoint[0]), int(m[1]) + 3 + + @property + def screen_location(self): + """返回元素左上角在屏幕上坐标，左上角为(0, 0)""" + vx, vy = self._ele.page.rect.viewport_location + ex, ey = self.viewport_location + pr = self._ele.page.run_js('return window.devicePixelRatio;') + return int((vx + ex) * pr), int((ey + vy) * pr) + + @property + def screen_midpoint(self): + """返回元素中点在屏幕上坐标，左上角为(0, 0)""" + vx, vy = self._ele.page.rect.viewport_location + ex, ey = self.viewport_midpoint + pr = self._ele.page.run_js('return window.devicePixelRatio;') + return int((vx + ex) * pr), int((ey + vy) * pr) + + @property + def screen_click_point(self): + """返回元素中点在屏幕上坐标，左上角为(0, 0)""" + vx, vy = self._ele.page.rect.viewport_location + ex, ey = self.viewport_click_point + pr = self._ele.page.run_js('return window.devicePixelRatio;') + return int((vx + ex) * pr), int((ey + vy) * pr) + + @property + def rect(self): + """返回元素四个角坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" + vr = self._get_viewport_rect('border') + r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] + sx = int(r['pageX']) + sy = int(r['pageY']) + return [(vr[0] + sx, vr[1] + sy), (vr[2] + sx, vr[3] + sy), + (vr[4] + sx, vr[5] + sy), (vr[6] + sx, vr[7] + sy)] + + @property + def viewport_rect(self): + """返回元素四个角视口坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" + r = self._get_viewport_rect('border') + return [(int(r[0]), int(r[1])), (int(r[2]), int(r[3])), (int(r[4]), int(r[5])), (int(r[6]), int(r[7]))] + + def _get_viewport_rect(self, quad): + """按照类型返回在可视窗口中的范围 + :param quad: 方框类型，margin border padding + :return: 四个角坐标 + """ + return self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele.ids.backend_id)['model'][quad] + + def _get_page_coord(self, x, y): + """根据视口坐标获取绝对坐标""" + r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] + sx = r['pageX'] + sy = r['pageY'] + return x + sx, y + sy diff --git a/DrissionPage/_units/locations.pyi b/DrissionPage/_units/locations.pyi new file mode 100644 index 0000000..5e61165 --- /dev/null +++ b/DrissionPage/_units/locations.pyi @@ -0,0 +1,50 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" +from typing import Tuple, Union, List + +from .._elements.chromium_element import ChromiumElement + + +class Locations(object): + def __init__(self, ele: ChromiumElement): + self._ele: ChromiumElement = ... + + @property + def location(self) -> Tuple[int, int]: ... + + @property + def midpoint(self) -> Tuple[int, int]: ... + + @property + def click_point(self) -> Tuple[int, int]: ... + + @property + def viewport_location(self) -> Tuple[int, int]: ... + + @property + def viewport_midpoint(self) -> Tuple[int, int]: ... + + @property + def viewport_click_point(self) -> Tuple[int, int]: ... + + @property + def screen_location(self) -> Tuple[int, int]: ... + + @property + def screen_midpoint(self) -> Tuple[int, int]: ... + + @property + def screen_click_point(self) -> Tuple[int, int]: ... + + @property + def rect(self) -> List[Tuple[int, int], ...]: ... + + @property + def viewport_rect(self) -> List[Tuple[int, int], ...]: ... + + def _get_viewport_rect(self, quad: str) -> Union[list, None]: ... + + def _get_page_coord(self, x: int, y: int) -> Tuple[int, int]: ... From 457eb27566c0061980db9e128d1d1d3aaa8d5d0c Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 8 Nov 2023 15:59:50 +0800 Subject: [PATCH 073/182] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E5=92=8C=E9=83=A8=E5=88=86=E7=B1=BB=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 4 +- DrissionPage/_base/browser.pyi | 4 +- DrissionPage/_commons/browser.py | 3 +- DrissionPage/_elements/chromium_element.py | 151 +---------------- DrissionPage/_elements/chromium_element.pyi | 80 ++------- DrissionPage/_pages/chromium_base.py | 50 +----- DrissionPage/_pages/chromium_base.pyi | 21 +-- DrissionPage/_pages/chromium_frame.py | 54 +------ DrissionPage/_pages/chromium_frame.pyi | 41 ++--- DrissionPage/_pages/chromium_page.py | 8 +- DrissionPage/_pages/chromium_page.pyi | 10 +- DrissionPage/_pages/chromium_tab.py | 8 +- DrissionPage/_pages/chromium_tab.pyi | 4 +- DrissionPage/_units/download_manager.py | 2 +- DrissionPage/_units/download_manager.pyi | 6 +- DrissionPage/_units/element_states.py | 2 +- DrissionPage/_units/element_states.pyi | 2 +- DrissionPage/_units/ids.py | 57 +++++++ DrissionPage/_units/ids.pyi | 45 ++++++ DrissionPage/_units/scroller.py | 171 ++++++++++++++++++++ DrissionPage/_units/scroller.pyi | 73 +++++++++ DrissionPage/_units/tab_rect.py | 2 +- DrissionPage/_units/tab_rect.pyi | 2 +- DrissionPage/_units/waiter.py | 12 +- DrissionPage/_units/waiter.pyi | 10 +- DrissionPage/common.py | 2 + 26 files changed, 436 insertions(+), 388 deletions(-) create mode 100644 DrissionPage/_units/ids.py create mode 100644 DrissionPage/_units/ids.pyi create mode 100644 DrissionPage/_units/scroller.py create mode 100644 DrissionPage/_units/scroller.pyi diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 1a0d932..dc4d4dd 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -6,7 +6,7 @@ from time import sleep from .chromium_driver import BrowserDriver, ChromiumDriver -from .._units.download_manager import BrowserDownloadManager +from .._units.download_manager import DownloadManager class Browser(object): @@ -77,7 +77,7 @@ class Browser(object): def connect_to_page(self): """执行与page相关的逻辑""" if not self._connected: - self._dl_mgr = BrowserDownloadManager(self) + self._dl_mgr = DownloadManager(self) self._connected = True def run_cdp(self, cmd, **cmd_args): diff --git a/DrissionPage/_base/browser.pyi b/DrissionPage/_base/browser.pyi index fed394c..a7a69c5 100644 --- a/DrissionPage/_base/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -7,7 +7,7 @@ from typing import List, Optional, Union from .chromium_driver import BrowserDriver, ChromiumDriver from .._pages.chromium_page import ChromiumPage -from .._units.download_manager import BrowserDownloadManager +from .._units.download_manager import DownloadManager class Browser(object): @@ -19,7 +19,7 @@ class Browser(object): _frames: dict = ... _drivers: dict = ... _process_id: Optional[int] = ... - _dl_mgr: BrowserDownloadManager = ... + _dl_mgr: DownloadManager = ... _connected: bool = ... def __new__(cls, address: str, browser_id: str, page: ChromiumPage): ... diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_commons/browser.py index 25909bb..6a52559 100644 --- a/DrissionPage/_commons/browser.py +++ b/DrissionPage/_commons/browser.py @@ -169,7 +169,8 @@ def test_connect(ip, port, timeout=30): raise BrowserConnectError(f'\n{ip}:{port}浏览器无法链接。\n请确认：\n1、该端口为浏览器\n' f'2、已添加--remote-allow-origins=*和--remote-debugging-port={port}启动项\n' f'3、用户文件夹没有和已打开的浏览器冲突\n' - f'4、如为无界面系统，请使用headless模式\n' + f'4、如为无界面系统，请添加--headless=new参数\n' + f'5、如果是Linux系统，可能还要添加--no-sandbox启动参数\n' f'可使用ChromiumOptions设置端口和用户文件夹路径。') diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 3b5c185..c57eb52 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -15,11 +15,13 @@ from .._commons.locator import get_loc from .._commons.tools import get_usable_path from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, offset_scroll from .._units.clicker import Clicker -from .._units.element_states import ChromiumElementStates, ShadowRootStates +from .._units.element_states import ElementStates, ShadowRootStates +from .._units.ids import Ids, ElementIds from .._units.locations import Locations +from .._units.scroller import ElementScroller from .._units.select_element import SelectElement from .._units.setter import ChromiumElementSetter -from .._units.waiter import ChromiumElementWaiter +from .._units.waiter import ElementWaiter from ..errors import (ContextLossError, ElementLossError, JavaScriptError, ElementNotFoundError, CDPError, NoResourceError, AlertExistsError) @@ -64,7 +66,7 @@ class ChromiumElement(DrissionElement): else: raise ElementLossError - self._ids = ChromiumElementIds(self) + self._ids = ElementIds(self) doc = self.run_js('return this.ownerDocument;') self._doc_id = doc['objectId'] if doc else None @@ -141,7 +143,7 @@ class ChromiumElement(DrissionElement): def states(self): """返回用于获取元素状态的对象""" if self._states is None: - self._states = ChromiumElementStates(self) + self._states = ElementStates(self) return self._states @property @@ -181,7 +183,7 @@ class ChromiumElement(DrissionElement): def scroll(self): """用于滚动滚动条的对象""" if self._scroll is None: - self._scroll = ChromiumElementScroll(self) + self._scroll = ElementScroller(self) return self._scroll @property @@ -195,7 +197,7 @@ class ChromiumElement(DrissionElement): def wait(self): """返回用于等待的对象""" if self._wait is None: - self._wait = ChromiumElementWaiter(self.page, self) + self._wait = ElementWaiter(self.page, self) return self._wait @property @@ -1081,33 +1083,6 @@ class ChromiumShadowRoot(BaseElement): return r['backendNodeId'] -class Ids(object): - def __init__(self, ele): - self._ele = ele - - @property - def node_id(self): - """返回元素cdp中的node id""" - return self._ele._node_id - - @property - def obj_id(self): - """返回元素js中的object id""" - return self._ele._obj_id - - @property - def backend_id(self): - """返回backend id""" - return self._ele._backend_id - - -class ChromiumElementIds(Ids): - @property - def doc_id(self): - """返回所在document的object id""" - return self._ele._doc_id - - def find_in_chromium_ele(ele, loc, single=True, timeout=None, relative=True): """在chromium元素中查找 :param ele: ChromiumElement对象 @@ -1439,116 +1414,6 @@ def send_key(ele, modifier, key): ele.page.run_cdp('Input.dispatchKeyEvent', **data) -class ChromiumScroll(object): - """用于滚动的对象""" - - def __init__(self, ele): - """ - :param ele: 元素对象 - """ - self._driver = ele - self.t1 = self.t2 = 'this' - self._wait_complete = False - - def _run_js(self, js): - js = js.format(self.t1, self.t2, self.t2) - self._driver.run_js(js) - self._wait_scrolled() - - def to_top(self): - """滚动到顶端，水平位置不变""" - self._run_js('{}.scrollTo({}.scrollLeft, 0);') - - def to_bottom(self): - """滚动到底端，水平位置不变""" - self._run_js('{}.scrollTo({}.scrollLeft, {}.scrollHeight);') - - def to_half(self): - """滚动到垂直中间位置，水平位置不变""" - self._run_js('{}.scrollTo({}.scrollLeft, {}.scrollHeight/2);') - - def to_rightmost(self): - """滚动到最右边，垂直位置不变""" - self._run_js('{}.scrollTo({}.scrollWidth, {}.scrollTop);') - - def to_leftmost(self): - """滚动到最左边，垂直位置不变""" - self._run_js('{}.scrollTo(0, {}.scrollTop);') - - def to_location(self, x, y): - """滚动到指定位置 - :param x: 水平距离 - :param y: 垂直距离 - :return: None - """ - self._run_js(f'{{}}.scrollTo({x}, {y});') - - def up(self, pixel=300): - """向上滚动若干像素，水平位置不变 - :param pixel: 滚动的像素 - :return: None - """ - pixel = -pixel - self._run_js(f'{{}}.scrollBy(0, {pixel});') - - def down(self, pixel=300): - """向下滚动若干像素，水平位置不变 - :param pixel: 滚动的像素 - :return: None - """ - self._run_js(f'{{}}.scrollBy(0, {pixel});') - - def left(self, pixel=300): - """向左滚动若干像素，垂直位置不变 - :param pixel: 滚动的像素 - :return: None - """ - pixel = -pixel - self._run_js(f'{{}}.scrollBy({pixel}, 0);') - - def right(self, pixel=300): - """向右滚动若干像素，垂直位置不变 - :param pixel: 滚动的像素 - :return: None - """ - self._run_js(f'{{}}.scrollBy({pixel}, 0);') - - def _wait_scrolled(self): - if not self._wait_complete: - return - - page = self._driver.page if isinstance(self._driver, ChromiumElement) else self._driver - r = page.run_cdp('Page.getLayoutMetrics') - x = r['layoutViewport']['pageX'] - y = r['layoutViewport']['pageY'] - - end_time = perf_counter() + self._driver.page.timeout - while perf_counter() < end_time: - sleep(.1) - r = page.run_cdp('Page.getLayoutMetrics') - x1 = r['layoutViewport']['pageX'] - y1 = r['layoutViewport']['pageY'] - - if x == x1 and y == y1: - break - - x = x1 - y = y1 - - -class ChromiumElementScroll(ChromiumScroll): - def to_see(self, center=None): - """滚动页面直到元素可见 - :param center: 是否尽量滚动到页面正中，为None时如果被遮挡，则滚动到页面正中 - :return: None - """ - self._driver.page.scroll.to_see(self._driver, center=center) - - def to_center(self): - """元素尽量滚动到视口中间""" - self._driver.page.scroll.to_see(self._driver, center=True) - - class Pseudo(object): def __init__(self, ele): """ diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index c41464f..cf64a41 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -14,11 +14,13 @@ from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage from .._pages.web_page import WebPage from .._units.clicker import Clicker -from .._units.element_states import ShadowRootStates, ChromiumElementStates +from .._units.element_states import ShadowRootStates, ElementStates +from .._units.ids import Ids, ElementIds from .._units.locations import Locations +from .._units.scroller import ElementScroller from .._units.select_element import SelectElement from .._units.setter import ChromiumElementSetter -from .._units.waiter import ChromiumElementWaiter +from .._units.waiter import ElementWaiter class ChromiumElement(DrissionElement): @@ -32,14 +34,14 @@ class ChromiumElement(DrissionElement): self._obj_id: str = ... self._backend_id: str = ... self._doc_id: str = ... - self._ids: ChromiumElementIds = ... - self._scroll: ChromiumElementScroll = ... + self._ids: ElementIds = ... + self._scroll: ElementScroller = ... self._clicker: Clicker = ... self._select: SelectElement = ... - self._wait: ChromiumElementWaiter = ... + self._wait: ElementWaiter = ... self._locations: Locations = ... self._set: ChromiumElementSetter = ... - self._states: ChromiumElementStates = ... + self._states: ElementStates = ... self._pseudo: Pseudo = ... def __repr__(self) -> str: ... @@ -68,7 +70,7 @@ class ChromiumElement(DrissionElement): # -----------------d模式独有属性------------------- @property - def ids(self) -> ChromiumElementIds: ... + def ids(self) -> ElementIds: ... @property def size(self) -> Tuple[int, int]: ... @@ -77,7 +79,7 @@ class ChromiumElement(DrissionElement): def set(self) -> ChromiumElementSetter: ... @property - def states(self) -> ChromiumElementStates: ... + def states(self) -> ElementStates: ... @property def location(self) -> Tuple[int, int]: ... @@ -95,7 +97,7 @@ class ChromiumElement(DrissionElement): def sr(self) -> Union[None, ChromiumShadowRoot]: ... @property - def scroll(self) -> ChromiumElementScroll: ... + def scroll(self) -> ElementScroller: ... @property def click(self) -> Clicker: ... @@ -148,7 +150,7 @@ class ChromiumElement(DrissionElement): ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... @property - def wait(self) -> ChromiumElementWaiter: ... + def wait(self) -> ElementWaiter: ... @property def select(self) -> SelectElement: ... @@ -301,25 +303,6 @@ class ChromiumShadowRoot(BaseElement): def _get_backend_id(self, node_id: str) -> str: ... -class Ids(object): - def __init__(self, ele: Union[ChromiumElement, ChromiumShadowRoot]): - self._ele: Union[ChromiumElement, ChromiumShadowRoot] = ... - - @property - def node_id(self) -> str: ... - - @property - def obj_id(self) -> str: ... - - @property - def backend_id(self) -> str: ... - - -class ChromiumElementIds(Ids): - @property - def doc_id(self) -> str: ... - - def find_in_chromium_ele(ele: ChromiumElement, loc: Union[str, Tuple[str, str]], single: bool = True, @@ -364,45 +347,6 @@ def send_enter(ele: ChromiumElement) -> None: ... def send_key(ele: ChromiumElement, modifier: int, key: str) -> None: ... -class ChromiumScroll(object): - def __init__(self, page_or_ele: Union[ChromiumBase, ChromiumElement, ChromiumFrame]): - self.t1: str = ... - self.t2: str = ... - self._driver: Union[ChromiumPage, ChromiumElement, ChromiumFrame] = ... - self._wait_complete: bool = ... - - def _run_js(self, js: str): ... - - def to_top(self) -> None: ... - - def to_bottom(self) -> None: ... - - def to_half(self) -> None: ... - - def to_rightmost(self) -> None: ... - - def to_leftmost(self) -> None: ... - - def to_location(self, x: int, y: int) -> None: ... - - def up(self, pixel: int = 300) -> None: ... - - def down(self, pixel: int = 300) -> None: ... - - def left(self, pixel: int = 300) -> None: ... - - def right(self, pixel: int = 300) -> None: ... - - def _wait_scrolled(self) -> None: ... - - -class ChromiumElementScroll(ChromiumScroll): - - def to_see(self, center: Union[bool, None] = None) -> None: ... - - def to_center(self) -> None: ... - - class Pseudo(object): def __init__(self, ele: ChromiumElement): self._ele: ChromiumElement = ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index e2c5b37..9b5f141 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -10,18 +10,19 @@ from re import findall from threading import Thread from time import perf_counter, sleep +from .._units.scroller import PageScroller from .._base.base import BasePage from .._commons.constants import ERROR, NoneElement from .._commons.locator import get_loc from .._commons.tools import get_usable_path from .._commons.web import location_in_viewport -from .._elements.chromium_element import ChromiumScroll, ChromiumElement, run_js, make_chromium_ele +from .._elements.chromium_element import ChromiumElement, run_js, make_chromium_ele from .._elements.session_element import make_session_ele from .._units.action_chains import ActionChains from .._units.network_listener import NetworkListener from .._units.screencast import Screencast from .._units.setter import ChromiumBaseSetter -from .._units.waiter import ChromiumBaseWaiter +from .._units.waiter import BaseWaiter from ..errors import (ContextLossError, ElementLossError, CDPError, TabClosedError, NoRectError, AlertExistsError, GetDocumentError) @@ -116,7 +117,6 @@ class ChromiumBase(BasePage): self._driver.set_listener('Page.frameAttached', self._onFrameAttached) self._driver.set_listener('Page.frameDetached', self._onFrameDetached) - def _get_document(self): if self._is_reading: return @@ -326,7 +326,7 @@ class ChromiumBase(BasePage): """返回用于滚动滚动条的对象""" self.wait.load_complete() if self._scroll is None: - self._scroll = ChromiumPageScroll(self) + self._scroll = PageScroller(self) return self._scroll @property @@ -343,7 +343,7 @@ class ChromiumBase(BasePage): def wait(self): """返回用于等待的对象""" if self._wait is None: - self._wait = ChromiumBaseWaiter(self) + self._wait = BaseWaiter(self) return self._wait @property @@ -971,46 +971,6 @@ class ChromiumBase(BasePage): return str(path.absolute()) -class ChromiumPageScroll(ChromiumScroll): - def __init__(self, page): - """ - :param page: 页面对象 - """ - super().__init__(page) - self.t1 = 'window' - self.t2 = 'document.documentElement' - - def to_see(self, loc_or_ele, center=None): - """滚动页面直到元素可见 - :param loc_or_ele: 元素的定位信息，可以是loc元组，或查询字符串 - :param center: 是否尽量滚动到页面正中，为None时如果被遮挡，则滚动到页面正中 - :return: None - """ - ele = self._driver._ele(loc_or_ele) - self._to_see(ele, center) - - def _to_see(self, ele, center): - """执行滚动页面直到元素可见 - :param ele: 元素对象 - :param center: 是否尽量滚动到页面正中，为None时如果被遮挡，则滚动到页面正中 - :return: None - """ - txt = 'true' if center else 'false' - ele.run_js(f'this.scrollIntoViewIfNeeded({txt});') - if center or (center is not False and ele.states.is_covered): - ele.run_js('''function getWindowScrollTop() {var scroll_top = 0; - if (document.documentElement && document.documentElement.scrollTop) { - scroll_top = document.documentElement.scrollTop; - } else if (document.body) {scroll_top = document.body.scrollTop;} - return scroll_top;} - const { top, height } = this.getBoundingClientRect(); - const elCenter = top + height / 2; - const center = window.innerHeight / 2; - window.scrollTo({top: getWindowScrollTop() - (center - elCenter), - behavior: 'instant'});''') - self._wait_scrolled() - - class Timeout(object): """用于保存d模式timeout信息的类""" diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 9a8aed2..826e95e 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -10,15 +10,16 @@ from .._base.base import BasePage from .._base.browser import Browser from .._base.chromium_driver import ChromiumDriver from .._commons.constants import NoneElement -from .._elements.chromium_element import ChromiumElement, ChromiumScroll +from .._elements.chromium_element import ChromiumElement from .._elements.session_element import SessionElement from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage from .._units.action_chains import ActionChains from .._units.network_listener import NetworkListener from .._units.screencast import Screencast +from .._units.scroller import Scroller, PageScroller from .._units.setter import ChromiumBaseSetter -from .._units.waiter import ChromiumBaseWaiter +from .._units.waiter import BaseWaiter class ChromiumBase(BasePage): @@ -37,12 +38,12 @@ class ChromiumBase(BasePage): self._first_run: bool = ... self._is_loading: bool = ... self._page_load_strategy: str = ... - self._scroll: ChromiumScroll = ... + self._scroll: Scroller = ... self._url: str = ... self._root_id: str = ... self._debug: bool = ... self._upload_list: list = ... - self._wait: ChromiumBaseWaiter = ... + self._wait: BaseWaiter = ... self._set: ChromiumBaseSetter = ... self._screencast: Screencast = ... self._actions: ActionChains = ... @@ -137,7 +138,7 @@ class ChromiumBase(BasePage): def user_agent(self) -> str: ... @property - def scroll(self) -> ChromiumPageScroll: ... + def scroll(self) -> PageScroller: ... @property def timeouts(self) -> Timeout: ... @@ -146,7 +147,7 @@ class ChromiumBase(BasePage): def upload_list(self) -> list: ... @property - def wait(self) -> ChromiumBaseWaiter: ... + def wait(self) -> BaseWaiter: ... @property def set(self) -> ChromiumBaseSetter: ... @@ -247,14 +248,6 @@ class ChromiumBase(BasePage): timeout: float = None) -> Union[bool, None]: ... -class ChromiumPageScroll(ChromiumScroll): - def __init__(self, page: ChromiumBase): ... - - def to_see(self, loc_or_ele: Union[str, tuple, ChromiumElement], center: Union[bool, None] = None) -> None: ... - - def _to_see(self, ele: ChromiumElement, center: Union[bool, None]) -> None: ... - - class Timeout(object): def __init__(self, page: ChromiumBase, implicit=None, page_load=None, script=None): diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 3ca9b25..1d412b8 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -8,10 +8,10 @@ from re import search, findall from threading import Thread from time import sleep, perf_counter -from requests import get - from .._elements.chromium_element import ChromiumElement -from .._pages.chromium_base import ChromiumBase, ChromiumPageScroll +from .._pages.chromium_base import ChromiumBase +from .._units.ids import FrameIds +from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter from .._units.waiter import FrameWaiter from ..errors import ContextLossError, ElementLossError, GetDocumentError @@ -40,7 +40,7 @@ class ChromiumFrame(ChromiumBase): self._backend_id = ele.ids.backend_id self._frame_ele = ele self._states = None - self._ids = ChromiumFrameIds(self) + self._ids = FrameIds(self) if self._is_inner_frame(): self._is_diff_domain = False @@ -329,7 +329,7 @@ class ChromiumFrame(ChromiumBase): @property def scroll(self): """返回用于等待的对象""" - return ChromiumFrameScroll(self) + return FrameScroller(self) @property def set(self): @@ -658,47 +658,3 @@ class ChromiumFrame(ChromiumBase): while self.is_alive: sleep(1) self.driver.stop() - - -class ChromiumFrameIds(object): - def __init__(self, frame): - self._frame = frame - - @property - def tab_id(self): - """返回当前标签页id""" - return self._frame._tab_id - - @property - def backend_id(self): - """返回cdp中的node id""" - return self._frame._backend_id - - @property - def obj_id(self): - """返回frame元素的object id""" - return self._frame.frame_ele.ids.obj_id - - @property - def node_id(self): - """返回cdp中的node id""" - return self._frame.frame_ele.ids.node_id - - -class ChromiumFrameScroll(ChromiumPageScroll): - def __init__(self, frame): - """ - :param frame: ChromiumFrame对象 - """ - self._driver = frame.doc_ele - self.t1 = self.t2 = 'this.documentElement' - self._wait_complete = False - - def to_see(self, loc_or_ele, center=None): - """滚动页面直到元素可见 - :param loc_or_ele: 元素的定位信息，可以是loc元组，或查询字符串 - :param center: 是否尽量滚动到页面正中，为None时如果被遮挡，则滚动到页面正中 - :return: None - """ - ele = loc_or_ele if isinstance(loc_or_ele, ChromiumElement) else self._driver._ele(loc_or_ele) - self._to_see(ele, center) diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 4121917..3e63295 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -6,11 +6,15 @@ from pathlib import Path from typing import Union, Tuple, List, Any -from .chromium_base import ChromiumBase, ChromiumPageScroll +from .chromium_base import ChromiumBase from .chromium_page import ChromiumPage from .chromium_tab import ChromiumTab from .web_page import WebPage -from .._elements.chromium_element import ChromiumElement, Locations, ChromiumElementStates +from .._elements.chromium_element import ChromiumElement +from .._units.element_states import ElementStates +from .._units.ids import FrameIds +from .._units.locations import Locations +from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter from .._units.waiter import FrameWaiter @@ -28,8 +32,8 @@ class ChromiumFrame(ChromiumBase): self._doc_ele: ChromiumElement = ... self._is_diff_domain: bool = ... self.doc_ele: ChromiumElement = ... - self._states: ChromiumElementStates = ... - self._ids: ChromiumFrameIds = ... + self._states: ElementStates = ... + self._ids: FrameIds = ... def __call__(self, loc_or_str: Union[Tuple[str, str], str], @@ -57,7 +61,7 @@ class ChromiumFrame(ChromiumBase): def page(self) -> Union[ChromiumPage, WebPage]: ... @property - def ids(self) -> ChromiumFrameIds: ... + def ids(self) -> FrameIds: ... @property def frame_ele(self) -> ChromiumElement: ... @@ -111,13 +115,13 @@ class ChromiumFrame(ChromiumBase): def is_alive(self) -> bool: ... @property - def scroll(self) -> ChromiumFrameScroll: ... + def scroll(self) -> FrameScroller: ... @property def set(self) -> ChromiumFrameSetter: ... @property - def states(self) -> ChromiumElementStates: ... + def states(self) -> ElementStates: ... @property def wait(self) -> FrameWaiter: ... @@ -197,26 +201,3 @@ class ChromiumFrame(ChromiumBase): timeout: float = None) -> Union[bool, None]: ... def _is_inner_frame(self) -> bool: ... - - -class ChromiumFrameIds(object): - def __init__(self, frame: ChromiumFrame): - self._frame: ChromiumFrame = ... - - @property - def tab_id(self) -> str: ... - - @property - def backend_id(self) -> str: ... - - @property - def obj_id(self) -> str: ... - - @property - def node_id(self) -> str: ... - - -class ChromiumFrameScroll(ChromiumPageScroll): - def __init__(self, frame: ChromiumFrame) -> None: ... - - def to_see(self, loc_or_ele: Union[str, tuple, ChromiumElement], center: Union[None, bool] = None) -> None: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index e19fbb6..ae06dd4 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -14,8 +14,8 @@ from .._configs.chromium_options import ChromiumOptions from .._pages.chromium_base import ChromiumBase, Timeout from .._pages.chromium_tab import ChromiumTab from .._units.setter import ChromiumPageSetter -from .._units.tab_rect import ChromiumTabRect -from .._units.waiter import ChromiumPageWaiter +from .._units.tab_rect import TabRect +from .._units.waiter import PageWaiter from ..errors import BrowserConnectError @@ -127,14 +127,14 @@ class ChromiumPage(ChromiumBase): @property def rect(self): if self._rect is None: - self._rect = ChromiumTabRect(self) + self._rect = TabRect(self) return self._rect @property def wait(self): """返回用于等待的对象""" if self._wait is None: - self._wait = ChromiumPageWaiter(self) + self._wait = PageWaiter(self) return self._wait def get_tab(self, tab_id=None): diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index 5295ba0..a5f5ef7 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -10,8 +10,8 @@ from .._configs.chromium_options import ChromiumOptions from .._pages.chromium_base import ChromiumBase from .._pages.chromium_tab import ChromiumTab from .._units.setter import ChromiumPageSetter -from .._units.tab_rect import ChromiumTabRect -from .._units.waiter import ChromiumPageWaiter +from .._units.tab_rect import TabRect +from .._units.waiter import PageWaiter class ChromiumPage(ChromiumBase): @@ -23,7 +23,7 @@ class ChromiumPage(ChromiumBase): self._driver_options: ChromiumOptions = ... self._main_tab: str = ... self._browser: Browser = ... - self._rect: Optional[ChromiumTabRect] = ... + self._rect: Optional[TabRect] = ... def _handle_options(self, addr_or_opts: Union[str, ChromiumOptions]) -> str: ... @@ -41,10 +41,10 @@ class ChromiumPage(ChromiumBase): def tabs(self) -> List[str]: ... @property - def rect(self) -> ChromiumTabRect: ... + def rect(self) -> TabRect: ... @property - def wait(self) -> ChromiumPageWaiter: ... + def wait(self) -> PageWaiter: ... @property def main_tab(self) -> str: ... diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 8944559..b65bb03 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -10,8 +10,8 @@ from .._commons.web import set_session_cookies, set_browser_cookies from .._pages.chromium_base import ChromiumBase from .._pages.session_page import SessionPage from .._units.setter import TabSetter, WebPageTabSetter -from .._units.tab_rect import ChromiumTabRect -from .._units.waiter import ChromiumTabWaiter +from .._units.tab_rect import TabRect +from .._units.waiter import TabWaiter class ChromiumTab(ChromiumBase): @@ -48,7 +48,7 @@ class ChromiumTab(ChromiumBase): def rect(self): """返回获取窗口坐标和大小的对象""" if self._rect is None: - self._rect = ChromiumTabRect(self) + self._rect = TabRect(self) return self._rect @property @@ -62,7 +62,7 @@ class ChromiumTab(ChromiumBase): def wait(self): """返回用于等待的对象""" if self._wait is None: - self._wait = ChromiumTabWaiter(self) + self._wait = TabWaiter(self) return self._wait diff --git a/DrissionPage/_pages/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi index 923c8c7..ef034b3 100644 --- a/DrissionPage/_pages/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -17,7 +17,7 @@ from .._base.browser import Browser from .._elements.chromium_element import ChromiumElement from .._elements.session_element import SessionElement from .._units.setter import TabSetter, WebPageTabSetter -from .._units.waiter import ChromiumTabWaiter +from .._units.waiter import TabWaiter class ChromiumTab(ChromiumBase): @@ -41,7 +41,7 @@ class ChromiumTab(ChromiumBase): def set(self) -> TabSetter: ... @property - def wait(self) -> ChromiumTabWaiter: ... + def wait(self) -> TabWaiter: ... class WebPageTab(SessionPage, ChromiumTab): diff --git a/DrissionPage/_units/download_manager.py b/DrissionPage/_units/download_manager.py index a36d362..bb04860 100644 --- a/DrissionPage/_units/download_manager.py +++ b/DrissionPage/_units/download_manager.py @@ -11,7 +11,7 @@ from time import sleep, perf_counter from .._commons.tools import get_usable_path -class BrowserDownloadManager(object): +class DownloadManager(object): def __init__(self, browser): """ diff --git a/DrissionPage/_units/download_manager.pyi b/DrissionPage/_units/download_manager.pyi index c77be03..466270b 100644 --- a/DrissionPage/_units/download_manager.pyi +++ b/DrissionPage/_units/download_manager.pyi @@ -5,7 +5,7 @@ from .._base.browser import Browser from .._pages.chromium_page import ChromiumPage -class BrowserDownloadManager(object): +class DownloadManager(object): _browser: Browser = ... _page: ChromiumPage = ... _missions: Dict[str, DownloadMission] = ... @@ -56,7 +56,7 @@ class TabDownloadSettings(object): class DownloadMission(object): tab_id: str = ... - _mgr: BrowserDownloadManager = ... + _mgr: DownloadManager = ... url: str = ... id: str = ... path: str = ... @@ -68,7 +68,7 @@ class DownloadMission(object): save_path: str = ... _is_done: bool = ... - def __init__(self, mgr: BrowserDownloadManager, tab_id: str, _id: str, path: str, name: str, url: str, + def __init__(self, mgr: DownloadManager, tab_id: str, _id: str, path: str, name: str, url: str, save_path: str): ... @property diff --git a/DrissionPage/_units/element_states.py b/DrissionPage/_units/element_states.py index 73d724d..8f3b524 100644 --- a/DrissionPage/_units/element_states.py +++ b/DrissionPage/_units/element_states.py @@ -7,7 +7,7 @@ from .._commons.web import location_in_viewport from ..errors import CDPError, NoRectError -class ChromiumElementStates(object): +class ElementStates(object): def __init__(self, ele): """ :param ele: ChromiumElement diff --git a/DrissionPage/_units/element_states.pyi b/DrissionPage/_units/element_states.pyi index 81ebad5..5c2485f 100644 --- a/DrissionPage/_units/element_states.pyi +++ b/DrissionPage/_units/element_states.pyi @@ -8,7 +8,7 @@ from typing import Union, Tuple, List from .._elements.chromium_element import ChromiumShadowRoot, ChromiumElement -class ChromiumElementStates(object): +class ElementStates(object): def __init__(self, ele: ChromiumElement): self._ele: ChromiumElement = ... diff --git a/DrissionPage/_units/ids.py b/DrissionPage/_units/ids.py new file mode 100644 index 0000000..5e1ee80 --- /dev/null +++ b/DrissionPage/_units/ids.py @@ -0,0 +1,57 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" + + +class Ids(object): + def __init__(self, ele): + self._ele = ele + + @property + def node_id(self): + """返回元素cdp中的node id""" + return self._ele._node_id + + @property + def obj_id(self): + """返回元素js中的object id""" + return self._ele._obj_id + + @property + def backend_id(self): + """返回backend id""" + return self._ele._backend_id + + +class ElementIds(Ids): + @property + def doc_id(self): + """返回所在document的object id""" + return self._ele._doc_id + + +class FrameIds(object): + def __init__(self, frame): + self._frame = frame + + @property + def tab_id(self): + """返回当前标签页id""" + return self._frame._tab_id + + @property + def backend_id(self): + """返回cdp中的node id""" + return self._frame._backend_id + + @property + def obj_id(self): + """返回frame元素的object id""" + return self._frame.frame_ele.ids.obj_id + + @property + def node_id(self): + """返回cdp中的node id""" + return self._frame.frame_ele.ids.node_id diff --git a/DrissionPage/_units/ids.pyi b/DrissionPage/_units/ids.pyi new file mode 100644 index 0000000..f224259 --- /dev/null +++ b/DrissionPage/_units/ids.pyi @@ -0,0 +1,45 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" +from typing import Union + +from .._pages.chromium_frame import ChromiumFrame +from .._elements.chromium_element import ChromiumElement, ChromiumShadowRoot + + +class Ids(object): + def __init__(self, ele: Union[ChromiumElement, ChromiumShadowRoot]): + self._ele: Union[ChromiumElement, ChromiumShadowRoot] = ... + + @property + def node_id(self) -> str: ... + + @property + def obj_id(self) -> str: ... + + @property + def backend_id(self) -> str: ... + + +class ElementIds(Ids): + @property + def doc_id(self) -> str: ... + + +class FrameIds(object): + def __init__(self, frame: ChromiumFrame): + self._frame: ChromiumFrame = ... + + @property + def tab_id(self) -> str: ... + + @property + def backend_id(self) -> str: ... + + @property + def obj_id(self) -> str: ... + + @property + def node_id(self) -> str: ... diff --git a/DrissionPage/_units/scroller.py b/DrissionPage/_units/scroller.py new file mode 100644 index 0000000..1771468 --- /dev/null +++ b/DrissionPage/_units/scroller.py @@ -0,0 +1,171 @@ +# -*- coding:utf-8 -*- +from time import sleep, perf_counter + + +class Scroller(object): + """用于滚动的对象""" + + def __init__(self, ele): + """ + :param ele: 元素对象 + """ + self._driver = ele + self.t1 = self.t2 = 'this' + self._wait_complete = False + + def _run_js(self, js): + js = js.format(self.t1, self.t2, self.t2) + self._driver.run_js(js) + self._wait_scrolled() + + def to_top(self): + """滚动到顶端，水平位置不变""" + self._run_js('{}.scrollTo({}.scrollLeft, 0);') + + def to_bottom(self): + """滚动到底端，水平位置不变""" + self._run_js('{}.scrollTo({}.scrollLeft, {}.scrollHeight);') + + def to_half(self): + """滚动到垂直中间位置，水平位置不变""" + self._run_js('{}.scrollTo({}.scrollLeft, {}.scrollHeight/2);') + + def to_rightmost(self): + """滚动到最右边，垂直位置不变""" + self._run_js('{}.scrollTo({}.scrollWidth, {}.scrollTop);') + + def to_leftmost(self): + """滚动到最左边，垂直位置不变""" + self._run_js('{}.scrollTo(0, {}.scrollTop);') + + def to_location(self, x, y): + """滚动到指定位置 + :param x: 水平距离 + :param y: 垂直距离 + :return: None + """ + self._run_js(f'{{}}.scrollTo({x}, {y});') + + def up(self, pixel=300): + """向上滚动若干像素，水平位置不变 + :param pixel: 滚动的像素 + :return: None + """ + pixel = -pixel + self._run_js(f'{{}}.scrollBy(0, {pixel});') + + def down(self, pixel=300): + """向下滚动若干像素，水平位置不变 + :param pixel: 滚动的像素 + :return: None + """ + self._run_js(f'{{}}.scrollBy(0, {pixel});') + + def left(self, pixel=300): + """向左滚动若干像素，垂直位置不变 + :param pixel: 滚动的像素 + :return: None + """ + pixel = -pixel + self._run_js(f'{{}}.scrollBy({pixel}, 0);') + + def right(self, pixel=300): + """向右滚动若干像素，垂直位置不变 + :param pixel: 滚动的像素 + :return: None + """ + self._run_js(f'{{}}.scrollBy({pixel}, 0);') + + def _wait_scrolled(self): + if not self._wait_complete: + return + + page = self._driver.page if 'ChromiumElement' in str(type(self._driver)) else self._driver + r = page.run_cdp('Page.getLayoutMetrics') + x = r['layoutViewport']['pageX'] + y = r['layoutViewport']['pageY'] + + end_time = perf_counter() + self._driver.page.timeout + while perf_counter() < end_time: + sleep(.1) + r = page.run_cdp('Page.getLayoutMetrics') + x1 = r['layoutViewport']['pageX'] + y1 = r['layoutViewport']['pageY'] + + if x == x1 and y == y1: + break + + x = x1 + y = y1 + + +class ElementScroller(Scroller): + def to_see(self, center=None): + """滚动页面直到元素可见 + :param center: 是否尽量滚动到页面正中，为None时如果被遮挡，则滚动到页面正中 + :return: None + """ + self._driver.page.scroll.to_see(self._driver, center=center) + + def to_center(self): + """元素尽量滚动到视口中间""" + self._driver.page.scroll.to_see(self._driver, center=True) + + +class PageScroller(Scroller): + def __init__(self, page): + """ + :param page: 页面对象 + """ + super().__init__(page) + self.t1 = 'window' + self.t2 = 'document.documentElement' + + def to_see(self, loc_or_ele, center=None): + """滚动页面直到元素可见 + :param loc_or_ele: 元素的定位信息，可以是loc元组，或查询字符串 + :param center: 是否尽量滚动到页面正中，为None时如果被遮挡，则滚动到页面正中 + :return: None + """ + ele = self._driver._ele(loc_or_ele) + self._to_see(ele, center) + + def _to_see(self, ele, center): + """执行滚动页面直到元素可见 + :param ele: 元素对象 + :param center: 是否尽量滚动到页面正中，为None时如果被遮挡，则滚动到页面正中 + :return: None + """ + txt = 'true' if center else 'false' + ele.run_js(f'this.scrollIntoViewIfNeeded({txt});') + if center or (center is not False and ele.states.is_covered): + ele.run_js('''function getWindowScrollTop() {var scroll_top = 0; + if (document.documentElement && document.documentElement.scrollTop) { + scroll_top = document.documentElement.scrollTop; + } else if (document.body) {scroll_top = document.body.scrollTop;} + return scroll_top;} + const { top, height } = this.getBoundingClientRect(); + const elCenter = top + height / 2; + const center = window.innerHeight / 2; + window.scrollTo({top: getWindowScrollTop() - (center - elCenter), + behavior: 'instant'});''') + self._wait_scrolled() + + +class FrameScroller(PageScroller): + def __init__(self, frame): + """ + :param frame: ChromiumFrame对象 + """ + self._driver = frame.doc_ele + self.t1 = self.t2 = 'this.documentElement' + self._wait_complete = False + + def to_see(self, loc_or_ele, center=None): + """滚动页面直到元素可见 + :param loc_or_ele: 元素的定位信息，可以是loc元组，或查询字符串 + :param center: 是否尽量滚动到页面正中，为None时如果被遮挡，则滚动到页面正中 + :return: None + """ + ele = loc_or_ele if 'ChromiumElement' in str(type(loc_or_ele)) else self._driver._ele(loc_or_ele) + self._to_see(ele, center) diff --git a/DrissionPage/_units/scroller.pyi b/DrissionPage/_units/scroller.pyi new file mode 100644 index 0000000..438da32 --- /dev/null +++ b/DrissionPage/_units/scroller.pyi @@ -0,0 +1,73 @@ +# -*- coding:utf-8 -*- +from typing import Union + +from .._elements.chromium_element import ChromiumElement +from .._pages.chromium_base import ChromiumBase +from .._pages.chromium_frame import ChromiumFrame +from .._pages.chromium_page import ChromiumPage + + +class Scroller(object): + def __init__(self, page_or_ele: Union[ChromiumBase, ChromiumElement, ChromiumFrame]): + self.t1: str = ... + self.t2: str = ... + self._driver: Union[ChromiumPage, ChromiumElement, ChromiumFrame] = ... + self._wait_complete: bool = ... + + def _run_js(self, js: str): ... + + def to_top(self) -> None: ... + + def to_bottom(self) -> None: ... + + def to_half(self) -> None: ... + + def to_rightmost(self) -> None: ... + + def to_leftmost(self) -> None: ... + + def to_location(self, x: int, y: int) -> None: ... + + def up(self, pixel: int = 300) -> None: ... + + def down(self, pixel: int = 300) -> None: ... + + def left(self, pixel: int = 300) -> None: ... + + def right(self, pixel: int = 300) -> None: ... + + def _wait_scrolled(self) -> None: ... + + +class ElementScroller(Scroller): + + def to_see(self, center: Union[bool, None] = None) -> None: ... + + def to_center(self) -> None: ... + + +class PageScroller(Scroller): + def __init__(self, page: ChromiumBase): ... + + def to_see(self, loc_or_ele: Union[str, tuple, ChromiumElement], center: Union[bool, None] = None) -> None: ... + + def _to_see(self, ele: ChromiumElement, center: Union[bool, None]) -> None: ... + + +class FrameScroller(PageScroller): + def __init__(self, frame): + """ + :param frame: ChromiumFrame对象 + """ + self._driver = frame.doc_ele + self.t1 = self.t2 = 'this.documentElement' + self._wait_complete = False + + def to_see(self, loc_or_ele, center=None): + """滚动页面直到元素可见 + :param loc_or_ele: 元素的定位信息，可以是loc元组，或查询字符串 + :param center: 是否尽量滚动到页面正中，为None时如果被遮挡，则滚动到页面正中 + :return: None + """ + ele = loc_or_ele if isinstance(loc_or_ele, ChromiumElement) else self._driver._ele(loc_or_ele) + self._to_see(ele, center) diff --git a/DrissionPage/_units/tab_rect.py b/DrissionPage/_units/tab_rect.py index df27036..b72107b 100644 --- a/DrissionPage/_units/tab_rect.py +++ b/DrissionPage/_units/tab_rect.py @@ -5,7 +5,7 @@ """ -class ChromiumTabRect(object): +class TabRect(object): def __init__(self, page): self._page = page diff --git a/DrissionPage/_units/tab_rect.pyi b/DrissionPage/_units/tab_rect.pyi index d21aa1e..6b80684 100644 --- a/DrissionPage/_units/tab_rect.pyi +++ b/DrissionPage/_units/tab_rect.pyi @@ -9,7 +9,7 @@ from .._pages.chromium_page import ChromiumPage from .._pages.chromium_tab import ChromiumTab -class ChromiumTabRect(object): +class TabRect(object): def __init__(self, page: Union[ChromiumPage, ChromiumTab]): self._page: Union[ChromiumPage, ChromiumTab] = ... diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 8236b30..56de34e 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -5,7 +5,7 @@ from .._commons.constants import Settings from ..errors import WaitTimeoutError, NoRectError -class ChromiumBaseWaiter(object): +class BaseWaiter(object): def __init__(self, page_or_ele): """ :param page_or_ele: 页面对象或元素对象 @@ -190,7 +190,7 @@ class ChromiumBaseWaiter(object): return False -class ChromiumTabWaiter(ChromiumBaseWaiter): +class TabWaiter(BaseWaiter): def downloads_done(self, timeout=None, cancel_if_timeout=True): """等待所有浏览器下载任务结束 @@ -219,7 +219,7 @@ class ChromiumTabWaiter(ChromiumBaseWaiter): return True -class ChromiumPageWaiter(ChromiumTabWaiter): +class PageWaiter(TabWaiter): def __init__(self, page): super().__init__(page) # self._listener = None @@ -270,7 +270,7 @@ class ChromiumPageWaiter(ChromiumTabWaiter): return True -class ChromiumElementWaiter(object): +class ElementWaiter(object): """等待元素在dom中某种状态，如删除、显示、隐藏""" def __init__(self, page, ele): @@ -417,10 +417,10 @@ class ChromiumElementWaiter(object): return False -class FrameWaiter(ChromiumBaseWaiter, ChromiumElementWaiter): +class FrameWaiter(BaseWaiter, ElementWaiter): def __init__(self, frame): """ :param frame: ChromiumFrame对象 """ super().__init__(frame) - super(ChromiumBaseWaiter, self).__init__(frame, frame.frame_ele) + super(BaseWaiter, self).__init__(frame, frame.frame_ele) diff --git a/DrissionPage/_units/waiter.pyi b/DrissionPage/_units/waiter.pyi index 1f4da31..d353178 100644 --- a/DrissionPage/_units/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -13,7 +13,7 @@ from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage -class ChromiumBaseWaiter(object): +class BaseWaiter(object): def __init__(self, page: ChromiumBase): self._driver: ChromiumBase = ... @@ -54,12 +54,12 @@ class ChromiumBaseWaiter(object): raise_err: bool = None) -> bool: ... -class ChromiumTabWaiter(ChromiumBaseWaiter): +class TabWaiter(BaseWaiter): def downloads_done(self, timeout: float = None, cancel_if_timeout: bool = True) -> bool: ... -class ChromiumPageWaiter(ChromiumTabWaiter): +class PageWaiter(TabWaiter): _driver: ChromiumPage = ... def new_tab(self, timeout: float = None, raise_err: bool = None) -> Union[str, bool]: ... @@ -67,7 +67,7 @@ class ChromiumPageWaiter(ChromiumTabWaiter): def all_downloads_done(self, timeout: float = None, cancel_if_timeout: bool = True) -> bool: ... -class ChromiumElementWaiter(object): +class ElementWaiter(object): def __init__(self, page: ChromiumBase, ele: ChromiumElement): @@ -97,5 +97,5 @@ class ChromiumElementWaiter(object): def _wait_state(self, attr: str, mode: bool = False, timeout: float = None, raise_err: bool = None) -> bool: ... -class FrameWaiter(ChromiumBaseWaiter, ChromiumElementWaiter): +class FrameWaiter(BaseWaiter, ElementWaiter): def __init__(self, frame: ChromiumFrame): ... diff --git a/DrissionPage/common.py b/DrissionPage/common.py index 1dab5d2..72a0deb 100644 --- a/DrissionPage/common.py +++ b/DrissionPage/common.py @@ -9,3 +9,5 @@ from ._commons.keys import Keys from ._commons.by import By from ._commons.constants import Settings from ._commons.tools import wait_until + +__all__ = ['make_session_ele', 'ActionChains', 'Keys', 'By', 'Settings', 'wait_until'] From a6037e960ebdf564932825d1e454af2162f5c001 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sat, 11 Nov 2023 19:33:52 +0800 Subject: [PATCH 074/182] =?UTF-8?q?=E5=88=A0=E9=99=A4to=5Ftab()=E3=80=81to?= =?UTF-8?q?=5Fmain=5Ftab()=E3=80=81main=5Ftab=EF=BC=9Bnew=5Ftab()=E5=88=A0?= =?UTF-8?q?=E9=99=A4switch=5Fto=E5=8F=82=E6=95=B0=EF=BC=8C=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0new=5Fwindow=E3=80=81background=E3=80=81new=5Fcontext?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_page.py | 99 ++++++--------------------- DrissionPage/_pages/chromium_page.pyi | 18 ++--- 2 files changed, 25 insertions(+), 92 deletions(-) diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index ae06dd4..2732bbe 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -85,7 +85,6 @@ class ChromiumPage(ChromiumBase): def _page_init(self): """浏览器相关设置""" self._rect = None - self._main_tab = self.tab_id self._browser.connect_to_page() @property @@ -103,10 +102,6 @@ class ChromiumPage(ChromiumBase): """返回所有标签页id组成的列表""" return self.browser.tabs - @property - def main_tab(self): - return self._main_tab - @property def latest_tab(self): """返回最新的标签页id，最新标签页指最后创建或最后被激活的""" @@ -154,75 +149,31 @@ class ChromiumPage(ChromiumBase): """ return self._browser.find_tabs(title, url, tab_type, single) - def _new_tab(self, url=None, switch_to=False): - """新建一个标签页,该标签页在最后面 + def new_tab(self, url=None, new_window=False, background=False, new_context=False): + """新建一个标签页 :param url: 新标签页跳转到的网址 - :param switch_to: 新建标签页后是否把焦点移过去 - :return: 新标签页的id + :param new_window: 是否在新窗口打开标签页 + :param background: 是否不激活新标签页，如new_window为True则无效 + :param new_context: 是否创建新的上下文 + :return: 新标签页对象 """ - if switch_to: - tid = self.run_cdp('Target.createTarget', url='')['targetId'] - self._to_tab(tid, read_doc=False) - if url: - self.get(url) + bid = None + if new_context: + bid = self.browser.run_cdp('Target.createBrowserContext', **kwargs)['browserContextId'] - elif url: - tid = self.run_cdp('Target.createTarget', url=url)['targetId'] + kwargs = {'url': ''} + if new_window: + kwargs['newWindow'] = True + if background: + kwargs['background'] = True + if bid: + kwargs['browserContextId'] = bid - else: - tid = self.run_cdp('Target.createTarget', url='')['targetId'] - - return tid - - def new_tab(self, url=None, switch_to=False): - """新建一个标签页,该标签页在最后面 - :param url: 新标签页跳转到的网址 - :param switch_to: 新建标签页后是否把焦点移过去 - :return: switch_to为False时返回新标签页对象，否则返回当前对象， - """ - tid = self._new_tab(url, switch_to) - return self if switch_to else ChromiumTab(self, tid) - - def to_main_tab(self): - """跳转到主标签页""" - self.to_tab(self._main_tab) - - def to_tab(self, tab_or_id=None, activate=True): - """跳转到标签页 - :param tab_or_id: 标签页对象或id，默认跳转到main_tab - :param activate: 切换后是否变为活动状态 - :return: None - """ - self._to_tab(tab_or_id, activate) - - def _to_tab(self, tab_or_id=None, activate=True, read_doc=True): - """跳转到标签页 - :param tab_or_id: 标签页对象或id，默认跳转到main_tab - :param activate: 切换后是否变为活动状态 - :param read_doc: 切换后是否读取文档 - :return: None - """ - tabs = self.tabs - if not tab_or_id: - tab_id = self._main_tab - elif isinstance(tab_or_id, ChromiumTab): - tab_id = tab_or_id.tab_id - else: - tab_id = tab_or_id - - if tab_id not in tabs: - tab_id = self.latest_tab - - if activate: - self.browser.activate_tab(tab_id) - - if tab_id == self.tab_id: - return - - self.driver.stop() - self._driver_init(tab_id) - if read_doc and self.ready_state in ('complete', None): - self._get_document() + tid = self.run_cdp('Target.createTarget', **kwargs)['targetId'] + tab = ChromiumTab(self, tab_id=tid) + if url: + tab.get(url) + return tab def close_tabs(self, tabs_or_ids=None, others=False): """关闭传入的标签页，默认关闭当前页。可传入多个 @@ -250,20 +201,12 @@ class ChromiumPage(ChromiumBase): self.quit() return - if self.tab_id in tabs: - self.driver.stop() - for tab in tabs: self.browser.close_tab(tab) sleep(.2) while self.tabs_count != end_len: sleep(.1) - if self._main_tab in tabs: - self._main_tab = self.tabs[0] - - self.to_tab() - def close_other_tabs(self, tabs_or_ids=None): """关闭传入的标签页以外标签页，默认保留当前页。可传入多个 :param tabs_or_ids: 要保留的标签页对象或id，可传入列表或元组，为None时保存当前页 diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index a5f5ef7..9dc2102 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -21,7 +21,6 @@ class ChromiumPage(ChromiumBase): tab_id: str = None, timeout: float = None): self._driver_options: ChromiumOptions = ... - self._main_tab: str = ... self._browser: Browser = ... self._rect: Optional[TabRect] = ... @@ -63,20 +62,11 @@ class ChromiumPage(ChromiumBase): def find_tabs(self, title: str = None, url: str = None, tab_type: Union[str, list, tuple] = None, single: bool = True) -> Union[str, List[str]]: ... - def _new_tab(self, url=None, switch_to=False) -> str: ... + def new_tab(self, url: str = None, new_window: bool = False, background: bool = False, + new_context: bool = False) -> Union[ChromiumTab, ChromiumPage]: ... - def new_tab(self, url: str = None, switch_to: bool = False) -> Union[ChromiumTab, ChromiumPage]: ... - - def to_main_tab(self) -> None: ... - - def to_tab(self, tab_or_id: Union[str, ChromiumTab] = None, activate: bool = True) -> None: ... - - def _to_tab(self, tab_or_id: Union[str, ChromiumTab] = None, activate: bool = True, - read_doc: bool = True) -> None: ... - - def close_tabs(self, tabs_or_ids: Union[ - str, ChromiumTab, List[Union[str, ChromiumTab]], Tuple[Union[str, ChromiumTab]]] = None, - others: bool = False) -> None: ... + def close_tabs(self, tabs_or_ids: Union[str, ChromiumTab, List[Union[str, ChromiumTab]], + Tuple[Union[str, ChromiumTab]]] = None, others: bool = False) -> None: ... def close_other_tabs(self, tabs_or_ids: Union[ str, ChromiumTab, List[Union[str, ChromiumTab]], Tuple[Union[str, ChromiumTab]]] = None) -> None: ... From 146e52749479c685d422381ed1ad154c415e20f0 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sat, 11 Nov 2023 20:05:51 +0800 Subject: [PATCH 075/182] =?UTF-8?q?=E5=AF=B9iframe=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E8=BF=9B=E8=A1=8C=E5=AE=8C=E5=85=A8=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/chromium_driver.py | 25 ++- DrissionPage/_pages/chromium_base.py | 54 ++++- DrissionPage/_pages/chromium_frame.py | 274 ++++++++++++------------- DrissionPage/_pages/chromium_frame.pyi | 12 +- setup.py | 2 +- 5 files changed, 193 insertions(+), 174 deletions(-) diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index 9d63a0d..c47f92f 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -97,6 +97,7 @@ class ChromiumDriver(object): def _recv_loop(self): """接收浏览器信息的守护线程方法""" while not self._stopped.is_set(): + print('收') try: # self._ws.settimeout(1) msg_json = self._ws.recv() @@ -110,7 +111,7 @@ class ChromiumDriver(object): if self._debug: if self._debug is True or 'id' in msg or (isinstance(self._debug, str) and msg.get('method', '').startswith(self._debug)): - print(f'<收 {msg_json}') + print(f'<收 {self.id} {msg_json}') elif isinstance(self._debug, (list, tuple, set)): for m in self._debug: if msg.get('method', '').startswith(m): @@ -139,11 +140,11 @@ class ChromiumDriver(object): function = self.event_handlers.get(event['method']) if function: - # if self._debug: - # print(f'开始执行 {function.__name__}') + if self._debug: + print(f'开始执行 {function.__name__}') function(**event['params']) - # if self._debug: - # print(f'执行 {function.__name__}完毕') + if self._debug: + print(f'执行 {function.__name__}完毕') self.event_queue.task_done() @@ -186,6 +187,20 @@ class ChromiumDriver(object): if self._ws: self._ws.close() self._ws = None + + while not self.event_queue.empty(): + event = self.event_queue.get_nowait() + function = self.event_handlers.get(event['method']) + if function: + if self._debug: + print(f'开始执行 {function.__name__}') + try: + function(**event['params']) + except: + pass + if self._debug: + print(f'执行 {function.__name__}完毕') + self.event_handlers.clear() self.method_results.clear() self.event_queue.queue.clear() diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 9b5f141..d09eaae 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -46,6 +46,8 @@ class ChromiumBase(BasePage): self._listener = None self._has_alert = False self._ready_state = None + if self._debug: + print('在__init__变成None') self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc self._download_path = str(Path('.').absolute()) @@ -88,6 +90,8 @@ class ChromiumBase(BasePage): if self.ready_state == 'complete' and self._ready_state is None: self._get_document() self._ready_state = 'complete' + if self._debug: + print(f'{self._frame_id}在connect_browser变成complete') def _driver_init(self, tab_id): """新建页面、页面刷新、切换标签页后要进行的cdp参数初始化 @@ -107,7 +111,8 @@ class ChromiumBase(BasePage): r = self.run_cdp('Page.getFrameTree') for i in findall(r"'id': '(.*?)'", str(r)): self.browser._frames[i] = self.tab_id - self._frame_id = r['frameTree']['frame']['id'] + if not hasattr(self, '_frame_id'): + self._frame_id = r['frameTree']['frame']['id'] self._driver.set_listener('Page.frameStartedLoading', self._onFrameStartedLoading) self._driver.set_listener('Page.frameNavigated', self._onFrameNavigated) @@ -149,6 +154,10 @@ class ChromiumBase(BasePage): """页面开始加载时执行""" self.browser._frames[kwargs['frameId']] = self.tab_id if kwargs['frameId'] == self._frame_id: + if self._debug: + print(f'{self._frame_id}触发FrameStartedLoading') + print('在FrameStartedLoading变成loading') + self._doc_got = False self._ready_state = 'loading' self._is_loading = True @@ -156,43 +165,64 @@ class ChromiumBase(BasePage): t = Thread(target=self._wait_to_stop) t.daemon = True t.start() + if self._debug: - print(f'frameStartedLoading {kwargs}') + print(f'{self._frame_id}执行FrameStartedLoading完毕') def _onFrameNavigated(self, **kwargs): """页面跳转时执行""" if kwargs['frame']['id'] == self._frame_id: + if self._debug: + print(f'{self._frame_id}触发FrameNavigated') + print('在FrameNavigated变成loading') + self._doc_got = False self._ready_state = 'loading' self._is_loading = True + if self._debug: - print(f'FrameNavigated {kwargs}') + print(f'>>> FrameNavigated {kwargs}') def _onDomContentEventFired(self, **kwargs): """在页面刷新、变化后重新读取页面内容""" + if self._debug: + print(f'{self._frame_id}触发DomContentEventFired') + print('在DomContentEventFired变成interactive') + self._ready_state = 'interactive' if self.page_load_strategy == 'eager': self.run_cdp('Page.stopLoading') + if self._debug: - print(f'DomContentEventFired {kwargs}') + print(f'{self._frame_id}执行DomContentEventFired完毕') def _onLoadEventFired(self, **kwargs): """在页面刷新、变化后重新读取页面内容""" - self._ready_state = 'complete' if self._debug: - print(f'LoadEventFired {kwargs}') + print(f'{self._frame_id}触发LoadEventFired') + print('在LoadEventFired变成complete') + + self._ready_state = 'complete' self._get_document() self._doc_got = True + if self._debug: + print(f'{self._frame_id}执行LoadEventFired完毕') + def _onFrameStoppedLoading(self, **kwargs): """页面加载完成后执行""" self.browser._frames[kwargs['frameId']] = self.tab_id if kwargs['frameId'] == self._frame_id and self._doc_got is False: - self._ready_state = 'complete' if self._debug: - print(f'FrameStoppedLoading {kwargs}') + print(f'{self._frame_id}触发FrameStoppedLoading') + print('在FrameStoppedLoading变成complete') + + self._ready_state = 'complete' self._get_document() + if self._debug: + print(f'{self._frame_id}执行FrameStoppedLoading完毕') + def _onFileChooserOpened(self, **kwargs): """文件选择框打开时执行""" if self._upload_list: @@ -836,7 +866,10 @@ class ChromiumBase(BasePage): sleep(.1) - self.stop_loading() + try: + self.stop_loading() + except CDPError: + pass return False def _d_connect(self, to_url, times=0, interval=1, show_errmsg=False, timeout=None): @@ -854,7 +887,7 @@ class ChromiumBase(BasePage): err = None end_time = perf_counter() + timeout try: - result = self.run_cdp('Page.navigate', url=to_url, _timeout=timeout) + result = self.run_cdp('Page.navigate', frameId=self._frame_id, url=to_url, _timeout=timeout) if 'errorText' in result: err = ConnectionError(result['errorText']) except TimeoutError: @@ -877,7 +910,6 @@ class ChromiumBase(BasePage): sleep(interval) if self._debug or show_errmsg: print(f'重试{t + 1} {to_url}') - self.stop_loading() continue if not err: diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 1d412b8..6d7b4c6 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -5,7 +5,6 @@ """ from copy import copy from re import search, findall -from threading import Thread from time import sleep, perf_counter from .._elements.chromium_element import ChromiumElement @@ -35,21 +34,22 @@ class ChromiumFrame(ChromiumBase): self.address = page.address node = page.run_cdp('DOM.describeNode', backendNodeId=ele.ids.backend_id)['node'] - self.frame_id = node['frameId'] self._tab_id = page.tab_id self._backend_id = ele.ids.backend_id self._frame_ele = ele self._states = None self._ids = FrameIds(self) + self._is_init_get_doc = True + self._frame_id = node['frameId'] if self._is_inner_frame(): self._is_diff_domain = False self.doc_ele = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) super().__init__(page.address, page.tab_id, page.timeout) - self._frame_id = self.frame_id else: self._is_diff_domain = True - super().__init__(page.address, self.frame_id, page.timeout) + delattr(self, '_frame_id') + super().__init__(page.address, node['frameId'], page.timeout) obj_id = super().run_js('document;', as_expr=True)['objectId'] self.doc_ele = ChromiumElement(self, obj_id=obj_id) @@ -57,10 +57,6 @@ class ChromiumFrame(ChromiumBase): while perf_counter() < end_time and self.url == 'about:blank': sleep(.1) - t = Thread(target=self._check_alive) - t.daemon = True - t.start() - def __call__(self, loc_or_str, timeout=None): """在内部查找元素 例：ele2 = ele1('@id=ele_id') @@ -77,11 +73,12 @@ class ChromiumFrame(ChromiumBase): def _d_set_runtime_settings(self): """重写设置浏览器运行参数方法""" - self._timeouts = copy(self._target_page.timeouts) - self.retry_times = self._target_page.retry_times - self.retry_interval = self._target_page.retry_interval - self._page_load_strategy = self._target_page.page_load_strategy - self._download_path = self._target_page.download_path + if not hasattr(self, '_timeouts'): + self._timeouts = copy(self._target_page.timeouts) + self.retry_times = self._target_page.retry_times + self.retry_interval = self._target_page.retry_interval + self._page_load_strategy = self._target_page.page_load_strategy if not self._is_diff_domain else 'normal' + self._download_path = self._target_page.download_path def _driver_init(self, tab_id, is_init=True): """避免出现服务器500错误 @@ -93,65 +90,90 @@ class ChromiumFrame(ChromiumBase): except: self.browser.driver.get(f'http://{self.address}/json') super()._driver_init(tab_id) - self.driver.set_listener('Inspector.detached', self._onInspectorDetached) + self._driver.set_listener('Inspector.detached', self._onInspectorDetached) def _reload(self): """重新获取document""" + self._is_loading = True debug = self._debug d_debug = self.driver._debug - old_driver = self.driver + self._is_init_get_doc = False + self._doc_got = False if debug: - print('重新获取document') + print(f'{self._frame_id} reload 开始') - self._frame_ele = ChromiumElement(self._target_page, backend_id=self._backend_id) - node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self._frame_ele.ids.backend_id)['node'] + try: + self._frame_ele = ChromiumElement(self._target_page, backend_id=self._backend_id) + except ElementLossError: + return + node = self._target_page.run_cdp('DOM.describeNode', + backendNodeId=self._frame_ele.ids.backend_id)['node'] - end_time = perf_counter() + self.timeout - while perf_counter() < end_time: - try: - if self._is_inner_frame(): - self._is_diff_domain = False - self.doc_ele = ChromiumElement(self._target_page, - backend_id=node['contentDocument']['backendNodeId']) - super().__init__(self.address, self._target_page.tab_id, self._target_page.timeout) - self._frame_id = self.frame_id - self._debug = debug - self.driver._debug = d_debug - else: - self._is_diff_domain = True - self._driver.stop() - super().__init__(self.address, self.frame_id, self._target_page.timeout) + self._driver.stop() + + if self._is_inner_frame(): + self._is_diff_domain = False + self.doc_ele = ChromiumElement(self._target_page, + backend_id=node['contentDocument']['backendNodeId']) + self._frame_id = node['frameId'] + super().__init__(self.address, self._target_page.tab_id, self._target_page.timeout) + self._debug = debug + self.driver._debug = d_debug + + else: + self._is_diff_domain = True + super().__init__(self.address, node['frameId'], self._target_page.timeout) + end_time = perf_counter() + self.timeouts.page_load + while perf_counter() < end_time: + try: obj_id = super().run_js('document;', as_expr=True)['objectId'] self.doc_ele = ChromiumElement(self, obj_id=obj_id) - self._debug = debug - self.driver._debug = d_debug + break + except Exception as e: + sleep(.1) + if self._debug: + print(f'获取doc失败，重试 {e}') + else: + raise GetDocumentError + self._debug = debug + self.driver._debug = d_debug + + self._is_loading = False + + if self._debug: + print(f'{self._frame_id} reload 完毕') + + def _get_document(self): + if self._is_reading: + return + self._is_reading = True + end_time = perf_counter() + 10 + while perf_counter() < end_time: + try: + b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] + self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id)['object']['objectId'] break except: - pass - - sleep(.1) - + continue else: raise GetDocumentError - old_driver.stop() - self.wait.load_complete() + r = self.run_cdp('Page.getFrameTree') + for i in findall(r"'id': '(.*?)'", str(r)): + self.browser._frames[i] = self.tab_id - def _check_ok(self): - """用于应付同域异域之间跳转导致元素丢失问题""" - if self._driver._stopped.is_set(): - self._reload() - - try: - self._target_page.run_cdp('DOM.describeNode', nodeId=self.ids.node_id) - except ElementLossError: - self._reload() - # sleep(2) + if self._is_init_get_doc: # 阻止reload时标识 + self._is_loading = False + self._is_reading = False def _get_new_document(self): """刷新cdp使用的document数据""" if self._is_reading: return + + if self._debug: + print('>>> get new doc') + self._is_reading = True end_time = perf_counter() + 10 while perf_counter() < end_time: @@ -179,19 +201,66 @@ class ChromiumFrame(ChromiumBase): self._is_loading = False self._is_reading = False + if self._debug: + print('>>> new doc got') + + def _onLoadEventFired(self, **kwargs): + """在页面刷新、变化后重新读取页面内容""" + if self._debug: + print(f'{self._frame_id}触发在frame的LoadEventFired') + print('在frame的LoadEventFired变成complete') + + self._ready_state = 'complete' + self._get_new_document() + self._doc_got = True + + if self._debug: + print(f'{self._frame_id}执行frame的LoadEventFired完毕') + def _onFrameStoppedLoading(self, **kwargs): """页面加载完成后触发""" self.browser._frames[kwargs['frameId']] = self.tab_id - if kwargs['frameId'] == self.frame_id: - self._ready_state = 'complete' + if kwargs['frameId'] == self._frame_id and self._doc_got is False: if self._debug: - print(f'FrameStoppedLoading {kwargs}') + print(f'{self._frame_id}触发frame的FrameStoppedLoading') + print('在frame的FrameStoppedLoading变成complete') + + self._ready_state = 'complete' self._get_new_document() + if self._debug: + print(f'{self._frame_id}执行frame的FrameStoppedLoading完毕') + def _onInspectorDetached(self, **kwargs): - self._is_loading = True + """异域转同域或退出""" + if self._debug: + print(f'{self._frame_id}触发InspectorDetached') + + try: + self._frame_ele.attrs + except ElementLossError: + self._driver.stop() self._reload() + if self._debug: + print(f'{self._frame_id}执行InspectorDetached完毕') + + def _onFrameDetached(self, **kwargs): + """同域变异域""" + self.browser._frames.pop(kwargs['frameId'], None) + if kwargs['frameId'] == self._frame_id: + if self._debug: + print(f'{self._frame_id}触发FrameDetached') + + try: + self._frame_ele.attrs + except ElementLossError: + self._driver.stop() + self._reload() + + if self._debug: + print(f'{self._frame_id}执行FrameDetached完毕') + @property def page(self): return self._page @@ -208,19 +277,16 @@ class ChromiumFrame(ChromiumBase): @property def tag(self): """返回元素tag""" - self._check_ok() return self.frame_ele.tag @property def url(self): """返回frame当前访问的url""" - self._check_ok() return self.doc_ele.run_js('return this.location.href;') @property def html(self): """返回元素outerHTML文本""" - self._check_ok() tag = self.tag out_html = self._target_page.run_cdp('DOM.getOuterHTML', backendNodeId=self.frame_ele.ids.backend_id)[ 'outerHTML'] @@ -230,32 +296,27 @@ class ChromiumFrame(ChromiumBase): @property def inner_html(self): """返回元素innerHTML文本""" - self._check_ok() return self.doc_ele.run_js('return this.documentElement.outerHTML;') @property def title(self): """返回页面title""" - self._check_ok() r = self._ele('t:title', raise_err=False) return r.text if r else None @property def cookies(self): """以dict格式返回cookies""" - self._check_ok() return super().cookies if self._is_diff_domain else self.doc_ele.run_js('return this.cookie;') @property def attrs(self): """返回frame元素所有attribute属性""" - self._check_ok() return self.frame_ele.attrs @property def frame_size(self): """返回frame内页面尺寸，格式：(长, 高)""" - self._check_ok() w = self.doc_ele.run_js('return this.body.scrollWidth') h = self.doc_ele.run_js('return this.body.scrollHeight') return w, h @@ -263,19 +324,16 @@ class ChromiumFrame(ChromiumBase): @property def size(self): """返回frame元素大小""" - self._check_ok() return self.frame_ele.size @property def active_ele(self): """返回当前焦点所在元素""" - self._check_ok() return self.doc_ele.run_js('return this.activeElement;') @property def location(self): """返回frame元素左上角的绝对坐标""" - self._check_ok() return self.frame_ele.location @property @@ -286,13 +344,11 @@ class ChromiumFrame(ChromiumBase): @property def xpath(self): """返回frame的xpath绝对路径""" - self._check_ok() return self.frame_ele.xpath @property def css_path(self): """返回frame的css selector绝对路径""" - self._check_ok() return self.frame_ele.css_path @property @@ -319,8 +375,6 @@ class ChromiumFrame(ChromiumBase): sleep(.1) - # raise RuntimeError('获取document失败。') - @property def is_alive(self): """返回是否仍可用""" @@ -361,7 +415,6 @@ class ChromiumFrame(ChromiumBase): def refresh(self): """刷新frame页面""" - self._check_ok() self.doc_ele.run_js('this.location.reload();') def attr(self, attr): @@ -369,7 +422,6 @@ class ChromiumFrame(ChromiumBase): :param attr: 属性名 :return: 属性值文本，没有该属性返回None """ - self._check_ok() return self.frame_ele.attr(attr) def remove_attr(self, attr): @@ -377,7 +429,6 @@ class ChromiumFrame(ChromiumBase): :param attr: 属性名 :return: None """ - self._check_ok() self.frame_ele.remove_attr(attr) def run_js(self, script, *args, as_expr=False): @@ -387,7 +438,6 @@ class ChromiumFrame(ChromiumBase): :param as_expr: 是否作为表达式运行，为True时args无效 :return: 运行的结果 """ - self._check_ok() if script.startswith('this.scrollIntoView'): return self.frame_ele.run_js(script, *args, as_expr=as_expr) else: @@ -399,7 +449,6 @@ class ChromiumFrame(ChromiumBase): :param index: 当level_or_loc传入定位符，使用此参数选择第几个结果 :return: 上级元素对象 """ - self._check_ok() return self.frame_ele.parent(level_or_loc, index) def prev(self, filter_loc='', index=1, timeout=0, ele_only=True): @@ -410,7 +459,6 @@ class ChromiumFrame(ChromiumBase): :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 同级元素或节点 """ - self._check_ok() return self.frame_ele.prev(filter_loc, index, timeout, ele_only=ele_only) def next(self, filter_loc='', index=1, timeout=0, ele_only=True): @@ -421,7 +469,6 @@ class ChromiumFrame(ChromiumBase): :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 同级元素或节点 """ - self._check_ok() return self.frame_ele.next(filter_loc, index, timeout, ele_only=ele_only) def before(self, filter_loc='', index=1, timeout=None, ele_only=True): @@ -433,7 +480,6 @@ class ChromiumFrame(ChromiumBase): :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素前面的某个元素或节点 """ - self._check_ok() return self.frame_ele.before(filter_loc, index, timeout, ele_only=ele_only) def after(self, filter_loc='', index=1, timeout=None, ele_only=True): @@ -445,7 +491,6 @@ class ChromiumFrame(ChromiumBase): :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素后面的某个元素或节点 """ - self._check_ok() return self.frame_ele.after(filter_loc, index, timeout, ele_only=ele_only) def prevs(self, filter_loc='', timeout=0, ele_only=True): @@ -455,7 +500,6 @@ class ChromiumFrame(ChromiumBase): :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 同级元素或节点文本组成的列表 """ - self._check_ok() return self.frame_ele.prevs(filter_loc, timeout, ele_only=ele_only) def nexts(self, filter_loc='', timeout=0, ele_only=True): @@ -465,7 +509,6 @@ class ChromiumFrame(ChromiumBase): :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 同级元素或节点文本组成的列表 """ - self._check_ok() return self.frame_ele.nexts(filter_loc, timeout, ele_only=ele_only) def befores(self, filter_loc='', timeout=None, ele_only=True): @@ -476,7 +519,6 @@ class ChromiumFrame(ChromiumBase): :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素前面的元素或节点组成的列表 """ - self._check_ok() return self.frame_ele.befores(filter_loc, timeout, ele_only=ele_only) def afters(self, filter_loc='', timeout=None, ele_only=True): @@ -487,7 +529,6 @@ class ChromiumFrame(ChromiumBase): :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素前面的元素或节点组成的列表 """ - self._check_ok() return self.frame_ele.afters(filter_loc, timeout, ele_only=ele_only) def get_screenshot(self, path=None, name=None, as_bytes=None, as_base64=None): @@ -586,75 +627,14 @@ class ChromiumFrame(ChromiumBase): :param raise_err: 找不到元素是是否抛出异常，为None时根据全局设置 :return: ChromiumElement对象 """ - self._check_ok() if isinstance(loc_or_ele, ChromiumElement): return loc_or_ele self.wait.load_complete() - return self.doc_ele._ele(loc_or_ele, timeout, raise_err=raise_err) \ - if single else self.doc_ele.eles(loc_or_ele, timeout) - - def _d_connect(self, to_url, times=0, interval=1, show_errmsg=False, timeout=None): - """尝试连接，重试若干次 - :param to_url: 要访问的url - :param times: 重试次数 - :param interval: 重试间隔（秒） - :param show_errmsg: 是否抛出异常 - :param timeout: 连接超时时间 - :return: 是否成功，返回None表示不确定 - """ - self._check_ok() - err = None - timeout = timeout if timeout is not None else self.timeouts.page_load - - for t in range(times + 1): - err = None - end_time = perf_counter() + timeout - try: - result = self.run_cdp('Page.navigate', url=to_url, _timeout=timeout) - if 'errorText' in result: - err = ConnectionError(result['errorText']) - except TimeoutError: - err = TimeoutError('页面连接超时。') - - if err: - sleep(interval) - if self._debug or show_errmsg: - print(f'重试{t + 1} {to_url}') - self.stop_loading() - continue - - if self.page_load_strategy == 'none': - return True - - yu = end_time - perf_counter() - ok = self._wait_loaded(1 if yu <= 0 else yu) - if not ok: - err = TimeoutError('页面连接超时。') - sleep(interval) - if self._debug or show_errmsg: - print(f'重试{t + 1} {to_url}') - self.stop_loading() - continue - - if not err: - break - - if err: - if show_errmsg: - raise err if err is not None else ConnectionError('连接异常。') - return False - - self._check_ok() - return True + return self.doc_ele._ele(loc_or_ele, timeout, + raise_err=raise_err) if single else self.doc_ele.eles(loc_or_ele, timeout) def _is_inner_frame(self): """返回当前frame是否同域""" - return self.frame_id in str(self._target_page.run_cdp('Page.getFrameTree')['frameTree']) - - def _check_alive(self): - """检测iframe是否有效线程方法""" - while self.is_alive: - sleep(1) - self.driver.stop() + return self._frame_id in str(self._target_page.run_cdp('Page.getFrameTree')['frameTree']) diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 3e63295..6159831 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -21,12 +21,11 @@ from .._units.waiter import FrameWaiter class ChromiumFrame(ChromiumBase): - def __init__(self, page: ChromiumBase, ele: ChromiumElement): + def __init__(self, page: Union[ChromiumPage, WebPage, ChromiumTab, ChromiumFrame], ele: ChromiumElement): self._page: ChromiumPage = ... self._target_page: ChromiumBase = ... self.tab: ChromiumTab = ... self._tab_id: str = ... - self.frame_id: str = ... self._frame_ele: ChromiumElement = ... self._backend_id: str = ... self._doc_ele: ChromiumElement = ... @@ -49,7 +48,7 @@ class ChromiumFrame(ChromiumBase): def _reload(self) -> None: ... - def _check_ok(self) -> None: ... + def _get_document(self) -> None: ... def _get_new_document(self) -> None: ... @@ -193,11 +192,4 @@ class ChromiumFrame(ChromiumBase): timeout: float = None, single: bool = True, relative: bool = False, raise_err: bool = None) \ -> Union[ChromiumElement, ChromiumFrame, None, List[Union[ChromiumElement, ChromiumFrame]]]: ... - def _d_connect(self, - to_url: str, - times: int = 0, - interval: float = 1, - show_errmsg: bool = False, - timeout: float = None) -> Union[bool, None]: ... - def _is_inner_frame(self) -> bool: ... diff --git a/setup.py b/setup.py index 780d1d4..39f53d1 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b5", + version="4.0.0b6", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From d24730cc2d0133ff46d6f72812443ab3d79fd875 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 12 Nov 2023 00:24:51 +0800 Subject: [PATCH 076/182] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=8A=93=E5=8C=85?= =?UTF-8?q?=E9=80=BB=E8=BE=91=EF=BC=8C=E4=BF=9D=E8=AF=81=E9=A2=9D=E5=A4=96?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E8=83=BD=E8=8E=B7=E5=8F=96=E5=88=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/chromium_driver.py | 1 - DrissionPage/_units/network_listener.py | 118 ++++++++++++++++++----- DrissionPage/_units/network_listener.pyi | 25 ++++- 3 files changed, 115 insertions(+), 29 deletions(-) diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index c47f92f..b2a578b 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -97,7 +97,6 @@ class ChromiumDriver(object): def _recv_loop(self): """接收浏览器信息的守护线程方法""" while not self._stopped.is_set(): - print('收') try: # self._ws.settimeout(1) msg_json = self._ws.recv() diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index 2b2214a..2ec8bc7 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -190,6 +190,7 @@ class NetworkListener(object): def _requestWillBeSent(self, **kwargs): """接收到请求时的回调函数""" + p = None if not self._targets: if not self._method or kwargs['request']['method'] in self._method: rid = kwargs['requestId'] @@ -197,7 +198,6 @@ class NetworkListener(object): p._raw_request = kwargs if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): p._raw_post_data = self._driver.call_method('Network.getRequestPostData', requestId=rid)['postData'] - return else: rid = kwargs['requestId'] @@ -208,11 +208,14 @@ class NetworkListener(object): p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, target)) p._raw_request = kwargs if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): - p._raw_post_data = self._driver.call_method('Network.getRequestPostData', requestId=rid)[ - 'postData'] + p._raw_post_data = self._driver.call_method('Network.getRequestPostData', + requestId=rid)['postData'] break + self._extra_info_ids.setdefault(kwargs['requestId'], {})['obj'] = p if p else False + def _requestWillBeSentExtraInfo(self, **kwargs): + """接收到请求额外信息时的回调函数""" self._extra_info_ids.setdefault(kwargs['requestId'], {})['request'] = kwargs def _response_received(self, **kwargs): @@ -223,9 +226,18 @@ class NetworkListener(object): request._resource_type = kwargs['type'] def _responseReceivedExtraInfo(self, **kwargs): - r = self._extra_info_ids.get(kwargs['requestId']) + """接收到返回额外信息时的回调函数""" + r = self._extra_info_ids.get(kwargs['requestId'], None) if r: - r['response'] = kwargs + obj = r.get('obj', None) + if obj is False: + self._extra_info_ids.pop(kwargs['requestId'], None) + elif isinstance(obj, DataPacket): + obj._requestExtraInfo = r['request'] + obj._responseExtraInfo = kwargs + self._extra_info_ids.pop(kwargs['requestId'], None) + else: + r['response'] = kwargs def _loading_finished(self, **kwargs): """请求完成时处理方法""" @@ -240,15 +252,21 @@ class NetworkListener(object): dp._raw_body = '' dp._base64_body = False - ei = self._extra_info_ids.get(r_id, None) - if ei: - dp._requestExtraInfo = ei.get('request', None) - dp._responseExtraInfo = ei.get('response', None) - - self._caught.put(dp) + r = self._extra_info_ids.get(kwargs['requestId'], None) + if r: + obj = r.get('obj', None) + if obj is False or (isinstance(obj, DataPacket) and not self._extra_info_ids.get('request')): + self._extra_info_ids.pop(kwargs['requestId'], None) + elif isinstance(obj, DataPacket) and self._extra_info_ids.get('response'): + response = r.get('response') + obj._requestExtraInfo = r['request'] + obj._responseExtraInfo = response + self._extra_info_ids.pop(kwargs['requestId'], None) self._request_ids.pop(r_id, None) - self._extra_info_ids.pop(r_id, None) + + if dp: + self._caught.put(dp) def _loading_failed(self, **kwargs): """请求失败时的回调方法""" @@ -257,14 +275,23 @@ class NetworkListener(object): if dp: dp.errorText = kwargs['errorText'] dp._resource_type = kwargs['type'] - ei = self._extra_info_ids.get(r_id, None) - if ei: - dp._requestExtraInfo = ei.get('request', None) - dp._responseExtraInfo = ei.get('response', None) - self._caught.put(dp) + + r = self._extra_info_ids.get(kwargs['requestId'], None) + if r: + obj = r.get('obj', None) + if obj is False and r.get('response'): + self._extra_info_ids.pop(kwargs['requestId'], None) + elif isinstance(obj, DataPacket): + response = r.get('response') + if response: + obj._requestExtraInfo = r['request'] + obj._responseExtraInfo = response + self._extra_info_ids.pop(kwargs['requestId'], None) self._request_ids.pop(r_id, None) - self._extra_info_ids.pop(r_id, None) + + if dp: + self._caught.put(dp) class DataPacket(object): @@ -296,6 +323,14 @@ class DataPacket(object): t = f'"{self.target}"' if self.target is not None else None return f'' + @property + def _request_extra_info(self): + return self._requestExtraInfo + + @property + def _response_extra_info(self): + return self._responseExtraInfo + @property def url(self): return self.request.url @@ -315,23 +350,45 @@ class DataPacket(object): @property def request(self): if self._request is None: - self._request = Request(self._raw_request['request'], self._raw_post_data, self._requestExtraInfo) + self._request = Request(self, self._raw_request['request'], self._raw_post_data) return self._request @property def response(self): if self._response is None: - self._response = Response(self._raw_response, self._raw_body, self._base64_body, self._responseExtraInfo) + self._response = Response(self, self._raw_response, self._raw_body, self._base64_body) return self._response + def wait_extra_info(self, timeout=None): + """等待额外的信息加载完成 + :param timeout: 超时时间，None为无限等待 + :return: 是否等待成功 + """ + if self._raw_request['redirectHasExtraInfo'] is False: + return True + + if timeout is None: + while self._responseExtraInfo is None: + sleep(.1) + return True + + else: + end_time = perf_counter() + timeout + while perf_counter() < end_time: + if self._responseExtraInfo is not None: + return True + sleep(.1) + else: + return False + class Request(object): - def __init__(self, raw_request, post_data, extra_info): + def __init__(self, data_packet, raw_request, post_data): + self._data_packet = data_packet self._request = raw_request self._raw_post_data = post_data self._postData = None self._headers = None - self.extra_info = RequestExtraInfo(extra_info or {}) def __getattr__(self, item): return self._request.get(item, None) @@ -359,15 +416,19 @@ class Request(object): self._postData = postData return self._postData + @property + def extra_info(self): + return RequestExtraInfo(self._data_packet._request_extra_info or {}) + class Response(object): - def __init__(self, raw_response, raw_body, base64_body, extra_info): + def __init__(self, data_packet, raw_response, raw_body, base64_body): + self._data_packet = data_packet self._response = raw_response self._raw_body = raw_body self._is_base64_body = base64_body self._body = None self._headers = None - self.extra_info = ResponseExtraInfo(extra_info or {}) def __getattr__(self, item): return self._response.get(item, None) @@ -399,11 +460,20 @@ class Response(object): return self._body + @property + def extra_info(self): + return ResponseExtraInfo(self._data_packet._response_extra_info or {}) + class ExtraInfo(object): def __init__(self, extra_info): self._extra_info = extra_info + @property + def all_info(self): + """以dict形式返回所有额外信息""" + return self._extra_info + def __getattr__(self, item): return self._extra_info.get(item, None) diff --git a/DrissionPage/_units/network_listener.pyi b/DrissionPage/_units/network_listener.pyi index e261483..081d4e3 100644 --- a/DrissionPage/_units/network_listener.pyi +++ b/DrissionPage/_units/network_listener.pyi @@ -84,6 +84,12 @@ class DataPacket(object): self._requestExtraInfo: Optional[dict] = ... self._responseExtraInfo: Optional[dict] = ... + @property + def _request_extra_info(self) -> Optional[dict]: ... + + @property + def _response_extra_info(self) -> Optional[dict]: ... + @property def url(self) -> str: ... @@ -102,10 +108,11 @@ class DataPacket(object): @property def response(self) -> Response: ... + def wait_extra_info(self, timeout: float = None) -> bool: ... + class Request(object): url: str = ... - extra_info: Optional[RequestExtraInfo] = ... _headers: Union[CaseInsensitiveDict, None] = ... method: str = ... @@ -119,7 +126,8 @@ class Request(object): trustTokenParams = ... isSameSite = ... - def __init__(self, raw_request: dict, post_data: str, extra_info: Optional[dict]): + def __init__(self, data_packet: DataPacket, raw_request: dict, post_data: str): + self._data_packet: DataPacket = ... self._request: dict = ... self._raw_post_data: str = ... self._postData: str = ... @@ -130,9 +138,11 @@ class Request(object): @property def postData(self) -> Union[str, dict]: ... + @property + def extra_info(self) -> Optional[RequestExtraInfo]: ... + class Response(object): - extra_info: Optional[ResponseExtraInfo] = ... url = ... status = ... statusText = ... @@ -157,13 +167,17 @@ class Response(object): securityState = ... securityDetails = ... - def __init__(self, raw_response: dict, raw_body: str, base64_body: bool, extra_info: Optional[dict]): + def __init__(self, data_packet: DataPacket, raw_response: dict, raw_body: str, base64_body: bool): + self._data_packet: DataPacket = ... self._response: dict = ... self._raw_body: str = ... self._is_base64_body: bool = ... self._body: Union[str, dict] = ... self._headers: dict = ... + @property + def extra_info(self) -> Optional[ResponseExtraInfo]: ... + @property def headers(self) -> CaseInsensitiveDict: ... @@ -178,6 +192,9 @@ class ExtraInfo(object): def __init__(self, extra_info: dict): self._extra_info: dict = ... + @property + def all_info(self) -> dict: ... + class RequestExtraInfo(ExtraInfo): requestId: str = ... From 89e1b3a29fd0f176d9869d3a753c9db0c1f8a4e0 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 12 Nov 2023 01:03:39 +0800 Subject: [PATCH 077/182] =?UTF-8?q?4.0.0b6=20quit()=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E8=B6=85=E6=97=B6=E5=92=8C=E5=BC=BA=E5=88=B6=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 22 +++++++++++++++----- DrissionPage/_base/browser.pyi | 4 ++-- DrissionPage/_commons/tools.py | 22 ++++++++++++++++++++ DrissionPage/_commons/tools.pyi | 3 +++ DrissionPage/_pages/chromium_page.py | 30 +++++++++++++++++++-------- DrissionPage/_pages/chromium_page.pyi | 6 ++++-- DrissionPage/_pages/web_page.py | 25 +++++++++++++++------- DrissionPage/_pages/web_page.pyi | 5 +++-- requirements.txt | 3 ++- setup.py | 3 ++- 10 files changed, 93 insertions(+), 30 deletions(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index dc4d4dd..c0e911b 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -3,9 +3,10 @@ @Author : g1879 @Contact : g1879@qq.com """ -from time import sleep +from time import sleep, perf_counter from .chromium_driver import BrowserDriver, ChromiumDriver +from .._commons.tools import stop_process_on_port from .._units.download_manager import DownloadManager @@ -151,8 +152,12 @@ class Browser(object): """ return self.run_cdp('Browser.getWindowForTarget', targetId=tab_id or self.id)['bounds'] - def quit(self): - """关闭浏览器""" + def quit(self, timeout=5, force=True): + """关闭浏览器 + :param timeout: 等待浏览器关闭超时时间 + :param force: 关闭超时是否强制终止进程 + :return: None + """ self.run_cdp('Browser.close') self.driver.stop() @@ -161,11 +166,18 @@ class Browser(object): from platform import system txt = f'tasklist | findstr {self.process_id}' if system().lower() == 'windows' \ else f'ps -ef | grep {self.process_id}' - while True: + end_time = perf_counter() + timeout + while perf_counter() < end_time: p = popen(txt) if f' {self.process_id} ' not in p.read(): - break + return sleep(.2) + if force: + ip, port = self.address.split(':') + if ip not in ('127.0.0.1', 'localhost'): + return + stop_process_on_port(port) + def _on_quit(self): Browser.BROWSERS.pop(self.id, None) diff --git a/DrissionPage/_base/browser.pyi b/DrissionPage/_base/browser.pyi index a7a69c5..190662f 100644 --- a/DrissionPage/_base/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -26,7 +26,7 @@ class Browser(object): def __init__(self, address: str, browser_id: str, page: ChromiumPage): ... - def _get_driver(self, tab_id: str)->ChromiumDriver: ... + def _get_driver(self, tab_id: str) -> ChromiumDriver: ... def run_cdp(self, cmd, **cmd_args) -> dict: ... @@ -57,6 +57,6 @@ class Browser(object): def _onTargetDestroyed(self, **kwargs) -> None: ... - def quit(self) -> None: ... + def quit(self, timeout: float = 5, force: bool = True) -> None: ... def _on_quit(self) -> None: ... diff --git a/DrissionPage/_commons/tools.py b/DrissionPage/_commons/tools.py index 7f48033..5e7dd80 100644 --- a/DrissionPage/_commons/tools.py +++ b/DrissionPage/_commons/tools.py @@ -9,6 +9,8 @@ from re import search, sub from shutil import rmtree from time import perf_counter, sleep +from psutil import process_iter, AccessDenied, NoSuchProcess, ZombieProcess + def get_usable_path(path, is_file=True, parents=True): """检查文件或文件夹是否有重名，并返回可以使用的路径 @@ -216,3 +218,23 @@ def wait_until(page, condition, timeout=10, poll=0.1, raise_err=True): raise TimeoutError('等待超时') else: return False + + +def stop_process_on_port(port): + """强制关闭某个端口内的进程 + :param port: 端口号 + :return: None + """ + for proc in process_iter(['pid', 'connections']): + try: + connections = proc.connections() + except AccessDenied: + continue + for conn in connections: + if conn.laddr.port == int(port): + try: + proc.terminate() + except (NoSuchProcess, AccessDenied, ZombieProcess): + pass + except Exception as e: + print(f"{proc.pid} {port}: {e}") diff --git a/DrissionPage/_commons/tools.pyi b/DrissionPage/_commons/tools.pyi index b430ba9..d4294fb 100644 --- a/DrissionPage/_commons/tools.pyi +++ b/DrissionPage/_commons/tools.pyi @@ -36,3 +36,6 @@ def get_chrome_hwnds_from_pid(pid: Union[str, int], title: str) -> list: ... def wait_until(page, condition: Union[FunctionType, str, tuple], timeout: float, poll: float, raise_err: bool): ... + + +def stop_process_on_port(port: Union[int, str]) -> None: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 2732bbe..4e60c52 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -157,9 +157,21 @@ class ChromiumPage(ChromiumBase): :param new_context: 是否创建新的上下文 :return: 新标签页对象 """ + tab = ChromiumTab(self, tab_id=self._new_tab(new_window, background, new_context)) + if url: + tab.get(url) + return tab + + def _new_tab(self, new_window=False, background=False, new_context=False): + """新建一个标签页 + :param new_window: 是否在新窗口打开标签页 + :param background: 是否不激活新标签页，如new_window为True则无效 + :param new_context: 是否创建新的上下文 + :return: 新标签页对象 + """ bid = None if new_context: - bid = self.browser.run_cdp('Target.createBrowserContext', **kwargs)['browserContextId'] + bid = self.browser.run_cdp('Target.createBrowserContext')['browserContextId'] kwargs = {'url': ''} if new_window: @@ -169,11 +181,7 @@ class ChromiumPage(ChromiumBase): if bid: kwargs['browserContextId'] = bid - tid = self.run_cdp('Target.createTarget', **kwargs)['targetId'] - tab = ChromiumTab(self, tab_id=tid) - if url: - tab.get(url) - return tab + return self.run_cdp('Target.createTarget', **kwargs)['targetId'] def close_tabs(self, tabs_or_ids=None, others=False): """关闭传入的标签页，默认关闭当前页。可传入多个 @@ -214,9 +222,13 @@ class ChromiumPage(ChromiumBase): """ self.close_tabs(tabs_or_ids, True) - def quit(self): - """关闭浏览器""" - self.browser.quit() + def quit(self, timeout=5, force=True): + """关闭浏览器 + :param timeout: 等待浏览器关闭超时时间 + :param force: 关闭超时是否强制终止进程 + :return: None + """ + self.browser.quit(timeout, force) def get_rename(original, rename): diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index 9dc2102..a6c7603 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -63,7 +63,9 @@ class ChromiumPage(ChromiumBase): tab_type: Union[str, list, tuple] = None, single: bool = True) -> Union[str, List[str]]: ... def new_tab(self, url: str = None, new_window: bool = False, background: bool = False, - new_context: bool = False) -> Union[ChromiumTab, ChromiumPage]: ... + new_context: bool = False) -> ChromiumTab: ... + + def _new_tab(self, new_window: bool = False, background: bool = False, new_context: bool = False) -> str: ... def close_tabs(self, tabs_or_ids: Union[str, ChromiumTab, List[Union[str, ChromiumTab]], Tuple[Union[str, ChromiumTab]]] = None, others: bool = False) -> None: ... @@ -71,7 +73,7 @@ class ChromiumPage(ChromiumBase): def close_other_tabs(self, tabs_or_ids: Union[ str, ChromiumTab, List[Union[str, ChromiumTab]], Tuple[Union[str, ChromiumTab]]] = None) -> None: ... - def quit(self) -> None: ... + def quit(self, timeout: float = 5, force: bool = True) -> None: ... def get_rename(original: str, rename: str) -> str: ... diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 698193d..69af552 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -296,13 +296,18 @@ class WebPage(SessionPage, ChromiumPage, BasePage): """ return tab_id if isinstance(tab_id, WebPageTab) else WebPageTab(self, tab_id or self.tab_id) - def new_tab(self, url=None, switch_to=False): - """新建一个标签页,该标签页在最后面 + def new_tab(self, url=None, new_window=False, background=False, new_context=False): + """新建一个标签页 :param url: 新标签页跳转到的网址 - :param switch_to: 新建标签页后是否把焦点移过去 - :return: switch_to为False时返回新标签页对象，否则返回当前对象， + :param new_window: 是否在新窗口打开标签页 + :param background: 是否不激活新标签页，如new_window为True则无效 + :param new_context: 是否创建新的上下文 + :return: 新标签页对象 """ - return self if switch_to else WebPageTab(self, self._new_tab(url, switch_to)) + tab = WebPageTab(self, tab_id=self._new_tab(new_window, background, new_context)) + if url: + tab.get(url) + return tab def close_driver(self): """关闭driver及浏览器""" @@ -340,14 +345,18 @@ class WebPage(SessionPage, ChromiumPage, BasePage): return super(SessionPage, self)._find_elements(loc_or_ele, timeout=timeout, single=single, relative=relative) - def quit(self): - """关闭浏览器，关闭session""" + def quit(self, timeout=5, force=True): + """关闭浏览器和Session + :param timeout: 等待浏览器关闭超时时间 + :param force: 关闭超时是否强制终止进程 + :return: None + """ if self._has_session: self._session.close() self._session = None self._response = None self._has_session = None if self._has_driver: - super(SessionPage, self).quit() + super(SessionPage, self).quit(timeout, force) self._driver = None self._has_driver = None diff --git a/DrissionPage/_pages/web_page.pyi b/DrissionPage/_pages/web_page.pyi index 6509692..60159d4 100644 --- a/DrissionPage/_pages/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -121,7 +121,8 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def get_tab(self, tab_id: Union[str, WebPageTab] = None) -> WebPageTab: ... - def new_tab(self, url: str = None, switch_to: bool = False) -> Union[WebPageTab, WebPage]: ... + def new_tab(self, url: str = None, new_window: bool = False, background: bool = False, + new_context: bool = False) -> WebPageTab: ... def close_driver(self) -> None: ... @@ -159,6 +160,6 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def _set_start_options(self, dr_opt: Union[ChromiumDriver, bool, None], se_opt: Union[Session, SessionOptions, bool, None]) -> None: ... - def quit(self) -> None: ... + def quit(self, timeout: float = 5, force: bool = True) -> None: ... def _on_download_begin(self, **kwargs): ... diff --git a/requirements.txt b/requirements.txt index 4e712ad..ba1160f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ DownloadKit>=1.0.0 FlowViewer>=0.3.0 websocket-client click -tldextract \ No newline at end of file +tldextract +psutil \ No newline at end of file diff --git a/setup.py b/setup.py index 39f53d1..e164891 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,8 @@ setup( 'FlowViewer>=0.3.0', 'websocket-client', 'click', - 'tldextract' + 'tldextract', + 'psutil' ], classifiers=[ "Programming Language :: Python :: 3.6", From 27bea5f57929df87dcf2c723b87eb91677921702 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 12 Nov 2023 22:56:52 +0800 Subject: [PATCH 078/182] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E6=B5=8F=E8=A7=88=E5=99=A8=E5=90=AF=E5=8A=A8=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E9=9A=90=E7=A7=81=E5=A3=B0=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 9 ++-- DrissionPage/_base/chromium_driver.py | 12 ++--- DrissionPage/_base/chromium_driver.pyi | 4 +- DrissionPage/_pages/chromium_base.py | 70 +++++++++++++++++++------ DrissionPage/_pages/chromium_frame.py | 2 +- DrissionPage/_pages/web_page.py | 2 +- DrissionPage/_units/action_chains.py | 18 +++---- DrissionPage/_units/download_manager.py | 4 +- DrissionPage/_units/network_listener.py | 33 ++++++------ DrissionPage/_units/screencast.py | 4 +- DrissionPage/_units/setter.py | 8 +-- 11 files changed, 97 insertions(+), 69 deletions(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index c0e911b..05d12c9 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -51,8 +51,8 @@ class Browser(object): break self.run_cdp('Target.setDiscoverTargets', discover=True) - self._driver.set_listener('Target.targetDestroyed', self._onTargetDestroyed) - self._driver.set_listener('Target.targetCreated', self._onTargetCreated) + self._driver.set_callback('Target.targetDestroyed', self._onTargetDestroyed) + self._driver.set_callback('Target.targetCreated', self._onTargetCreated) def _get_driver(self, tab_id): """获取对应tab id的ChromiumDriver @@ -70,7 +70,8 @@ class Browser(object): def _onTargetDestroyed(self, **kwargs): """标签页关闭时执行""" tab_id = kwargs['targetId'] - self._dl_mgr.clear_tab_info(tab_id) + if hasattr(self, '_dl_mgr'): + self._dl_mgr.clear_tab_info(tab_id) for key in [k for k, i in self._frames.items() if i == tab_id]: self._frames.pop(key, None) self._drivers.pop(tab_id, None) @@ -87,7 +88,7 @@ class Browser(object): :param cmd_args: 参数 :return: 执行的结果 """ - return self._driver.call_method(cmd, **cmd_args) + return self._driver.run(cmd, **cmd_args) @property def driver(self): diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index b2a578b..26284e3 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -139,15 +139,15 @@ class ChromiumDriver(object): function = self.event_handlers.get(event['method']) if function: - if self._debug: - print(f'开始执行 {function.__name__}') + # if self._debug: + # print(f'开始执行 {function.__name__}') function(**event['params']) - if self._debug: - print(f'执行 {function.__name__}完毕') + # if self._debug: + # print(f'执行 {function.__name__}完毕') self.event_queue.task_done() - def call_method(self, _method, **kwargs): + def run(self, _method, **kwargs): """执行cdp方法 :param _method: cdp方法名 :param args: cdp参数 @@ -205,7 +205,7 @@ class ChromiumDriver(object): self.event_queue.queue.clear() return True - def set_listener(self, event, callback): + def set_callback(self, event, callback): """绑定cdp event和回调方法 :param event: cdp event :param callback: 绑定到cdp event的回调方法 diff --git a/DrissionPage/_base/chromium_driver.pyi b/DrissionPage/_base/chromium_driver.pyi index 8977874..28d8692 100644 --- a/DrissionPage/_base/chromium_driver.pyi +++ b/DrissionPage/_base/chromium_driver.pyi @@ -47,13 +47,13 @@ class ChromiumDriver(object): def __getattr__(self, item: str) -> Callable: ... - def call_method(self, _method: str, **kwargs) -> dict: ... + def run(self, _method: str, **kwargs) -> dict: ... def start(self) -> bool: ... def stop(self) -> bool: ... - def set_listener(self, event: str, callback: Union[Callable, None]) -> None: ... + def set_callback(self, event: str, callback: Union[Callable, None]) -> None: ... def __str__(self) -> str: ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index d09eaae..d3ec6f7 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -100,13 +100,17 @@ class ChromiumBase(BasePage): """ self._is_loading = True self._driver = self.browser._get_driver(tab_id) - self._alert = Alert() - self._driver.set_listener('Page.javascriptDialogOpening', self._on_alert_open) - self._driver.set_listener('Page.javascriptDialogClosed', self._on_alert_close) + if self._driver.run('Target.getTargetInfo', + targetId=self._target_id)['targetInfo']['url'] == 'chrome://privacy-sandbox-dialog/notice': + self._driver = close_privacy_dialog(self) - self._driver.call_method('DOM.enable') - self._driver.call_method('Page.enable') - self._driver.call_method('Emulation.setFocusEmulationEnabled', enabled=True) + self._alert = Alert() + self._driver.set_callback('Page.javascriptDialogOpening', self._on_alert_open) + self._driver.set_callback('Page.javascriptDialogClosed', self._on_alert_close) + + self._driver.run('DOM.enable') + self._driver.run('Page.enable') + self._driver.run('Emulation.setFocusEmulationEnabled', enabled=True) r = self.run_cdp('Page.getFrameTree') for i in findall(r"'id': '(.*?)'", str(r)): @@ -114,13 +118,13 @@ class ChromiumBase(BasePage): if not hasattr(self, '_frame_id'): self._frame_id = r['frameTree']['frame']['id'] - self._driver.set_listener('Page.frameStartedLoading', self._onFrameStartedLoading) - self._driver.set_listener('Page.frameNavigated', self._onFrameNavigated) - self._driver.set_listener('Page.domContentEventFired', self._onDomContentEventFired) - self._driver.set_listener('Page.loadEventFired', self._onLoadEventFired) - self._driver.set_listener('Page.frameStoppedLoading', self._onFrameStoppedLoading) - self._driver.set_listener('Page.frameAttached', self._onFrameAttached) - self._driver.set_listener('Page.frameDetached', self._onFrameDetached) + self._driver.set_callback('Page.frameStartedLoading', self._onFrameStartedLoading) + self._driver.set_callback('Page.frameNavigated', self._onFrameNavigated) + self._driver.set_callback('Page.domContentEventFired', self._onDomContentEventFired) + self._driver.set_callback('Page.loadEventFired', self._onLoadEventFired) + self._driver.set_callback('Page.frameStoppedLoading', self._onFrameStoppedLoading) + self._driver.set_callback('Page.frameAttached', self._onFrameAttached) + self._driver.set_callback('Page.frameDetached', self._onFrameDetached) def _get_document(self): if self._is_reading: @@ -231,7 +235,7 @@ class ChromiumBase(BasePage): files = self._upload_list if kwargs['mode'] == 'selectMultiple' else self._upload_list[:1] self.run_cdp('DOM.setFileInputFiles', files=files, backendNodeId=kwargs['backendNodeId']) - self.driver.set_listener('Page.fileChooserOpened', None) + self.driver.set_callback('Page.fileChooserOpened', None) self.run_cdp('Page.setInterceptFileChooserDialog', enabled=False) self._upload_list = None @@ -416,7 +420,7 @@ class ChromiumBase(BasePage): :param cmd_args: 参数 :return: 执行的结果 """ - r = self.driver.call_method(cmd, **cmd_args) + r = self.driver.run(cmd, **cmd_args) if ERROR not in r: return r @@ -822,9 +826,9 @@ class ChromiumBase(BasePage): res_text = self._alert.text if self._alert.type == 'prompt': - self.driver.call_method('Page.handleJavaScriptDialog', accept=accept, promptText=send) + self.driver.run('Page.handleJavaScriptDialog', accept=accept, promptText=send) else: - self.driver.call_method('Page.handleJavaScriptDialog', accept=accept) + self.driver.run('Page.handleJavaScriptDialog', accept=accept) return res_text def _on_alert_close(self, **kwargs): @@ -1032,3 +1036,35 @@ class Alert(object): self.defaultPrompt = None self.response_accept = None self.response_text = None + + +def close_privacy_dialog(page): + """关闭隐私声明弹窗 + :param page: ChromiumBase对象 + :return: ChromiumDriver对象 + """ + tid = page.tab_id + page._driver.run('Runtime.enable') + page._driver.run('DOM.enable') + page._driver.run('DOM.getDocument') + sid = page._driver.run('DOM.performSearch', query='//*[name()="privacy-sandbox-notice-dialog-app"]', + includeUserAgentShadowDOM=True)['searchId'] + r = page._driver.run('DOM.getSearchResults', searchId=sid, fromIndex=0, toIndex=1)['nodeIds'][0] + while True: + try: + r = page._driver.run('DOM.describeNode', nodeId=r)['node']['shadowRoots'][0]['backendNodeId'] + break + except KeyError: + pass + page._driver.run('DOM.discardSearchResults', searchId=sid) + r = page._driver.run('DOM.resolveNode', backendNodeId=r)['object']['objectId'] + r = page._driver.run('Runtime.callFunctionOn', objectId=r, + functionDeclaration='function()' + '{return this.getElementById("ackButton");}')['result']['objectId'] + page._driver.run('Runtime.callFunctionOn', objectId=r, functionDeclaration='function(){return this.click();}') + while True: + new_tid = page.browser.tabs[0] + if new_tid != tid: + break + sleep(.1) + return page.browser._get_driver(new_tid) diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 6d7b4c6..dc8f5ec 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -90,7 +90,7 @@ class ChromiumFrame(ChromiumBase): except: self.browser.driver.get(f'http://{self.address}/json') super()._driver_init(tab_id) - self._driver.set_listener('Inspector.detached', self._onInspectorDetached) + self._driver.set_callback('Inspector.detached', self._onInspectorDetached) def _reload(self): """重新获取document""" diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 69af552..679745a 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -314,7 +314,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): if self._has_driver: self.change_mode('s') try: - self.driver.call_method('Browser.close') + self.driver.run('Browser.close') except Exception: pass self._driver.stop() diff --git a/DrissionPage/_units/action_chains.py b/DrissionPage/_units/action_chains.py index 0469bef..b5d9c2f 100644 --- a/DrissionPage/_units/action_chains.py +++ b/DrissionPage/_units/action_chains.py @@ -59,7 +59,7 @@ class ActionChains: cx = x + offset_x cy = y + offset_y - self._dr.call_method('Input.dispatchMouseEvent', type='mouseMoved', x=cx, y=cy, modifiers=self.modifier) + self._dr.run('Input.dispatchMouseEvent', type='mouseMoved', x=cx, y=cy, modifiers=self.modifier) self.curr_x = cx self.curr_y = cy return self @@ -72,8 +72,8 @@ class ActionChains: """ self.curr_x += offset_x self.curr_y += offset_y - self._dr.call_method('Input.dispatchMouseEvent', type='mouseMoved', x=self.curr_x, y=self.curr_y, - modifiers=self.modifier) + self._dr.run('Input.dispatchMouseEvent', type='mouseMoved', x=self.curr_x, y=self.curr_y, + modifiers=self.modifier) return self def click(self, on_ele=None): @@ -171,8 +171,8 @@ class ActionChains: """ if on_ele: self.move_to(on_ele) - self._dr.call_method('Input.dispatchMouseEvent', type='mousePressed', button=button, clickCount=count, - x=self.curr_x, y=self.curr_y, modifiers=self.modifier) + self._dr.run('Input.dispatchMouseEvent', type='mousePressed', button=button, clickCount=count, + x=self.curr_x, y=self.curr_y, modifiers=self.modifier) return self def _release(self, button): @@ -180,8 +180,8 @@ class ActionChains: :param button: 要释放的按键 :return: self """ - self._dr.call_method('Input.dispatchMouseEvent', type='mouseReleased', button=button, clickCount=1, - x=self.curr_x, y=self.curr_y, modifiers=self.modifier) + self._dr.run('Input.dispatchMouseEvent', type='mouseReleased', button=button, clickCount=1, + x=self.curr_x, y=self.curr_y, modifiers=self.modifier) return self def scroll(self, delta_x=0, delta_y=0, on_ele=None): @@ -193,8 +193,8 @@ class ActionChains: """ if on_ele: self.move_to(on_ele) - self._dr.call_method('Input.dispatchMouseEvent', type='mouseWheel', x=self.curr_x, y=self.curr_y, - deltaX=delta_x, deltaY=delta_y, modifiers=self.modifier) + self._dr.run('Input.dispatchMouseEvent', type='mouseWheel', x=self.curr_x, y=self.curr_y, + deltaX=delta_x, deltaY=delta_y, modifiers=self.modifier) return self def up(self, pixel): diff --git a/DrissionPage/_units/download_manager.py b/DrissionPage/_units/download_manager.py index bb04860..6a6a9c4 100644 --- a/DrissionPage/_units/download_manager.py +++ b/DrissionPage/_units/download_manager.py @@ -27,8 +27,8 @@ class DownloadManager(object): self._tab_missions = {} # {tab_id: DownloadMission} self._flags = {} # {tab_id: [bool, DownloadMission]} - self._browser.driver.set_listener('Browser.downloadProgress', self._onDownloadProgress) - self._browser.driver.set_listener('Browser.downloadWillBegin', self._onDownloadWillBegin) + self._browser.driver.set_callback('Browser.downloadProgress', self._onDownloadProgress) + self._browser.driver.set_callback('Browser.downloadWillBegin', self._onDownloadWillBegin) self._browser.run_cdp('Browser.setDownloadBehavior', downloadPath=self._page.download_path, behavior='allowAndName', eventsEnabled=True) diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index 2ec8bc7..8b5d2cf 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -78,7 +78,7 @@ class NetworkListener(object): return self._driver = ChromiumDriver(self._page.tab_id, 'page', self._page.address) - self._driver.call_method('Network.enable') + self._driver.run('Network.enable') self.listening = True self._request_ids = {} @@ -158,10 +158,10 @@ class NetworkListener(object): :return: None """ if self.listening: - self._driver.set_listener('Network.requestWillBeSent', None) - self._driver.set_listener('Network.responseReceived', None) - self._driver.set_listener('Network.loadingFinished', None) - self._driver.set_listener('Network.loadingFailed', None) + self._driver.set_callback('Network.requestWillBeSent', None) + self._driver.set_callback('Network.responseReceived', None) + self._driver.set_callback('Network.loadingFinished', None) + self._driver.set_callback('Network.loadingFailed', None) self.listening = False if clear: self.clear() @@ -181,12 +181,12 @@ class NetworkListener(object): def _set_callback(self): """设置监听请求的回调函数""" - self._driver.set_listener('Network.requestWillBeSent', self._requestWillBeSent) - self._driver.set_listener('Network.requestWillBeSentExtraInfo', self._requestWillBeSentExtraInfo) - self._driver.set_listener('Network.responseReceived', self._response_received) - self._driver.set_listener('Network.responseReceivedExtraInfo', self._responseReceivedExtraInfo) - self._driver.set_listener('Network.loadingFinished', self._loading_finished) - self._driver.set_listener('Network.loadingFailed', self._loading_failed) + self._driver.set_callback('Network.requestWillBeSent', self._requestWillBeSent) + self._driver.set_callback('Network.requestWillBeSentExtraInfo', self._requestWillBeSentExtraInfo) + self._driver.set_callback('Network.responseReceived', self._response_received) + self._driver.set_callback('Network.responseReceivedExtraInfo', self._responseReceivedExtraInfo) + self._driver.set_callback('Network.loadingFinished', self._loading_finished) + self._driver.set_callback('Network.loadingFailed', self._loading_failed) def _requestWillBeSent(self, **kwargs): """接收到请求时的回调函数""" @@ -197,7 +197,7 @@ class NetworkListener(object): p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, None)) p._raw_request = kwargs if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): - p._raw_post_data = self._driver.call_method('Network.getRequestPostData', requestId=rid)['postData'] + p._raw_post_data = self._driver.run('Network.getRequestPostData', requestId=rid)['postData'] else: rid = kwargs['requestId'] @@ -208,8 +208,8 @@ class NetworkListener(object): p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, target)) p._raw_request = kwargs if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): - p._raw_post_data = self._driver.call_method('Network.getRequestPostData', - requestId=rid)['postData'] + p._raw_post_data = self._driver.run('Network.getRequestPostData', + requestId=rid)['postData'] break self._extra_info_ids.setdefault(kwargs['requestId'], {})['obj'] = p if p else False @@ -244,7 +244,7 @@ class NetworkListener(object): r_id = kwargs['requestId'] dp = self._request_ids.get(r_id) if dp: - r = self._driver.call_method('Network.getResponseBody', requestId=r_id) + r = self._driver.run('Network.getResponseBody', requestId=r_id) if 'body' in r: dp._raw_body = r['body'] dp._base64_body = r['base64Encoded'] @@ -364,9 +364,6 @@ class DataPacket(object): :param timeout: 超时时间，None为无限等待 :return: 是否等待成功 """ - if self._raw_request['redirectHasExtraInfo'] is False: - return True - if timeout is None: while self._responseExtraInfo is None: sleep(.1) diff --git a/DrissionPage/_units/screencast.py b/DrissionPage/_units/screencast.py index 1d25683..592fc89 100644 --- a/DrissionPage/_units/screencast.py +++ b/DrissionPage/_units/screencast.py @@ -35,7 +35,7 @@ class Screencast(object): raise ValueError('save_path必须设置。') clean_folder(self._path) if self._mode.startswith('frugal'): - self._page.driver.set_listener('Page.screencastFrame', self._onScreencastFrame) + self._page.driver.set_callback('Page.screencastFrame', self._onScreencastFrame) self._page.run_cdp('Page.startScreencast', everyNthFrame=1, quality=100) elif not self._mode.startswith('js'): @@ -92,7 +92,7 @@ class Screencast(object): return path if self._mode.startswith('frugal'): - self._page.driver.set_listener('Page.screencastFrame', None) + self._page.driver.set_callback('Page.screencastFrame', None) self._page.run_cdp('Page.stopScreencast') else: self._enable = False diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index d8bf2af..78b20c3 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -102,7 +102,7 @@ class ChromiumBaseSetter(object): :return: None """ if not self._page._upload_list: - self._page.driver.set_listener('Page.fileChooserOpened', self._page._onFileChooserOpened) + self._page.driver.set_callback('Page.fileChooserOpened', self._page._onFileChooserOpened) self._page.run_cdp('Page.setInterceptFileChooserDialog', enabled=True) if isinstance(files, str): @@ -164,12 +164,6 @@ class TabSetter(ChromiumBaseSetter): class ChromiumPageSetter(TabSetter): - def main_tab(self, tab_id=None): - """设置主tab - :param tab_id: 标签页id，不传入则设置当前tab - :return: None - """ - self._page._main_tab = tab_id or self._page.tab_id def tab_to_front(self, tab_or_id=None): """激活标签页使其处于最前面 From 9fb0f84507883976f11ee93f10b468234ee590b6 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 13 Nov 2023 18:20:06 +0800 Subject: [PATCH 079/182] =?UTF-8?q?click()=E5=A2=9E=E5=8A=A0wait=5Fstop?= =?UTF-8?q?=E5=8F=82=E6=95=B0=EF=BC=9B=E5=90=AF=E5=8A=A8=E6=97=B6=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E5=85=B3=E9=97=AD=E6=B5=8F=E8=A7=88=E5=99=A8=E9=9A=90?= =?UTF-8?q?=E7=A7=81=E5=A3=B0=E6=98=8E=EF=BC=9B=E5=8A=A8=E4=BD=9C=E9=93=BE?= =?UTF-8?q?=E7=A7=BB=E5=8A=A8=E5=A2=9E=E5=8A=A0=E6=97=B6=E9=95=BF=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 34 ++---------- DrissionPage/_pages/chromium_base.py | 63 +++++++++++----------- DrissionPage/_pages/chromium_frame.pyi | 1 + DrissionPage/_units/action_chains.py | 47 ++++++++++------ DrissionPage/_units/action_chains.pyi | 4 +- DrissionPage/_units/clicker.py | 15 +++--- DrissionPage/_units/clicker.pyi | 4 +- DrissionPage/_units/select_element.py | 2 +- setup.py | 2 +- 9 files changed, 84 insertions(+), 88 deletions(-) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index c57eb52..2d9d072 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -666,41 +666,15 @@ class ChromiumElement(DrissionElement): def drag_to(self, ele_or_loc, duration=.5): """拖拽当前元素，目标为另一个元素或坐标元组(x, y) :param ele_or_loc: 另一个元素或坐标元组，坐标为元素中点的坐标 - :param duration: 拖动用时，传入0即瞬间到j达 + :param duration: 拖动用时，传入0即瞬间到达 :return: None """ - # x, y：目标点坐标 if isinstance(ele_or_loc, ChromiumElement): - target_x, target_y = ele_or_loc.locations.midpoint - elif isinstance(ele_or_loc, (list, tuple)): - target_x, target_y = ele_or_loc - else: + ele_or_loc = ele_or_loc.locations.midpoint + elif not isinstance(ele_or_loc, (list, tuple)): raise TypeError('需要ChromiumElement对象或坐标。') - current_x, current_y = self.locations.midpoint - width = target_x - current_x - height = target_y - current_y - - duration = .02 if duration < .02 else duration - num = int(duration * 50) - - # 将要经过的点存入列表 - points = [(int(current_x + i * (width / num)), int(current_y + i * (height / num))) for i in range(1, num)] - points.append((target_x, target_y)) - - from .._units.action_chains import ActionChains - actions = ActionChains(self.page) - actions.hold(self) - - # 逐个访问要经过的点 - for x, y in points: - t = perf_counter() - actions.move(x - current_x, y - current_y) - current_x, current_y = x, y - ss = .02 - perf_counter() + t - if ss > 0: - sleep(ss) - actions.release() + self.page.actions.hold(self).move_to(ele_or_loc, duration=duration).release() def _get_obj_id(self, node_id=None, backend_id=None): """根据传入node id或backend id获取js中的object id diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index d3ec6f7..e9d4bde 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -84,7 +84,13 @@ class ChromiumBase(BasePage): self._scroll = None if not tab_id: - tab_id = self.browser.tabs[0] + tabs = self.browser.driver.get(f'http://{self.address}/json').json() + tabs = [(i['id'], i['url']) for i in tabs if i['type'] == 'page' and not i['url'].startswith('devtools://')] + if tabs[0][1] == 'chrome://privacy-sandbox-dialog/notice' and len(tabs) > 1: + tab_id = tabs[1][0] + close_privacy_dialog(self, tabs[0][0]) + else: + tab_id = tabs[0][0] self._driver_init(tab_id) if self.ready_state == 'complete' and self._ready_state is None: @@ -100,9 +106,6 @@ class ChromiumBase(BasePage): """ self._is_loading = True self._driver = self.browser._get_driver(tab_id) - if self._driver.run('Target.getTargetInfo', - targetId=self._target_id)['targetInfo']['url'] == 'chrome://privacy-sandbox-dialog/notice': - self._driver = close_privacy_dialog(self) self._alert = Alert() self._driver.set_callback('Page.javascriptDialogOpening', self._on_alert_open) @@ -1038,33 +1041,33 @@ class Alert(object): self.response_text = None -def close_privacy_dialog(page): +def close_privacy_dialog(page, tid): """关闭隐私声明弹窗 :param page: ChromiumBase对象 + :param tid: tab id :return: ChromiumDriver对象 """ - tid = page.tab_id - page._driver.run('Runtime.enable') - page._driver.run('DOM.enable') - page._driver.run('DOM.getDocument') - sid = page._driver.run('DOM.performSearch', query='//*[name()="privacy-sandbox-notice-dialog-app"]', - includeUserAgentShadowDOM=True)['searchId'] - r = page._driver.run('DOM.getSearchResults', searchId=sid, fromIndex=0, toIndex=1)['nodeIds'][0] - while True: - try: - r = page._driver.run('DOM.describeNode', nodeId=r)['node']['shadowRoots'][0]['backendNodeId'] - break - except KeyError: - pass - page._driver.run('DOM.discardSearchResults', searchId=sid) - r = page._driver.run('DOM.resolveNode', backendNodeId=r)['object']['objectId'] - r = page._driver.run('Runtime.callFunctionOn', objectId=r, - functionDeclaration='function()' - '{return this.getElementById("ackButton");}')['result']['objectId'] - page._driver.run('Runtime.callFunctionOn', objectId=r, functionDeclaration='function(){return this.click();}') - while True: - new_tid = page.browser.tabs[0] - if new_tid != tid: - break - sleep(.1) - return page.browser._get_driver(new_tid) + try: + driver = page.browser._get_driver(tid) + driver.run('Runtime.enable') + driver.run('DOM.enable') + driver.run('DOM.getDocument') + sid = driver.run('DOM.performSearch', query='//*[name()="privacy-sandbox-notice-dialog-app"]', + includeUserAgentShadowDOM=True)['searchId'] + r = driver.run('DOM.getSearchResults', searchId=sid, fromIndex=0, toIndex=1)['nodeIds'][0] + end_time = perf_counter() + 3 + while perf_counter() < end_time: + try: + r = driver.run('DOM.describeNode', nodeId=r)['node']['shadowRoots'][0]['backendNodeId'] + break + except KeyError: + pass + driver.run('DOM.discardSearchResults', searchId=sid) + r = driver.run('DOM.resolveNode', backendNodeId=r)['object']['objectId'] + r = driver.run('Runtime.callFunctionOn', objectId=r, + functionDeclaration='function(){return this.getElementById("ackButton");}')['result']['objectId'] + driver.run('Runtime.callFunctionOn', objectId=r, functionDeclaration='function(){return this.click();}') + driver.close() + + except: + pass diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 6159831..12aff73 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -33,6 +33,7 @@ class ChromiumFrame(ChromiumBase): self.doc_ele: ChromiumElement = ... self._states: ElementStates = ... self._ids: FrameIds = ... + self._is_init_get_doc: bool = ... def __call__(self, loc_or_str: Union[Tuple[str, str], str], diff --git a/DrissionPage/_units/action_chains.py b/DrissionPage/_units/action_chains.py index b5d9c2f..8b177d1 100644 --- a/DrissionPage/_units/action_chains.py +++ b/DrissionPage/_units/action_chains.py @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from time import sleep +from time import sleep, perf_counter from .._commons.keys import modifierBit, keyDescriptionForString from .._commons.web import location_in_viewport @@ -22,12 +22,13 @@ class ActionChains: self.curr_x = 0 # 视口坐标 self.curr_y = 0 - def move_to(self, ele_or_loc, offset_x=0, offset_y=0): + def move_to(self, ele_or_loc, offset_x=0, offset_y=0, duration=.5): """鼠标移动到元素中点，或页面上的某个绝对坐标。可设置偏移量 当带偏移量时，偏移量相对于元素左上角坐标 :param ele_or_loc: 元素对象、绝对坐标或文本定位符，坐标为tuple(int, int)形式 :param offset_x: 偏移量x :param offset_y: 偏移量y + :param duration: 拖动用时，传入0即瞬间到达 :return: self """ is_loc = False @@ -50,7 +51,7 @@ class ActionChains: clientHeight = self.page.run_js('return document.body.clientHeight;') self.page.scroll.to_location(lx - clientWidth // 2, ly - clientHeight // 2) - # # 这样设计为了应付那些不随滚动条滚动的元素 + # 这样设计为了应付那些不随滚动条滚动的元素 if is_loc: cx, cy = location_to_client(self.page, lx, ly) else: @@ -59,21 +60,35 @@ class ActionChains: cx = x + offset_x cy = y + offset_y - self._dr.run('Input.dispatchMouseEvent', type='mouseMoved', x=cx, y=cy, modifiers=self.modifier) - self.curr_x = cx - self.curr_y = cy + ox = cx - self.curr_x + oy = cy - self.curr_y + self.move(ox, oy, duration) return self - def move(self, offset_x=0, offset_y=0): + def move(self, offset_x=0, offset_y=0, duration=.5): """鼠标相对当前位置移动若干位置 :param offset_x: 偏移量x :param offset_y: 偏移量y + :param duration: 拖动用时，传入0即瞬间到达 :return: self """ - self.curr_x += offset_x - self.curr_y += offset_y - self._dr.run('Input.dispatchMouseEvent', type='mouseMoved', x=self.curr_x, y=self.curr_y, - modifiers=self.modifier) + duration = .02 if duration < .02 else duration + num = int(duration * 50) + + points = [(self.curr_x + i * (offset_x / num), + self.curr_y + i * (offset_y / num)) for i in range(1, num)] + points.append((self.curr_x + offset_x, self.curr_y + offset_y)) + + for x, y in points: + t = perf_counter() + self.curr_x = x + self.curr_y = y + self._dr.run('Input.dispatchMouseEvent', type='mouseMoved', x=self.curr_x, y=self.curr_y, + modifiers=self.modifier) + ss = .02 - perf_counter() + t + if ss > 0: + sleep(ss) + return self def click(self, on_ele=None): @@ -122,7 +137,7 @@ class ActionChains: :return: self """ if on_ele: - self.move_to(on_ele) + self.move_to(on_ele, duration=0) self._release('left') return self @@ -140,7 +155,7 @@ class ActionChains: :return: self """ if on_ele: - self.move_to(on_ele) + self.move_to(on_ele, duration=0) self._release('right') return self @@ -158,7 +173,7 @@ class ActionChains: :return: self """ if on_ele: - self.move_to(on_ele) + self.move_to(on_ele, duration=0) self._release('middle') return self @@ -170,7 +185,7 @@ class ActionChains: :return: self """ if on_ele: - self.move_to(on_ele) + self.move_to(on_ele, duration=0) self._dr.run('Input.dispatchMouseEvent', type='mousePressed', button=button, clickCount=count, x=self.curr_x, y=self.curr_y, modifiers=self.modifier) return self @@ -192,7 +207,7 @@ class ActionChains: :return: self """ if on_ele: - self.move_to(on_ele) + self.move_to(on_ele, duration=0) self._dr.run('Input.dispatchMouseEvent', type='mouseWheel', x=self.curr_x, y=self.curr_y, deltaX=delta_x, deltaY=delta_y, modifiers=self.modifier) return self diff --git a/DrissionPage/_units/action_chains.pyi b/DrissionPage/_units/action_chains.pyi index 471c797..1c66af6 100644 --- a/DrissionPage/_units/action_chains.pyi +++ b/DrissionPage/_units/action_chains.pyi @@ -20,9 +20,9 @@ class ActionChains: self.curr_y: int = ... def move_to(self, ele_or_loc: Union[ChromiumElement, Tuple[int, int], str], - offset_x: int = 0, offset_y: int = 0) -> ActionChains: ... + offset_x: int = 0, offset_y: int = 0, duration: float = .5) -> ActionChains: ... - def move(self, offset_x: int = 0, offset_y: int = 0) -> ActionChains: ... + def move(self, offset_x: int = 0, offset_y: int = 0, duration: float = .5) -> ActionChains: ... def click(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index b71688d..a64ebb5 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -17,19 +17,21 @@ class Clicker(object): """ self._ele = ele - def __call__(self, by_js=False, timeout=1): + def __call__(self, by_js=False, timeout=2, wait_stop=True): """点击元素 如果遇到遮挡，可选择是否用js点击 :param by_js: 是否用js点击，为None时先用模拟点击，遇到遮挡改用js，为True时直接用js点击，为False时只用模拟点击 - :param timeout: 模拟点击的超时时间，等待元素可见、不被遮挡、进入视口 + :param timeout: 模拟点击的超时时间，等待元素可见、可用、进入视口 + :param wait_stop: 是否等待元素运动结束再执行点击 :return: 是否点击成功 """ - return self.left(by_js, timeout) + return self.left(by_js, timeout, wait_stop) - def left(self, by_js=False, timeout=2): + def left(self, by_js=False, timeout=2, wait_stop=True): """点击元素，可选择是否用js点击 :param by_js: 是否用js点击，为None时先用模拟点击，遇到遮挡改用js，为True时直接用js点击，为False时只用模拟点击 - :param timeout: 模拟点击的超时时间，等待元素可见、不被遮挡、进入视口 + :param timeout: 模拟点击的超时时间，等待元素可见、可用、进入视口 + :param wait_stop: 是否等待元素运动结束再执行点击 :return: 是否点击成功 """ if not by_js: # 模拟点击 @@ -53,7 +55,8 @@ class Clicker(object): rect = self._ele.states.has_rect sleep(.001) - self._ele.wait.stop_moving(timeout=end_time - perf_counter()) + if wait_stop: + self._ele.wait.stop_moving(timeout=end_time - perf_counter()) if rect: self._ele.scroll.to_see() rect = self._ele.locations.rect diff --git a/DrissionPage/_units/clicker.pyi b/DrissionPage/_units/clicker.pyi index dbdcf58..04ec474 100644 --- a/DrissionPage/_units/clicker.pyi +++ b/DrissionPage/_units/clicker.pyi @@ -12,9 +12,9 @@ class Clicker(object): def __init__(self, ele: ChromiumElement): self._ele: ChromiumElement = ... - def __call__(self, by_js: Union[None, bool] = False, timeout: float = 1) -> bool: ... + def __call__(self, by_js: Union[None, bool] = False, timeout: float = 2, wait_stop: bool = True) -> bool: ... - def left(self, by_js: Union[None, bool] = False, timeout: float = 1) -> bool: ... + def left(self, by_js: Union[None, bool] = False, timeout: float = 2, wait_stop: bool = True) -> bool: ... def right(self) -> None: ... diff --git a/DrissionPage/_units/select_element.py b/DrissionPage/_units/select_element.py index cc1c29c..9399bb4 100644 --- a/DrissionPage/_units/select_element.py +++ b/DrissionPage/_units/select_element.py @@ -241,4 +241,4 @@ class SelectElement(object): def _dispatch_change(self): """触发修改动作""" - self._ele.run_js('this.dispatchEvent(new UIEvent("change"));') + self._ele.run_js('this.dispatchEvent(new Event("change", {bubbles: true}));') diff --git a/setup.py b/setup.py index e164891..35730d9 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b6", + version="4.0.0b7", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From f85076065140938fab6289357e4cec2e08e2e468 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 13 Nov 2023 18:20:50 +0800 Subject: [PATCH 080/182] =?UTF-8?q?=E6=8E=A5=E7=AE=A1=E6=97=A0=E5=A4=B4?= =?UTF-8?q?=E6=B5=8F=E8=A7=88=E5=99=A8=E6=97=B6=E5=88=A4=E6=96=AD=E6=98=AF?= =?UTF-8?q?=E5=90=A6=E9=9C=80=E8=A6=81=E7=94=A8=E6=97=A0=E5=A4=B4=EF=BC=8C?= =?UTF-8?q?=E6=9C=AA=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_commons/browser.py | 20 +++++++++++++------- DrissionPage/_commons/browser.pyi | 2 +- DrissionPage/_configs/chromium_options.py | 1 + DrissionPage/_configs/chromium_options.pyi | 1 + DrissionPage/_pages/chromium_page.py | 12 +++++++++++- 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_commons/browser.py index 6a52559..76b8d67 100644 --- a/DrissionPage/_commons/browser.py +++ b/DrissionPage/_commons/browser.py @@ -19,7 +19,7 @@ from .tools import port_is_using def connect_browser(option): """连接或启动浏览器 :param option: ChromiumOptions对象 - :return: None + :return: 返回是否接管的浏览器 """ debugger_address = option.debugger_address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') chrome_path = option.browser_path @@ -27,13 +27,18 @@ def connect_browser(option): ip, port = debugger_address.split(':') if ip != '127.0.0.1' or port_is_using(ip, port) or option.is_existing_only: test_connect(ip, port) - return + option._headless = False + for i in option.arguments: + if i.startswith('--headless') and not i.endswith('=false'): + option._headless = True + break + return True # ----------创建浏览器进程---------- args = get_launch_args(option) set_prefs(option) try: - debugger = _run_browser(port, chrome_path, args) + _run_browser(port, chrome_path, args) # 传入的路径找不到，主动在ini文件、注册表、系统变量中找 except FileNotFoundError: @@ -43,10 +48,10 @@ def connect_browser(option): if not chrome_path: raise FileNotFoundError('无法找到chrome路径，请手动配置。') - debugger = _run_browser(port, chrome_path, args) + _run_browser(port, chrome_path, args) test_connect(ip, port) - return chrome_path, debugger + return False def get_launch_args(opt): @@ -71,7 +76,6 @@ def get_launch_args(opt): elif i.startswith('--headless'): if i == '--headless=false': headless = False - continue elif i == '--headless': i = '--headless=new' headless = True @@ -89,13 +93,15 @@ def get_launch_args(opt): if not remote_allow: result.add('--remote-allow-origins=*') - if headless is not None and system().lower() == 'linux': + if headless is None and system().lower() == 'linux': from os import popen r = popen('systemctl list-units | grep graphical.target') if 'graphical.target' not in r.read(): + headless = True result.add('--headless=new') result = list(result) + opt._headless = headless # ----------处理插件extensions------------- ext = opt.extensions diff --git a/DrissionPage/_commons/browser.pyi b/DrissionPage/_commons/browser.pyi index 8c041ef..9e74e87 100644 --- a/DrissionPage/_commons/browser.pyi +++ b/DrissionPage/_commons/browser.pyi @@ -8,7 +8,7 @@ from typing import Union from .._configs.chromium_options import ChromiumOptions -def connect_browser(option: ChromiumOptions) -> None: ... +def connect_browser(option: ChromiumOptions) -> bool: ... def get_launch_args(opt: ChromiumOptions) -> list: ... diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index 214ccd0..e5a09d0 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -21,6 +21,7 @@ class ChromiumOptions(object): self._user_data_path = None self._user = 'Default' self._prefs_to_del = [] + self._headless = None if read_file is not False: ini_path = str(ini_path) if ini_path else None diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index 161ff85..b378421 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -26,6 +26,7 @@ class ChromiumOptions(object): self._auto_port: bool = ... self._system_user_path: bool = ... self._existing_only: bool = ... + self._headless: bool = ... @property def download_path(self) -> str: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 4e60c52..b201c83 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -63,7 +63,7 @@ class ChromiumPage(ChromiumBase): def _run_browser(self): """连接浏览器""" - connect_browser(self._driver_options) + is_exist = connect_browser(self._driver_options) ws = get(f'http://{self._driver_options.debugger_address}/json/version', headers={'Connection': 'close'}) if not ws: @@ -71,6 +71,16 @@ class ChromiumPage(ChromiumBase): ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] self._browser = Browser(self._driver_options.debugger_address, ws, self) + print(is_exist, self._driver_options._headless, self._browser.run_cdp('Browser.getVersion')['userAgent']) + if (is_exist and self._driver_options._headless is False and + 'headless' in self._browser.run_cdp('Browser.getVersion')['userAgent'].lower()): + print('aaa') + self._browser.quit(3) + connect_browser(self._driver_options) + ws = get(f'http://{self._driver_options.debugger_address}/json/version', headers={'Connection': 'close'}) + ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] + self._browser = Browser(self._driver_options.debugger_address, ws, self) + def _d_set_runtime_settings(self): """设置运行时用到的属性""" self._timeouts = Timeout(self, From a5e86167e275293153c5eb7e40f161d7badf3759 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 13 Nov 2023 20:23:04 +0800 Subject: [PATCH 081/182] =?UTF-8?q?4.0.0b7=E5=BD=93set=5Fheadless(False)?= =?UTF-8?q?=E4=BD=86=E6=8E=A5=E7=AE=A1=E4=BA=86=E6=97=A0=E5=A4=B4=E6=B5=8F?= =?UTF-8?q?=E8=A7=88=E5=99=A8=EF=BC=8C=E5=B0=86=E8=B5=B7=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E5=B9=B6=E5=90=AF=E5=8A=A8=E6=96=B0=E7=9A=84=E6=9C=89=E5=A4=B4?= =?UTF-8?q?=E6=B5=8F=E8=A7=88=E5=99=A8=EF=BC=9Bprop()=E6=94=B9=E7=94=A8js?= =?UTF-8?q?=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_commons/browser.py | 1 + DrissionPage/_elements/chromium_element.py | 13 +++++-------- DrissionPage/_pages/chromium_page.py | 2 -- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_commons/browser.py index 76b8d67..dfd0ce1 100644 --- a/DrissionPage/_commons/browser.py +++ b/DrissionPage/_commons/browser.py @@ -76,6 +76,7 @@ def get_launch_args(opt): elif i.startswith('--headless'): if i == '--headless=false': headless = False + continue elif i == '--headless': i = '--headless=new' headless = True diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 2d9d072..dcc0a4d 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -369,14 +369,11 @@ class ChromiumElement(DrissionElement): :param prop: 属性名 :return: 属性值文本 """ - p = self.page.run_cdp('Runtime.getProperties', objectId=self._obj_id)['result'] - for i in p: - if i['name'] == prop: - if 'value' not in i or 'value' not in i['value']: - return None - - value = i['value']['value'] - return format_html(value) if isinstance(value, str) else value + try: + value = self.run_js(f'return this.{prop};') + return format_html(value) if isinstance(value, str) else value + except: + return None def run_js(self, script, *args, as_expr=False): """对本元素执行javascript代码 diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index b201c83..e6aacc4 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -71,10 +71,8 @@ class ChromiumPage(ChromiumBase): ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] self._browser = Browser(self._driver_options.debugger_address, ws, self) - print(is_exist, self._driver_options._headless, self._browser.run_cdp('Browser.getVersion')['userAgent']) if (is_exist and self._driver_options._headless is False and 'headless' in self._browser.run_cdp('Browser.getVersion')['userAgent'].lower()): - print('aaa') self._browser.quit(3) connect_browser(self._driver_options) ws = get(f'http://{self._driver_options.debugger_address}/json/version', headers={'Connection': 'close'}) From 47557844e069d69b8d37c2376384300a8526f2d3 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 14 Nov 2023 11:41:33 +0800 Subject: [PATCH 082/182] =?UTF-8?q?=E6=8F=92=E4=BB=B6=E8=B7=AF=E5=BE=84?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=8F=98=E6=88=90=E7=BB=9D=E5=AF=B9=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=EF=BC=9B=E6=8A=93=E5=8C=85=E5=8A=A0=E4=B8=8A=E5=88=A4?= =?UTF-8?q?=E6=96=ADframe=EF=BC=9B=E5=85=83=E7=B4=A0=E5=9D=90=E6=A0=87?= =?UTF-8?q?=E6=94=B9=E4=B8=BAfloat=EF=BC=9B=E4=BF=AE=E5=A4=8D=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E6=96=87=E4=BB=B6=E6=8D=9F=E5=9D=8F=E6=97=B6=E5=87=BA?= =?UTF-8?q?=E7=8E=B0=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_commons/browser.py | 9 ++-- DrissionPage/_commons/web.pyi | 6 +-- DrissionPage/_elements/chromium_element.py | 6 +-- DrissionPage/_elements/chromium_element.pyi | 53 +++++++-------------- DrissionPage/_pages/chromium_page.py | 2 + DrissionPage/_units/clicker.py | 2 +- DrissionPage/_units/clicker.pyi | 4 +- DrissionPage/_units/ids.py | 4 +- DrissionPage/_units/ids.pyi | 4 +- DrissionPage/_units/locations.py | 18 +++---- DrissionPage/_units/locations.pyi | 24 +++++----- DrissionPage/_units/network_listener.py | 4 ++ DrissionPage/_units/tab_rect.pyi | 7 +-- setup.py | 2 +- 14 files changed, 68 insertions(+), 77 deletions(-) diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_commons/browser.py index dfd0ce1..c0a5e80 100644 --- a/DrissionPage/_commons/browser.py +++ b/DrissionPage/_commons/browser.py @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from json import load, dump +from json import load, dump, JSONDecodeError from pathlib import Path from subprocess import Popen, DEVNULL from tempfile import gettempdir @@ -105,7 +105,7 @@ def get_launch_args(opt): opt._headless = headless # ----------处理插件extensions------------- - ext = opt.extensions + ext = [str(Path(e).absolute()) for e in opt.extensions] if ext: ext = ','.join(set(ext)) ext = f'--load-extension={ext}' @@ -140,7 +140,10 @@ def set_prefs(opt): f.write('{}') with open(prefs_file, "r", encoding='utf-8') as f: - prefs_dict = load(f) + try: + prefs_dict = load(f) + except JSONDecodeError: + prefs_dict = {} for pref in prefs: value = prefs[pref] diff --git a/DrissionPage/_commons/web.pyi b/DrissionPage/_commons/web.pyi index eae78ce..d6b9dc2 100644 --- a/DrissionPage/_commons/web.pyi +++ b/DrissionPage/_commons/web.pyi @@ -9,9 +9,9 @@ from typing import Union from requests import Session from requests.cookies import RequestsCookieJar +from .._base.base import BasePage, DrissionElement from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase -from ..base import DrissionElement, BasePage def get_ele_txt(e: DrissionElement) -> str: ... @@ -20,10 +20,10 @@ def get_ele_txt(e: DrissionElement) -> str: ... def format_html(text: str) -> str: ... -def location_in_viewport(page: ChromiumBase, loc_x: int, loc_y: int) -> bool: ... +def location_in_viewport(page: ChromiumBase, loc_x: float, loc_y: float) -> bool: ... -def offset_scroll(ele: ChromiumElement, offset_x: int, offset_y: int) -> tuple: ... +def offset_scroll(ele: ChromiumElement, offset_x: float, offset_y: float) -> tuple: ... def make_absolute_link(link, page: BasePage = None) -> str: ... diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index dcc0a4d..a6549bf 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -16,7 +16,7 @@ from .._commons.tools import get_usable_path from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, offset_scroll from .._units.clicker import Clicker from .._units.element_states import ElementStates, ShadowRootStates -from .._units.ids import Ids, ElementIds +from .._units.ids import ShadowRootIds, ElementIds from .._units.locations import Locations from .._units.scroller import ElementScroller from .._units.select_element import SelectElement @@ -130,7 +130,7 @@ class ChromiumElement(DrissionElement): def size(self): """返回元素宽和高组成的元组""" border = self.page.run_cdp('DOM.getBoxModel', backendNodeId=self._backend_id)['model']['border'] - return int(border[2] - border[0]), int(border[5] - border[1]) + return border[2] - border[0], border[5] - border[1] @property def set(self): @@ -772,7 +772,7 @@ class ChromiumShadowRoot(BaseElement): self._obj_id = obj_id self._node_id = self._get_node_id(obj_id) self._backend_id = self._get_backend_id(self._node_id) - self._ids = Ids(self) + self._ids = ShadowRootIds(self) self._states = None def __repr__(self): diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index cf64a41..1108d9d 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -15,7 +15,7 @@ from .._pages.chromium_page import ChromiumPage from .._pages.web_page import WebPage from .._units.clicker import Clicker from .._units.element_states import ShadowRootStates, ElementStates -from .._units.ids import Ids, ElementIds +from .._units.ids import ElementIds, ShadowRootIds from .._units.locations import Locations from .._units.scroller import ElementScroller from .._units.select_element import SelectElement @@ -25,9 +25,7 @@ from .._units.waiter import ElementWaiter class ChromiumElement(DrissionElement): - def __init__(self, - page: ChromiumBase, - node_id: str = None, obj_id: str = None, backend_id: str = None): + def __init__(self, page: ChromiumBase, node_id: str = None, obj_id: str = None, backend_id: str = None): self._tag: str = ... self.page: Union[ChromiumPage, WebPage] = ... self._node_id: str = ... @@ -46,8 +44,7 @@ class ChromiumElement(DrissionElement): def __repr__(self) -> str: ... - def __call__(self, - loc_or_str: Union[Tuple[str, str], str], + def __call__(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> Union[ChromiumElement, str, None]: ... @property @@ -73,7 +70,7 @@ class ChromiumElement(DrissionElement): def ids(self) -> ElementIds: ... @property - def size(self) -> Tuple[int, int]: ... + def size(self) -> Tuple[float, float]: ... @property def set(self) -> ChromiumElementSetter: ... @@ -82,7 +79,7 @@ class ChromiumElement(DrissionElement): def states(self) -> ElementStates: ... @property - def location(self) -> Tuple[int, int]: ... + def location(self) -> Tuple[float, float]: ... @property def locations(self) -> Locations: ... @@ -167,12 +164,10 @@ class ChromiumElement(DrissionElement): def run_async_js(self, script: str, *args: Any, as_expr: bool = False) -> None: ... - def ele(self, - loc_or_str: Union[Tuple[str, str], str], + def ele(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> Union[ChromiumElement, str]: ... - def eles(self, - loc_or_str: Union[Tuple[str, str], str], + def eles(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> List[Union[ChromiumElement, str]]: ... def s_ele(self, loc_or_str: Union[Tuple[str, str], str] = None) -> Union[SessionElement, str, NoneElement]: ... @@ -220,12 +215,9 @@ class ChromiumElement(DrissionElement): class ChromiumShadowRoot(BaseElement): - def __init__(self, - parent_ele: ChromiumElement, - obj_id: str = None, - backend_id: str = None): + def __init__(self, parent_ele: ChromiumElement, obj_id: str = None, backend_id: str = None): self._obj_id: str = ... - self._ids: Ids = ... + self._ids: ShadowRootIds = ... self._node_id: str = ... self._backend_id: str = ... self.page: ChromiumPage = ... @@ -234,12 +226,11 @@ class ChromiumShadowRoot(BaseElement): def __repr__(self) -> str: ... - def __call__(self, - loc_or_str: Union[Tuple[str, str], str], + def __call__(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> ChromiumElement: ... @property - def ids(self) -> Ids: ... + def ids(self) -> ShadowRootIds: ... @property def states(self) -> ShadowRootStates: ... @@ -279,12 +270,10 @@ class ChromiumShadowRoot(BaseElement): def afters(self, filter_loc: Union[tuple, str] = '') -> List[Union[ChromiumElement, str]]: ... - def ele(self, - loc_or_str: Union[Tuple[str, str], str], + def ele(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> Union[ChromiumElement]: ... - def eles(self, - loc_or_str: Union[Tuple[str, str], str], + def eles(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> List[ChromiumElement]: ... def s_ele(self, loc_or_str: Union[Tuple[str, str], str] = None) -> Union[SessionElement, str, NoneElement]: ... @@ -303,24 +292,16 @@ class ChromiumShadowRoot(BaseElement): def _get_backend_id(self, node_id: str) -> str: ... -def find_in_chromium_ele(ele: ChromiumElement, - loc: Union[str, Tuple[str, str]], - single: bool = True, - timeout: float = None, - relative: bool = True) \ +def find_in_chromium_ele(ele: ChromiumElement, loc: Union[str, Tuple[str, str]], + single: bool = True, timeout: float = None, relative: bool = True) \ -> Union[ChromiumElement, str, NoneElement, List[Union[ChromiumElement, str]]]: ... -def find_by_xpath(ele: ChromiumElement, - xpath: str, - single: bool, - timeout: float, +def find_by_xpath(ele: ChromiumElement, xpath: str, single: bool, timeout: float, relative: bool = True) -> Union[ChromiumElement, List[ChromiumElement], NoneElement]: ... -def find_by_css(ele: ChromiumElement, - selector: str, - single: bool, +def find_by_css(ele: ChromiumElement, selector: str, single: bool, timeout: float) -> Union[ChromiumElement, List[ChromiumElement], NoneElement]: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index e6aacc4..c5fa80f 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -129,6 +129,8 @@ class ChromiumPage(ChromiumBase): @property def rect(self): + """返回保存窗口方位信息的对象""" + self.wait.load_complete() if self._rect is None: self._rect = TabRect(self) return self._rect diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index a64ebb5..0ab9234 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -73,7 +73,7 @@ class Clicker(object): by_js = True elif can_click and (by_js is False or not self._ele.states.is_covered): - x = int(rect[1][0] - (rect[1][0] - rect[0][0]) / 2) + x = rect[1][0] - (rect[1][0] - rect[0][0]) / 2 y = rect[0][0] + 3 try: r = self._ele.page.run_cdp('DOM.getNodeForLocation', x=x, y=y, includeUserAgentShadowDOM=True, diff --git a/DrissionPage/_units/clicker.pyi b/DrissionPage/_units/clicker.pyi index 04ec474..10a8121 100644 --- a/DrissionPage/_units/clicker.pyi +++ b/DrissionPage/_units/clicker.pyi @@ -20,8 +20,8 @@ class Clicker(object): def middle(self) -> None: ... - def at(self, offset_x: int = None, offset_y: int = None, button: str = 'left', count: int = 1) -> None: ... + def at(self, offset_x: float = None, offset_y: float = None, button: str = 'left', count: int = 1) -> None: ... def twice(self, by_js: bool = False) -> None: ... - def _click(self, client_x: int, client_y: int, button: str = 'left', count: int = 1) -> None: ... + def _click(self, client_x: float, client_y: float, button: str = 'left', count: int = 1) -> None: ... diff --git a/DrissionPage/_units/ids.py b/DrissionPage/_units/ids.py index 5e1ee80..f5fb8d8 100644 --- a/DrissionPage/_units/ids.py +++ b/DrissionPage/_units/ids.py @@ -5,7 +5,7 @@ """ -class Ids(object): +class ShadowRootIds(object): def __init__(self, ele): self._ele = ele @@ -25,7 +25,7 @@ class Ids(object): return self._ele._backend_id -class ElementIds(Ids): +class ElementIds(ShadowRootIds): @property def doc_id(self): """返回所在document的object id""" diff --git a/DrissionPage/_units/ids.pyi b/DrissionPage/_units/ids.pyi index f224259..20d4271 100644 --- a/DrissionPage/_units/ids.pyi +++ b/DrissionPage/_units/ids.pyi @@ -9,7 +9,7 @@ from .._pages.chromium_frame import ChromiumFrame from .._elements.chromium_element import ChromiumElement, ChromiumShadowRoot -class Ids(object): +class ShadowRootIds(object): def __init__(self, ele: Union[ChromiumElement, ChromiumShadowRoot]): self._ele: Union[ChromiumElement, ChromiumShadowRoot] = ... @@ -23,7 +23,7 @@ class Ids(object): def backend_id(self) -> str: ... -class ElementIds(Ids): +class ElementIds(ShadowRootIds): @property def doc_id(self) -> str: ... diff --git a/DrissionPage/_units/locations.py b/DrissionPage/_units/locations.py index a402681..010a962 100644 --- a/DrissionPage/_units/locations.py +++ b/DrissionPage/_units/locations.py @@ -34,19 +34,19 @@ class Locations(object): def viewport_location(self): """返回元素左上角在视口中的坐标""" m = self._get_viewport_rect('border') - return int(m[0]), int(m[1]) + return m[0], m[1] @property def viewport_midpoint(self): """返回元素中间点在视口中的坐标""" m = self._get_viewport_rect('border') - return int(m[0] + (m[2] - m[0]) // 2), int(m[3] + (m[5] - m[3]) // 2) + return m[0] + (m[2] - m[0]) // 2, m[3] + (m[5] - m[3]) // 2 @property def viewport_click_point(self): """返回元素接受点击的点视口坐标""" m = self._get_viewport_rect('padding') - return int(self.viewport_midpoint[0]), int(m[1]) + 3 + return self.viewport_midpoint[0], m[1] + 3 @property def screen_location(self): @@ -54,7 +54,7 @@ class Locations(object): vx, vy = self._ele.page.rect.viewport_location ex, ey = self.viewport_location pr = self._ele.page.run_js('return window.devicePixelRatio;') - return int((vx + ex) * pr), int((ey + vy) * pr) + return (vx + ex) * pr, (ey + vy) * pr @property def screen_midpoint(self): @@ -62,7 +62,7 @@ class Locations(object): vx, vy = self._ele.page.rect.viewport_location ex, ey = self.viewport_midpoint pr = self._ele.page.run_js('return window.devicePixelRatio;') - return int((vx + ex) * pr), int((ey + vy) * pr) + return (vx + ex) * pr, (ey + vy) * pr @property def screen_click_point(self): @@ -70,15 +70,15 @@ class Locations(object): vx, vy = self._ele.page.rect.viewport_location ex, ey = self.viewport_click_point pr = self._ele.page.run_js('return window.devicePixelRatio;') - return int((vx + ex) * pr), int((ey + vy) * pr) + return (vx + ex) * pr, (ey + vy) * pr @property def rect(self): """返回元素四个角坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" vr = self._get_viewport_rect('border') r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] - sx = int(r['pageX']) - sy = int(r['pageY']) + sx = r['pageX'] + sy = r['pageY'] return [(vr[0] + sx, vr[1] + sy), (vr[2] + sx, vr[3] + sy), (vr[4] + sx, vr[5] + sy), (vr[6] + sx, vr[7] + sy)] @@ -86,7 +86,7 @@ class Locations(object): def viewport_rect(self): """返回元素四个角视口坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" r = self._get_viewport_rect('border') - return [(int(r[0]), int(r[1])), (int(r[2]), int(r[3])), (int(r[4]), int(r[5])), (int(r[6]), int(r[7]))] + return [(r[0], r[1]), (r[2], r[3]), (r[4], r[5]), (r[6], r[7])] def _get_viewport_rect(self, quad): """按照类型返回在可视窗口中的范围 diff --git a/DrissionPage/_units/locations.pyi b/DrissionPage/_units/locations.pyi index 5e61165..5e7ca31 100644 --- a/DrissionPage/_units/locations.pyi +++ b/DrissionPage/_units/locations.pyi @@ -13,38 +13,38 @@ class Locations(object): self._ele: ChromiumElement = ... @property - def location(self) -> Tuple[int, int]: ... + def location(self) -> Tuple[float, float]: ... @property - def midpoint(self) -> Tuple[int, int]: ... + def midpoint(self) -> Tuple[float, float]: ... @property - def click_point(self) -> Tuple[int, int]: ... + def click_point(self) -> Tuple[float, float]: ... @property - def viewport_location(self) -> Tuple[int, int]: ... + def viewport_location(self) -> Tuple[float, float]: ... @property - def viewport_midpoint(self) -> Tuple[int, int]: ... + def viewport_midpoint(self) -> Tuple[float, float]: ... @property - def viewport_click_point(self) -> Tuple[int, int]: ... + def viewport_click_point(self) -> Tuple[float, float]: ... @property - def screen_location(self) -> Tuple[int, int]: ... + def screen_location(self) -> Tuple[float, float]: ... @property - def screen_midpoint(self) -> Tuple[int, int]: ... + def screen_midpoint(self) -> Tuple[float, float]: ... @property - def screen_click_point(self) -> Tuple[int, int]: ... + def screen_click_point(self) -> Tuple[float, float]: ... @property - def rect(self) -> List[Tuple[int, int], ...]: ... + def rect(self) -> List[Tuple[float, float], ...]: ... @property - def viewport_rect(self) -> List[Tuple[int, int], ...]: ... + def viewport_rect(self) -> List[Tuple[float, float], ...]: ... def _get_viewport_rect(self, quad: str) -> Union[list, None]: ... - def _get_page_coord(self, x: int, y: int) -> Tuple[int, int]: ... + def _get_page_coord(self, x: float, y: float) -> Tuple[float, float]: ... diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index 8b5d2cf..7ab0df9 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -190,6 +190,8 @@ class NetworkListener(object): def _requestWillBeSent(self, **kwargs): """接收到请求时的回调函数""" + if kwargs['frameId'] != self._page._frame_id: + return p = None if not self._targets: if not self._method or kwargs['request']['method'] in self._method: @@ -220,6 +222,8 @@ class NetworkListener(object): def _response_received(self, **kwargs): """接收到返回信息时处理方法""" + if kwargs['frameId'] != self._page._frame_id: + return request = self._request_ids.get(kwargs['requestId'], None) if request: request._raw_response = kwargs['response'] diff --git a/DrissionPage/_units/tab_rect.pyi b/DrissionPage/_units/tab_rect.pyi index 6b80684..d4c1223 100644 --- a/DrissionPage/_units/tab_rect.pyi +++ b/DrissionPage/_units/tab_rect.pyi @@ -6,12 +6,13 @@ from typing import Tuple, Union from .._pages.chromium_page import ChromiumPage -from .._pages.chromium_tab import ChromiumTab +from .._pages.chromium_tab import ChromiumTab, WebPageTab +from .._pages.web_page import WebPage class TabRect(object): - def __init__(self, page: Union[ChromiumPage, ChromiumTab]): - self._page: Union[ChromiumPage, ChromiumTab] = ... + def __init__(self, page: Union[ChromiumPage, ChromiumTab, WebPage, WebPageTab]): + self._page: Union[ChromiumPage, ChromiumTab, WebPage, WebPageTab] = ... @property def window_state(self) -> str: ... diff --git a/setup.py b/setup.py index 35730d9..77ed58a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b7", + version="4.0.0b8", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From a4bf7da0bd06fb8e6d15c6945d211cf882f9f4d9 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 15 Nov 2023 00:31:46 +0800 Subject: [PATCH 083/182] =?UTF-8?q?run=5Fjs()=E7=AD=89=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E5=8A=A0=E4=B8=8Atimeout=E5=8F=82=E6=95=B0=EF=BC=9B=E4=BF=AE?= =?UTF-8?q?=E5=A4=8Dtab=E7=9B=B8=E5=85=B3=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 22 ++++++++----- DrissionPage/_elements/chromium_element.pyi | 8 ++--- DrissionPage/_pages/chromium_base.py | 16 ++++++---- DrissionPage/_pages/chromium_base.pyi | 34 +++++++-------------- DrissionPage/_pages/chromium_frame.py | 15 ++++----- DrissionPage/_pages/chromium_frame.pyi | 2 +- DrissionPage/_pages/chromium_page.py | 7 +++-- 7 files changed, 52 insertions(+), 52 deletions(-) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index a6549bf..a19b02f 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -375,24 +375,27 @@ class ChromiumElement(DrissionElement): except: return None - def run_js(self, script, *args, as_expr=False): + def run_js(self, script, *args, as_expr=False, timeout=None): """对本元素执行javascript代码 :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 + :param timeout: js超时时间，为None则使用页面timeouts.script设置 :return: 运行的结果 """ - return run_js(self, script, as_expr, self.page.timeouts.script, args) + return run_js(self, script, as_expr, self.page.timeouts.script if timeout is None else timeout, args) - def run_async_js(self, script, *args, as_expr=False): + def run_async_js(self, script, *args, as_expr=False, timeout=None): """以异步方式对本元素执行javascript代码 :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 + :param timeout: js超时时间，为None则使用页面timeouts.script设置 :return: None """ from threading import Thread - Thread(target=run_js, args=(self, script, as_expr, self.page.timeouts.script, args, True)).start() + Thread(target=run_js, args=(self, script, as_expr, self.page.timeouts.script if timeout is None else timeout, + args, True)).start() def ele(self, loc_or_str, timeout=None): """返回当前元素下级符合条件的第一个元素、属性或节点文本 @@ -814,24 +817,27 @@ class ChromiumShadowRoot(BaseElement): self._states = ShadowRootStates(self) return self._states - def run_js(self, script, *args, as_expr=False): + def run_js(self, script, *args, as_expr=False, timeout=None): """运行javascript代码 :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 + :param timeout: js超时时间，为None则使用页面timeouts.script设置 :return: 运行的结果 """ - return run_js(self, script, as_expr, self.page.timeouts.script, args) + return run_js(self, script, as_expr, self.page.timeouts.script if timeout is None else timeout, args) - def run_async_js(self, script, *args, as_expr=False): + def run_async_js(self, script, *args, as_expr=False, timeout=None): """以异步方式执行js代码 :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 + :param timeout: js超时时间，为None则使用页面timeouts.script设置 :return: None """ from threading import Thread - Thread(target=run_js, args=(self, script, as_expr, self.page.timeouts.script, args)).start() + Thread(target=run_js, args=(self, script, as_expr, + self.page.timeouts.script if timeout is None else timeout, args)).start() def parent(self, level_or_loc=1, index=1): """返回上面某一级父元素，可指定层数或用查询语法定位 diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 1108d9d..315461e 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -160,9 +160,9 @@ class ChromiumElement(DrissionElement): def prop(self, prop: str) -> Union[str, int, None]: ... - def run_js(self, script: str, *args: Any, as_expr: bool = False) -> Any: ... + def run_js(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> Any: ... - def run_async_js(self, script: str, *args: Any, as_expr: bool = False) -> None: ... + def run_async_js(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> None: ... def ele(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> Union[ChromiumElement, str]: ... @@ -244,9 +244,9 @@ class ChromiumShadowRoot(BaseElement): @property def inner_html(self) -> str: ... - def run_js(self, script: str, *args: Any, as_expr: bool = False) -> Any: ... + def run_js(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> Any: ... - def run_async_js(self, script: str, *args: Any, as_expr: bool = False) -> None: ... + def run_async_js(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> None: ... def parent(self, level_or_loc: Union[str, int] = 1, index: int = 1) -> ChromiumElement: ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index e9d4bde..91c2e06 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -456,34 +456,38 @@ class ChromiumBase(BasePage): self.wait.load_complete() return self.run_cdp(cmd, **cmd_args) - def run_js(self, script, *args, as_expr=False): + def run_js(self, script, *args, as_expr=False, timeout=None): """运行javascript代码 :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 + :param timeout: js超时时间，为None则使用页面timeouts.script设置 :return: 运行的结果 """ - return run_js(self, script, as_expr, self.timeouts.script, args) + return run_js(self, script, as_expr, self.timeouts.script if timeout is None else timeout, args) - def run_js_loaded(self, script, *args, as_expr=False): + def run_js_loaded(self, script, *args, as_expr=False, timeout=None): """运行javascript代码，执行前等待页面加载完毕 :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 + :param timeout: js超时时间，为None则使用页面timeouts.script设置 :return: 运行的结果 """ self.wait.load_complete() - return run_js(self, script, as_expr, self.timeouts.script, args) + return run_js(self, script, as_expr, self.timeouts.script if timeout is None else timeout, args) - def run_async_js(self, script, *args, as_expr=False): + def run_async_js(self, script, *args, as_expr=False, timeout=None): """以异步方式执行js代码 :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 + :param timeout: js超时时间，为None则使用页面timeouts.script设置 :return: None """ from threading import Thread - Thread(target=run_js, args=(self, script, as_expr, self.timeouts.script, args)).start() + Thread(target=run_js, args=(self, script, as_expr, self.timeouts.script if timeout is None else timeout, + args)).start() def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None): """访问url diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 826e95e..8d3a1ca 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -164,11 +164,11 @@ class ChromiumBase(BasePage): @property def has_alert(self) -> bool: ... - def run_js(self, script: str, *args: Any, as_expr: bool = False) -> Any: ... + def run_js(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> Any: ... - def run_js_loaded(self, script: str, *args: Any, as_expr: bool = False) -> Any: ... + def run_js_loaded(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> Any: ... - def run_async_js(self, script: str, *args: Any, as_expr: bool = False) -> None: ... + def run_async_js(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> None: ... def get(self, url: str, show_errmsg: bool = False, retry: int = None, interval: float = None, timeout: float = None) -> Union[None, bool]: ... @@ -215,23 +215,15 @@ class ChromiumBase(BasePage): def get_local_storage(self, item: str = None) -> Union[str, dict, None]: ... - def get_screenshot(self, path: [str, Path] = None, name: str = None, - as_bytes: [bool, str] = None, as_base64: [bool, str] = None, - full_page: bool = False, - left_top: Tuple[int, int] = None, - right_bottom: Tuple[int, int] = None) -> Union[str, bytes]: ... + def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, + as_base64: [bool, str] = None, full_page: bool = False, + left_top: Tuple[int, int] = None, right_bottom: Tuple[int, int] = None) -> Union[str, bytes]: ... - def _get_screenshot(self, path: [str, Path] = None, name: str = None, - as_bytes: [bool, str] = None, as_base64: [bool, str] = None, - full_page: bool = False, - left_top: Tuple[int, int] = None, - right_bottom: Tuple[int, int] = None, - ele: ChromiumElement = None) -> Union[str, bytes]: ... + def _get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, + as_base64: [bool, str] = None, full_page: bool = False, left_top: Tuple[int, int] = None, + right_bottom: Tuple[int, int] = None, ele: ChromiumElement = None) -> Union[str, bytes]: ... - def clear_cache(self, - session_storage: bool = True, - local_storage: bool = True, - cache: bool = True, + def clear_cache(self, session_storage: bool = True, local_storage: bool = True, cache: bool = True, cookies: bool = True) -> None: ... def handle_alert(self, accept: bool = True, send: str = None, timeout: float = None) -> Union[str, False]: ... @@ -240,11 +232,7 @@ class ChromiumBase(BasePage): def _on_alert_open(self, **kwargs): ... - def _d_connect(self, - to_url: str, - times: int = 0, - interval: float = 1, - show_errmsg: bool = False, + def _d_connect(self, to_url: str, times: int = 0, interval: float = 1, show_errmsg: bool = False, timeout: float = None) -> Union[bool, None]: ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index dc8f5ec..887e697 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -13,7 +13,7 @@ from .._units.ids import FrameIds from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter from .._units.waiter import FrameWaiter -from ..errors import ContextLossError, ElementLossError, GetDocumentError +from ..errors import ContextLossError, ElementLossError, GetDocumentError, TabClosedError class ChromiumFrame(ChromiumBase): @@ -67,7 +67,7 @@ class ChromiumFrame(ChromiumBase): return self.ele(loc_or_str, timeout) def __repr__(self): - attrs = self.frame_ele.attrs + attrs = self._frame_ele.attrs attrs = [f"{attr}='{attrs[attr]}'" for attr in attrs] return f'' @@ -238,7 +238,7 @@ class ChromiumFrame(ChromiumBase): try: self._frame_ele.attrs - except ElementLossError: + except (ElementLossError, TabClosedError): self._driver.stop() self._reload() @@ -254,7 +254,7 @@ class ChromiumFrame(ChromiumBase): try: self._frame_ele.attrs - except ElementLossError: + except (ElementLossError, TabClosedError): self._driver.stop() self._reload() @@ -431,17 +431,18 @@ class ChromiumFrame(ChromiumBase): """ self.frame_ele.remove_attr(attr) - def run_js(self, script, *args, as_expr=False): + def run_js(self, script, *args, as_expr=False, timeout=None): """运行javascript代码 :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 + :param timeout: js超时时间，为None则使用页面timeouts.script设置 :return: 运行的结果 """ if script.startswith('this.scrollIntoView'): - return self.frame_ele.run_js(script, *args, as_expr=as_expr) + return self.frame_ele.run_js(script, args, as_expr=as_expr, timeout=timeout) else: - return self.doc_ele.run_js(script, *args, as_expr=as_expr) + return self.doc_ele.run_js(script, args, as_expr=as_expr, timeout=timeout) def parent(self, level_or_loc=1, index=1): """返回上面某一级父元素，可指定层数或用查询语法定位 diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 12aff73..1aebd95 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -138,7 +138,7 @@ class ChromiumFrame(ChromiumBase): def remove_attr(self, attr: str) -> None: ... - def run_js(self, script: str, *args: Any, as_expr: bool = False) -> Any: ... + def run_js(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> Any: ... def parent(self, level_or_loc: Union[tuple, str, int] = 1, index: int = 1) -> Union[ChromiumElement, None]: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index c5fa80f..11353fb 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from pathlib import Path -from time import sleep +from time import sleep, perf_counter from requests import get @@ -214,7 +214,7 @@ class ChromiumPage(ChromiumBase): if others: tabs = all_tabs - tabs - end_len = len(all_tabs) - len(tabs) + end_len = len(set(all_tabs) - set(tabs)) if end_len <= 0: self.quit() return @@ -222,7 +222,8 @@ class ChromiumPage(ChromiumBase): for tab in tabs: self.browser.close_tab(tab) sleep(.2) - while self.tabs_count != end_len: + end_time = perf_counter() + 3 + while self.tabs_count != end_len and perf_counter() < end_time: sleep(.1) def close_other_tabs(self, tabs_or_ids=None): From ee89ce948c6804f748a918725aeccbd34befcbf0 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 15 Nov 2023 15:52:49 +0800 Subject: [PATCH 084/182] =?UTF-8?q?ChromiumFrame=E5=A2=9E=E5=8A=A0rect?= =?UTF-8?q?=E5=B1=9E=E6=80=A7=EF=BC=9B=E4=BF=AE=E5=A4=8D=E5=86=85=E9=83=A8?= =?UTF-8?q?=E5=85=83=E7=B4=A0=E4=B8=8D=E8=83=BD=E8=8E=B7=E5=8F=96=E5=B1=8F?= =?UTF-8?q?=E5=B9=95=E5=9D=90=E6=A0=87=E9=97=AE=E9=A2=98=EF=BC=9Bframe=5Fs?= =?UTF-8?q?ize=E6=94=B9=E5=90=8D=E4=B8=BApage=5Fsize=EF=BC=9B=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E5=B0=8F=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/base.py | 2 +- DrissionPage/_base/chromium_driver.py | 2 +- DrissionPage/_configs/chromium_options.py | 4 ++-- DrissionPage/_elements/chromium_element.py | 3 +-- DrissionPage/_pages/chromium_frame.py | 12 +++++++--- DrissionPage/_pages/chromium_frame.pyi | 6 ++++- DrissionPage/_pages/chromium_page.py | 2 +- DrissionPage/_pages/chromium_page.pyi | 2 +- DrissionPage/_pages/chromium_tab.py | 2 +- DrissionPage/_pages/chromium_tab.pyi | 6 ++--- DrissionPage/_units/{tab_rect.py => rect.py} | 22 +++++++++++++++++++ .../_units/{tab_rect.pyi => rect.pyi} | 15 +++++++++++++ 12 files changed, 62 insertions(+), 16 deletions(-) rename DrissionPage/_units/{tab_rect.py => rect.py} (81%) rename DrissionPage/_units/{tab_rect.pyi => rect.pyi} (73%) diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index fbf4f67..eea2c3e 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -414,7 +414,7 @@ class BasePage(BaseParser): :param interval: 重试间隔 :return: 重试次数和间隔组成的tuple """ - self._url = quote(url, safe='-_.~!*\'();:@&=+$,/?#[]') + self._url = quote(url, safe='-_.~!*\'"();:@&=+$,/\\?#[]%') retry = retry if retry is not None else self.retry_times interval = interval if interval is not None else self.retry_interval return retry, interval diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index 26284e3..a7392a4 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -110,7 +110,7 @@ class ChromiumDriver(object): if self._debug: if self._debug is True or 'id' in msg or (isinstance(self._debug, str) and msg.get('method', '').startswith(self._debug)): - print(f'<收 {self.id} {msg_json}') + print(f'<收 {msg_json}') elif isinstance(self._debug, (list, tuple, set)): for m in self._debug: if msg.get('method', '').startswith(m): diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index e5a09d0..b35c8f9 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -38,7 +38,7 @@ class ChromiumOptions(object): self._page_load_strategy = options.get('page_load_strategy', 'normal') self._proxy = om.proxies.get('http', None) self._system_user_path = options.get('system_user_path', False) - self._existing_only = options.get('existing_only', False) + self._existing_only = options.get('is_existing_only', False) user_path = user = False for arg in self._arguments: @@ -446,7 +446,7 @@ class ChromiumOptions(object): # 设置chrome_options attrs = ('debugger_address', 'binary_location', 'arguments', 'extensions', 'user', 'page_load_strategy', - 'auto_port', 'system_user_path', 'is_existing_only') + 'auto_port', 'system_user_path', 'existing_only') for i in attrs: om.set_item('chrome_options', i, self.__getattribute__(f'_{i}')) # 设置代理 diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index a19b02f..f9a2d4c 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -450,8 +450,7 @@ class ChromiumElement(DrissionElement): """ if pseudo_ele: pseudo_ele = f', "{pseudo_ele}"' if pseudo_ele.startswith(':') else f', "::{pseudo_ele}"' - js = f'return window.getComputedStyle(this{pseudo_ele}).getPropertyValue("{style}");' - return self.run_js(js) + return self.run_js(f'return window.getComputedStyle(this{pseudo_ele}).getPropertyValue("{style}");') def get_src(self, timeout=None, base64_to_bytes=True): """返回元素src资源，base64的可转为bytes返回，其它返回str diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 887e697..ba0871f 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -10,6 +10,7 @@ from time import sleep, perf_counter from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase from .._units.ids import FrameIds +from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter from .._units.waiter import FrameWaiter @@ -315,7 +316,7 @@ class ChromiumFrame(ChromiumBase): return self.frame_ele.attrs @property - def frame_size(self): + def page_size(self): """返回frame内页面尺寸，格式：(长, 高)""" w = self.doc_ele.run_js('return this.body.scrollWidth') h = self.doc_ele.run_js('return this.body.scrollHeight') @@ -326,6 +327,11 @@ class ChromiumFrame(ChromiumBase): """返回frame元素大小""" return self.frame_ele.size + @property + def rect(self): + """返回获取坐标和大小的对象""" + return FrameRect(self) + @property def active_ele(self): """返回当前焦点所在元素""" @@ -440,9 +446,9 @@ class ChromiumFrame(ChromiumBase): :return: 运行的结果 """ if script.startswith('this.scrollIntoView'): - return self.frame_ele.run_js(script, args, as_expr=as_expr, timeout=timeout) + return self.frame_ele.run_js(script, *args, as_expr=as_expr, timeout=timeout) else: - return self.doc_ele.run_js(script, args, as_expr=as_expr, timeout=timeout) + return self.doc_ele.run_js(script, *args, as_expr=as_expr, timeout=timeout) def parent(self, level_or_loc=1, index=1): """返回上面某一级父元素，可指定层数或用查询语法定位 diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 1aebd95..facbb5f 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -14,6 +14,7 @@ from .._elements.chromium_element import ChromiumElement from .._units.element_states import ElementStates from .._units.ids import FrameIds from .._units.locations import Locations +from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter from .._units.waiter import FrameWaiter @@ -88,11 +89,14 @@ class ChromiumFrame(ChromiumBase): def attrs(self) -> dict: ... @property - def frame_size(self) -> Tuple[int, int]: ... + def page_size(self) -> Tuple[int, int]: ... @property def size(self) -> Tuple[int, int]: ... + @property + def rect(self) -> FrameRect: ... + @property def active_ele(self) -> ChromiumElement: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 11353fb..87c7c10 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -13,8 +13,8 @@ from .._commons.browser import connect_browser from .._configs.chromium_options import ChromiumOptions from .._pages.chromium_base import ChromiumBase, Timeout from .._pages.chromium_tab import ChromiumTab +from .._units.rect import TabRect from .._units.setter import ChromiumPageSetter -from .._units.tab_rect import TabRect from .._units.waiter import PageWaiter from ..errors import BrowserConnectError diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index a6c7603..62f7e95 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -9,8 +9,8 @@ from .._base.browser import Browser from .._configs.chromium_options import ChromiumOptions from .._pages.chromium_base import ChromiumBase from .._pages.chromium_tab import ChromiumTab +from .._units.rect import TabRect from .._units.setter import ChromiumPageSetter -from .._units.tab_rect import TabRect from .._units.waiter import PageWaiter diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index b65bb03..66edc32 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -9,8 +9,8 @@ from .._base.base import BasePage from .._commons.web import set_session_cookies, set_browser_cookies from .._pages.chromium_base import ChromiumBase from .._pages.session_page import SessionPage +from .._units.rect import TabRect from .._units.setter import TabSetter, WebPageTabSetter -from .._units.tab_rect import TabRect from .._units.waiter import TabWaiter diff --git a/DrissionPage/_pages/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi index ef034b3..5adb85f 100644 --- a/DrissionPage/_pages/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -7,7 +7,6 @@ from typing import Union, Tuple, Any, List, Optional from requests import Session, Response -from .._units.tab_rect import ChromiumTabRect from .chromium_base import ChromiumBase from .chromium_frame import ChromiumFrame from .chromium_page import ChromiumPage @@ -16,6 +15,7 @@ from .web_page import WebPage from .._base.browser import Browser from .._elements.chromium_element import ChromiumElement from .._elements.session_element import SessionElement +from .._units.rect import TabRect from .._units.setter import TabSetter, WebPageTabSetter from .._units.waiter import TabWaiter @@ -25,7 +25,7 @@ class ChromiumTab(ChromiumBase): def __init__(self, page: ChromiumPage, tab_id: str = None): self._page: ChromiumPage = ... self._browser: Browser = ... - self._rect: Optional[ChromiumTabRect] = ... + self._rect: Optional[TabRect] = ... def _d_set_runtime_settings(self) -> None: ... @@ -35,7 +35,7 @@ class ChromiumTab(ChromiumBase): def page(self) -> ChromiumPage: ... @property - def rect(self) -> ChromiumTabRect: ... + def rect(self) -> TabRect: ... @property def set(self) -> TabSetter: ... diff --git a/DrissionPage/_units/tab_rect.py b/DrissionPage/_units/rect.py similarity index 81% rename from DrissionPage/_units/tab_rect.py rename to DrissionPage/_units/rect.py index b72107b..8ebf0fe 100644 --- a/DrissionPage/_units/tab_rect.py +++ b/DrissionPage/_units/rect.py @@ -74,3 +74,25 @@ class TabRect(object): def _get_window_rect(self): """获取窗口范围信息""" return self._page.browser.get_window_bounds(self._page.tab_id) + + +class FrameRect(object): + """异域iframe使用""" + + def __init__(self, frame): + self._frame = frame + + @property + def viewport_location(self): + """返回视口在屏幕中坐标，左上角为(0, 0)""" + return self._frame.frame_ele.locations.screen_location + + @property + def page_size(self): + """返回页面总宽高，格式：(宽, 高)""" + return self._frame.page_size + + @property + def viewport_size(self): + """返回视口宽高，不包括滚动条，格式：(宽, 高)""" + return self._frame.frame_ele.size diff --git a/DrissionPage/_units/tab_rect.pyi b/DrissionPage/_units/rect.pyi similarity index 73% rename from DrissionPage/_units/tab_rect.pyi rename to DrissionPage/_units/rect.pyi index d4c1223..678b34f 100644 --- a/DrissionPage/_units/tab_rect.pyi +++ b/DrissionPage/_units/rect.pyi @@ -5,6 +5,7 @@ """ from typing import Tuple, Union +from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage from .._pages.chromium_tab import ChromiumTab, WebPageTab from .._pages.web_page import WebPage @@ -41,3 +42,17 @@ class TabRect(object): def _get_page_rect(self) -> dict: ... def _get_window_rect(self) -> dict: ... + + +class FrameRect(object): + def __init__(self, frame: ChromiumFrame): + self._frame: ChromiumFrame = ... + + @property + def viewport_location(self) -> Tuple[float, float]: ... + + @property + def page_size(self) -> Tuple[float, float]: ... + + @property + def viewport_size(self) -> Tuple[float, float]: ... From 66de3a6db1c6e1cf78c32043a609eedf024bbe27 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 15 Nov 2023 19:51:16 +0800 Subject: [PATCH 085/182] =?UTF-8?q?TabClosedError=E6=94=B9=E4=B8=BAPageClo?= =?UTF-8?q?sedError=EF=BC=9B=E4=BF=AE=E5=A4=8Diframe=E4=B8=80=E4=B8=AA?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 14 ++++++++------ DrissionPage/_pages/chromium_frame.py | 25 ++++++++++++------------- DrissionPage/errors.py | 4 ++-- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 91c2e06..ba10874 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -23,7 +23,7 @@ from .._units.network_listener import NetworkListener from .._units.screencast import Screencast from .._units.setter import ChromiumBaseSetter from .._units.waiter import BaseWaiter -from ..errors import (ContextLossError, ElementLossError, CDPError, TabClosedError, NoRectError, AlertExistsError, +from ..errors import (ContextLossError, ElementLossError, CDPError, PageClosedError, NoRectError, AlertExistsError, GetDocumentError) @@ -46,8 +46,6 @@ class ChromiumBase(BasePage): self._listener = None self._has_alert = False self._ready_state = None - if self._debug: - print('在__init__变成None') self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc self._download_path = str(Path('.').absolute()) @@ -130,6 +128,8 @@ class ChromiumBase(BasePage): self._driver.set_callback('Page.frameDetached', self._onFrameDetached) def _get_document(self): + if self._debug: + print('获取文档开始') if self._is_reading: return self._is_reading = True @@ -150,6 +150,8 @@ class ChromiumBase(BasePage): self._is_loading = False self._is_reading = False + if self._debug: + print('获取文档结束') def _onFrameDetached(self, **kwargs): self.browser._frames.pop(kwargs['frameId'], None) @@ -285,7 +287,7 @@ class ChromiumBase(BasePage): try: self.run_cdp('Page.getLayoutMetrics') return True - except TabClosedError: + except PageClosedError: return False @property @@ -435,7 +437,7 @@ class ChromiumBase(BasePage): 'No node found for given backend id'): raise ElementLossError elif error == 'tab closed': - raise TabClosedError + raise PageClosedError elif error == 'timeout': raise TimeoutError elif error == 'alert exists.': @@ -680,7 +682,7 @@ class ChromiumBase(BasePage): print('停止页面加载') try: self.run_cdp('Page.stopLoading') - except TabClosedError: + except PageClosedError: pass end_time = perf_counter() + self.timeouts.page_load while self._ready_state != 'complete' and perf_counter() < end_time: diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index ba0871f..5e21623 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -14,7 +14,7 @@ from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter from .._units.waiter import FrameWaiter -from ..errors import ContextLossError, ElementLossError, GetDocumentError, TabClosedError +from ..errors import ContextLossError, ElementLossError, GetDocumentError, PageClosedError class ChromiumFrame(ChromiumBase): @@ -103,14 +103,21 @@ class ChromiumFrame(ChromiumBase): if debug: print(f'{self._frame_id} reload 开始') + self._driver.stop() try: self._frame_ele = ChromiumElement(self._target_page, backend_id=self._backend_id) - except ElementLossError: + except (ElementLossError, PageClosedError): return - node = self._target_page.run_cdp('DOM.describeNode', - backendNodeId=self._frame_ele.ids.backend_id)['node'] - self._driver.stop() + end_time = perf_counter() + 2 + while perf_counter() < end_time: + node = self._target_page.run_cdp('DOM.describeNode', + backendNodeId=self._frame_ele.ids.backend_id)['node'] + if 'frameId' in node: + break + + else: + return if self._is_inner_frame(): self._is_diff_domain = False @@ -237,10 +244,6 @@ class ChromiumFrame(ChromiumBase): if self._debug: print(f'{self._frame_id}触发InspectorDetached') - try: - self._frame_ele.attrs - except (ElementLossError, TabClosedError): - self._driver.stop() self._reload() if self._debug: @@ -253,10 +256,6 @@ class ChromiumFrame(ChromiumBase): if self._debug: print(f'{self._frame_id}触发FrameDetached') - try: - self._frame_ele.attrs - except (ElementLossError, TabClosedError): - self._driver.stop() self._reload() if self._debug: diff --git a/DrissionPage/errors.py b/DrissionPage/errors.py index 589fb95..ba515cb 100644 --- a/DrissionPage/errors.py +++ b/DrissionPage/errors.py @@ -32,8 +32,8 @@ class CDPError(BaseError): _info = '方法调用错误。' -class TabClosedError(BaseError): - _info = '标签页已关闭。' +class PageClosedError(BaseError): + _info = '页面已关闭。' class ElementNotFoundError(BaseError): From a089bcbffc289737e33b35247eb99dedd37f1bf9 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 16 Nov 2023 15:11:04 +0800 Subject: [PATCH 086/182] =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E5=A2=9E=E5=8A=A0ste?= =?UTF-8?q?ates=E5=B1=9E=E6=80=A7=EF=BC=8C=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 页面对象的is_loading、ready_state、is_alive属性移到states属性中； 重构Frame的steates属性； page_load_strategy改为load_mode； ini文件experimental_options改为prefs； ChromiumOptinos增加ignore_certificate_errors() set_headless()、set_no_imgs()、set_no_js()、set_mute()数个设置去掉set_ --- DrissionPage/_configs/chromium_options.py | 85 ++++++-- DrissionPage/_configs/chromium_options.pyi | 18 +- DrissionPage/_configs/configs.ini | 7 +- DrissionPage/_elements/chromium_element.py | 2 +- DrissionPage/_elements/chromium_element.pyi | 2 +- DrissionPage/_pages/chromium_base.py | 187 ++++++++++-------- DrissionPage/_pages/chromium_base.pyi | 20 +- DrissionPage/_pages/chromium_frame.py | 126 ++++++------ DrissionPage/_pages/chromium_frame.pyi | 13 +- DrissionPage/_pages/chromium_page.py | 50 ++--- DrissionPage/_pages/chromium_tab.py | 16 +- DrissionPage/_units/ids.pyi | 2 +- DrissionPage/_units/rect.py | 5 + DrissionPage/_units/rect.pyi | 3 + DrissionPage/_units/scroller.py | 3 +- DrissionPage/_units/setter.py | 22 ++- DrissionPage/_units/setter.pyi | 11 +- .../_units/{element_states.py => states.py} | 55 +++++- .../_units/{element_states.pyi => states.pyi} | 29 ++- DrissionPage/_units/waiter.py | 2 +- DrissionPage/easy_set.py | 4 +- 21 files changed, 409 insertions(+), 253 deletions(-) rename DrissionPage/_units/{element_states.py => states.py} (66%) rename DrissionPage/_units/{element_states.pyi => states.pyi} (62%) diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index b35c8f9..cdcf8c8 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -31,11 +31,11 @@ class ChromiumOptions(object): self._download_path = om.paths.get('download_path', '') self._arguments = options.get('arguments', []) - self._binary_location = options.get('binary_location', '') + self._browser_path = options.get('browser_path', '') self._extensions = options.get('extensions', []) - self._prefs = options.get('experimental_options', {}).get('prefs', {}) + self._prefs = options.get('prefs', {}) self._debugger_address = options.get('debugger_address', None) - self._page_load_strategy = options.get('page_load_strategy', 'normal') + self._load_mode = options.get('load_mode', 'normal') self._proxy = om.proxies.get('http', None) self._system_user_path = options.get('system_user_path', False) self._existing_only = options.get('is_existing_only', False) @@ -64,14 +64,14 @@ class ChromiumOptions(object): return self.ini_path = None - self._binary_location = "chrome" + self._browser_path = "chrome" self._arguments = [] self._download_path = '' self._extensions = [] self._prefs = {} self._timeouts = {'implicit': 10, 'pageLoad': 30, 'script': 30} self._debugger_address = '127.0.0.1:9222' - self._page_load_strategy = 'normal' + self._load_mode = 'normal' self._proxy = None self._auto_port = False self._system_user_path = False @@ -85,7 +85,7 @@ class ChromiumOptions(object): @property def browser_path(self): """浏览器启动文件路径""" - return self._binary_location + return self._browser_path @property def user_data_path(self): @@ -98,9 +98,9 @@ class ChromiumOptions(object): return self._user @property - def page_load_strategy(self): + def load_mode(self): """返回页面加载策略，'normal', 'eager', 'none'""" - return self._page_load_strategy + return self._load_mode @property def timeouts(self): @@ -246,7 +246,7 @@ class ChromiumOptions(object): self._user = user return self - def set_headless(self, on_off=True): + def headless(self, on_off=True): """设置是否隐藏浏览器界面 :param on_off: 开或关 :return: 当前对象 @@ -254,7 +254,7 @@ class ChromiumOptions(object): on_off = 'new' if on_off else 'false' return self.set_argument('--headless', on_off) - def set_no_imgs(self, on_off=True): + def no_imgs(self, on_off=True): """设置是否加载图片 :param on_off: 开或关 :return: 当前对象 @@ -262,7 +262,7 @@ class ChromiumOptions(object): on_off = None if on_off else False return self.set_argument('--blink-settings=imagesEnabled=false', on_off) - def set_no_js(self, on_off=True): + def no_js(self, on_off=True): """设置是否禁用js :param on_off: 开或关 :return: 当前对象 @@ -270,7 +270,7 @@ class ChromiumOptions(object): on_off = None if on_off else False return self.set_argument('--disable-javascript', on_off) - def set_mute(self, on_off=True): + def mute(self, on_off=True): """设置是否静音 :param on_off: 开或关 :return: 当前对象 @@ -278,6 +278,14 @@ class ChromiumOptions(object): on_off = None if on_off else False return self.set_argument('--mute-audio', on_off) + def ignore_certificate_errors(self, on_off=True): + """设置是否忽略证书错误 + :param on_off: 开或关 + :return: 当前对象 + """ + on_off = None if on_off else False + return self.set_argument('--ignore-certificate-errors', on_off) + def set_user_agent(self, user_agent): """设置user agent :param user_agent: user agent文本 @@ -293,8 +301,8 @@ class ChromiumOptions(object): self._proxy = proxy return self.set_argument('--proxy-server', proxy) - def set_page_load_strategy(self, value): - """设置page_load_strategy，可接收 'normal', 'eager', 'none' + def set_load_mode(self, value): + """设置load_mode，可接收 'normal', 'eager', 'none' normal：默认情况下使用, 等待所有资源下载完成 eager：DOM访问已准备就绪, 但其他资源 (如图像) 可能仍在加载中 none：完全不阻塞 @@ -302,8 +310,8 @@ class ChromiumOptions(object): :return: 当前对象 """ if value not in ('normal', 'eager', 'none'): - raise ValueError("只能选择'normal', 'eager', 'none'。") - self._page_load_strategy = value.lower() + raise ValueError("只能选择 'normal', 'eager', 'none'。") + self._load_mode = value.lower() return self def set_paths(self, browser_path=None, local_port=None, debugger_address=None, download_path=None, @@ -360,7 +368,7 @@ class ChromiumOptions(object): :param path: 浏览器路径 :return: 当前对象 """ - self._binary_location = str(path) + self._browser_path = str(path) self._auto_port = False return self @@ -445,7 +453,7 @@ class ChromiumOptions(object): om = OptionsManager(self.ini_path or str(Path(__file__).parent / 'configs.ini')) # 设置chrome_options - attrs = ('debugger_address', 'binary_location', 'arguments', 'extensions', 'user', 'page_load_strategy', + attrs = ('debugger_address', 'browser_path', 'arguments', 'extensions', 'user', 'load_mode', 'auto_port', 'system_user_path', 'existing_only') for i in attrs: om.set_item('chrome_options', i, self.__getattribute__(f'_{i}')) @@ -459,9 +467,7 @@ class ChromiumOptions(object): om.set_item('timeouts', 'page_load', self._timeouts['pageLoad']) om.set_item('timeouts', 'script', self._timeouts['script']) # 设置prefs - eo = om.chrome_options.get('experimental_options', {}) - eo['prefs'] = self._prefs - om.set_item('chrome_options', 'experimental_options', eo) + om.set_item('chrome_options', 'prefs', self._prefs) path = str(path) om.save(path) @@ -472,6 +478,43 @@ class ChromiumOptions(object): """保存当前配置到默认ini文件""" return self.save('default') + # ---------------即将废弃-------------- + + def set_page_load_strategy(self, value): + return self.set_load_mode(value) + + def set_headless(self, on_off=True): + """设置是否隐藏浏览器界面 + :param on_off: 开或关 + :return: 当前对象 + """ + on_off = 'new' if on_off else 'false' + return self.set_argument('--headless', on_off) + + def set_no_imgs(self, on_off=True): + """设置是否加载图片 + :param on_off: 开或关 + :return: 当前对象 + """ + on_off = None if on_off else False + return self.set_argument('--blink-settings=imagesEnabled=false', on_off) + + def set_no_js(self, on_off=True): + """设置是否禁用js + :param on_off: 开或关 + :return: 当前对象 + """ + on_off = None if on_off else False + return self.set_argument('--disable-javascript', on_off) + + def set_mute(self, on_off=True): + """设置是否静音 + :param on_off: 开或关 + :return: 当前对象 + """ + on_off = None if on_off else False + return self.set_argument('--mute-audio', on_off) + class PortFinder(object): used_port = {} diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index b378421..e42c6de 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -14,9 +14,9 @@ class ChromiumOptions(object): self._user_data_path: str = ... self._download_path: str = ... self._arguments: list = ... - self._binary_location: str = ... + self._browser_path: str = ... self._user: str = ... - self._page_load_strategy: str = ... + self._load_mode: str = ... self._timeouts: dict = ... self._proxy: str = ... self._debugger_address: str = ... @@ -41,7 +41,7 @@ class ChromiumOptions(object): def user(self) -> str: ... @property - def page_load_strategy(self) -> str: ... + def load_mode(self) -> str: ... @property def timeouts(self) -> dict: ... @@ -86,19 +86,21 @@ class ChromiumOptions(object): def set_user(self, user: str = 'Default') -> ChromiumOptions: ... - def set_headless(self, on_off: bool = True) -> ChromiumOptions: ... + def headless(self, on_off: bool = True) -> ChromiumOptions: ... - def set_no_imgs(self, on_off: bool = True) -> ChromiumOptions: ... + def no_imgs(self, on_off: bool = True) -> ChromiumOptions: ... - def set_no_js(self, on_off: bool = True) -> ChromiumOptions: ... + def no_js(self, on_off: bool = True) -> ChromiumOptions: ... - def set_mute(self, on_off: bool = True) -> ChromiumOptions: ... + def mute(self, on_off: bool = True) -> ChromiumOptions: ... def set_user_agent(self, user_agent: str) -> ChromiumOptions: ... def set_proxy(self, proxy: str) -> ChromiumOptions: ... - def set_page_load_strategy(self, value: str) -> ChromiumOptions: ... + def ignore_certificate_errors(self, on_off=True) -> ChromiumOptions: ... + + def set_load_mode(self, value: str) -> ChromiumOptions: ... def set_browser_path(self, path: Union[str, Path]) -> ChromiumOptions: ... diff --git a/DrissionPage/_configs/configs.ini b/DrissionPage/_configs/configs.ini index 0190176..e6cb8f5 100644 --- a/DrissionPage/_configs/configs.ini +++ b/DrissionPage/_configs/configs.ini @@ -3,15 +3,16 @@ download_path = [chrome_options] debugger_address = 127.0.0.1:9222 -binary_location = chrome +browser_path = chrome arguments = ['--remote-allow-origins=*', '--no-first-run', '--disable-infobars', '--disable-popup-blocking'] extensions = [] -experimental_options = {'prefs': {'profile.default_content_settings.popups': 0, 'profile.default_content_setting_values': {'notifications': 2}}} -page_load_strategy = normal +prefs = {'profile.default_content_settings.popups': 0, 'profile.default_content_setting_values': {'notifications': 2}} +load_mode = normal user = Default auto_port = False system_user_path = False is_existing_only = False +existing_only = False [session_options] headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8', 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'connection': 'keep-alive', 'accept-charset': 'GB2312,utf-8;q=0.7,*;q=0.7'} diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index f9a2d4c..bf052c7 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -15,12 +15,12 @@ from .._commons.locator import get_loc from .._commons.tools import get_usable_path from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, offset_scroll from .._units.clicker import Clicker -from .._units.element_states import ElementStates, ShadowRootStates from .._units.ids import ShadowRootIds, ElementIds from .._units.locations import Locations from .._units.scroller import ElementScroller from .._units.select_element import SelectElement from .._units.setter import ChromiumElementSetter +from .._units.states import ElementStates, ShadowRootStates from .._units.waiter import ElementWaiter from ..errors import (ContextLossError, ElementLossError, JavaScriptError, ElementNotFoundError, CDPError, NoResourceError, AlertExistsError) diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 315461e..90f3caa 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -14,12 +14,12 @@ from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage from .._pages.web_page import WebPage from .._units.clicker import Clicker -from .._units.element_states import ShadowRootStates, ElementStates from .._units.ids import ElementIds, ShadowRootIds from .._units.locations import Locations from .._units.scroller import ElementScroller from .._units.select_element import SelectElement from .._units.setter import ChromiumElementSetter +from .._units.states import ShadowRootStates, ElementStates from .._units.waiter import ElementWaiter diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index ba10874..92c8c56 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -10,7 +10,6 @@ from re import findall from threading import Thread from time import perf_counter, sleep -from .._units.scroller import PageScroller from .._base.base import BasePage from .._commons.constants import ERROR, NoneElement from .._commons.locator import get_loc @@ -21,7 +20,9 @@ from .._elements.session_element import make_session_ele from .._units.action_chains import ActionChains from .._units.network_listener import NetworkListener from .._units.screencast import Screencast +from .._units.scroller import PageScroller from .._units.setter import ChromiumBaseSetter +from .._units.states import PageStates from .._units.waiter import BaseWaiter from ..errors import (ContextLossError, ElementLossError, CDPError, PageClosedError, NoRectError, AlertExistsError, GetDocumentError) @@ -44,6 +45,7 @@ class ChromiumBase(BasePage): self._screencast = None self._actions = None self._listener = None + self._states = None self._has_alert = False self._ready_state = None self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc @@ -69,7 +71,7 @@ class ChromiumBase(BasePage): def _d_set_runtime_settings(self): self._timeouts = Timeout(self) - self._page_load_strategy = 'normal' + self._load_mode = 'normal' def _connect_browser(self, tab_id=None): """连接浏览器，在第一次时运行 @@ -91,7 +93,7 @@ class ChromiumBase(BasePage): tab_id = tabs[0][0] self._driver_init(tab_id) - if self.ready_state == 'complete' and self._ready_state is None: + if self._js_ready_state == 'complete' and self._ready_state is None: self._get_document() self._ready_state = 'complete' if self._debug: @@ -170,7 +172,7 @@ class ChromiumBase(BasePage): self._doc_got = False self._ready_state = 'loading' self._is_loading = True - if self.page_load_strategy == 'eager': + if self._load_mode == 'eager': t = Thread(target=self._wait_to_stop) t.daemon = True t.start() @@ -199,7 +201,7 @@ class ChromiumBase(BasePage): print('在DomContentEventFired变成interactive') self._ready_state = 'interactive' - if self.page_load_strategy == 'eager': + if self._load_mode == 'eager': self.run_cdp('Page.stopLoading') if self._debug: @@ -261,9 +263,65 @@ class ChromiumBase(BasePage): if self._ready_state in ('interactive', 'complete') and self._is_loading: self.stop_loading() + # ----------挂件---------- + @property - def main(self): - return self._page + def wait(self): + """返回用于等待的对象""" + if self._wait is None: + self._wait = BaseWaiter(self) + return self._wait + + @property + def set(self): + """返回用于等待的对象""" + if self._set is None: + self._set = ChromiumBaseSetter(self) + return self._set + + @property + def screencast(self): + """返回用于录屏的对象""" + if self._screencast is None: + self._screencast = Screencast(self) + return self._screencast + + @property + def actions(self): + """返回用于执行动作链的对象""" + if self._actions is None: + self._actions = ActionChains(self) + self.wait.load_complete() + return self._actions + + @property + def listen(self): + """返回用于聆听数据包的对象""" + if self._listener is None: + self._listener = NetworkListener(self) + return self._listener + + @property + def states(self): + """返回用于获取状态信息的对象""" + if self._states is None: + self._states = PageStates(self) + return self._states + + @property + def scroll(self): + """返回用于滚动滚动条的对象""" + self.wait.load_complete() + if self._scroll is None: + self._scroll = PageScroller(self) + return self._scroll + + @property + def timeouts(self): + """返回timeouts设置""" + return self._timeouts + + # ----------挂件---------- @property def browser(self): @@ -276,20 +334,6 @@ class ChromiumBase(BasePage): raise RuntimeError('浏览器已关闭或链接已断开。') return self._driver - @property - def is_loading(self): - """返回页面是否正在加载状态""" - return self._is_loading - - @property - def is_alive(self): - """返回页面对象是否仍然可用""" - try: - self.run_cdp('Page.getLayoutMetrics') - return True - except PageClosedError: - return False - @property def title(self): """返回当前页面title""" @@ -329,16 +373,6 @@ class ChromiumBase(BasePage): """返回当前标签页id""" return self.driver.id if not self.driver._stopped.is_set() else '' - @property - def ready_state(self): - """返回当前页面加载状态，'loading' 'interactive' 'complete'，'timeout' 表示可能有弹出框""" - try: - return self.run_cdp('Runtime.evaluate', expression='document.readyState;', _timeout=3)['result']['value'] - except ContextLossError: - return None - except TimeoutError: - return 'timeout' - @property def size(self): """返回页面总宽高，格式：(宽, 高)""" @@ -351,74 +385,36 @@ class ChromiumBase(BasePage): return self.run_js_loaded('return document.activeElement;') @property - def page_load_strategy(self): + def load_mode(self): """返回页面加载策略，有3种：'none'、'normal'、'eager'""" - return self._page_load_strategy + return self._load_mode @property def user_agent(self): """返回user agent""" return self.run_cdp('Runtime.evaluate', expression='navigator.userAgent;')['result']['value'] - @property - def scroll(self): - """返回用于滚动滚动条的对象""" - self.wait.load_complete() - if self._scroll is None: - self._scroll = PageScroller(self) - return self._scroll - - @property - def timeouts(self): - """返回timeouts设置""" - return self._timeouts - @property def upload_list(self): """返回等待上传文件列表""" return self._upload_list - @property - def wait(self): - """返回用于等待的对象""" - if self._wait is None: - self._wait = BaseWaiter(self) - return self._wait - - @property - def set(self): - """返回用于等待的对象""" - if self._set is None: - self._set = ChromiumBaseSetter(self) - return self._set - - @property - def screencast(self): - """返回用于录屏的对象""" - if self._screencast is None: - self._screencast = Screencast(self) - return self._screencast - - @property - def actions(self): - """返回用于执行动作链的对象""" - if self._actions is None: - self._actions = ActionChains(self) - self.wait.load_complete() - return self._actions - - @property - def listen(self): - """返回用于聆听数据包的对象""" - if self._listener is None: - self._listener = NetworkListener(self) - return self._listener - @property def has_alert(self): """返回是否存在提示框""" return self._has_alert + @property + def _js_ready_state(self): + """返回js获取的ready state信息""" + try: + return self.run_cdp('Runtime.evaluate', expression='document.readyState;', + _timeout=3)['result']['value'] + except ContextLossError: + return None + except TimeoutError: + return 'timeout' + def run_cdp(self, cmd, **cmd_args): """执行Chrome DevTools Protocol语句 :param cmd: 协议项目 @@ -865,7 +861,7 @@ class ChromiumBase(BasePage): :param timeout: 超时时间 :return: 是否成功，超时返回False """ - if self.page_load_strategy == 'none': + if self._load_mode == 'none': return True timeout = timeout if timeout is not None else self.timeouts.page_load @@ -873,8 +869,8 @@ class ChromiumBase(BasePage): while perf_counter() < end_time: if self._ready_state == 'complete': return True - elif self.page_load_strategy == 'eager' and self._ready_state in ('interactive', - 'complete') and not self._is_loading: + elif self._load_mode == 'eager' and self._ready_state in ('interactive', + 'complete') and not self._is_loading: return True sleep(.1) @@ -913,7 +909,7 @@ class ChromiumBase(BasePage): self.stop_loading() continue - if self.page_load_strategy == 'none': + if self._load_mode == 'none': return True yu = end_time - perf_counter() @@ -1015,6 +1011,25 @@ class ChromiumBase(BasePage): f.write(png) return str(path.absolute()) + # --------------------即将废弃--------------------- + + @property + def page_load_strategy(self): + return self._load_mode + + @property + def is_alive(self): + return self.states.is_alive + + @property + def is_loading(self): + """返回页面是否正在加载状态""" + return self._is_loading + + @property + def ready_state(self): + return self._ready_state + class Timeout(object): """用于保存d模式timeout信息的类""" diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 8d3a1ca..41caec9 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -19,6 +19,7 @@ from .._units.network_listener import NetworkListener from .._units.screencast import Screencast from .._units.scroller import Scroller, PageScroller from .._units.setter import ChromiumBaseSetter +from .._units.states import PageStates from .._units.waiter import BaseWaiter @@ -37,7 +38,7 @@ class ChromiumBase(BasePage): self._timeouts: Timeout = ... self._first_run: bool = ... self._is_loading: bool = ... - self._page_load_strategy: str = ... + self._load_mode: str = ... self._scroll: Scroller = ... self._url: str = ... self._root_id: str = ... @@ -48,6 +49,7 @@ class ChromiumBase(BasePage): self._screencast: Screencast = ... self._actions: ActionChains = ... self._listener: NetworkListener = ... + self._states: PageStates = ... self._alert: Alert = ... self._has_alert: bool = ... self._doc_got: bool = ... @@ -87,7 +89,7 @@ class ChromiumBase(BasePage): timeout: float = None) -> ChromiumElement: ... @property - def main(self) -> ChromiumPage: ... + def _js_ready_state(self) -> str: ... @property def browser(self) -> Browser: ... @@ -98,12 +100,6 @@ class ChromiumBase(BasePage): @property def driver(self) -> ChromiumDriver: ... - @property - def is_loading(self) -> bool: ... - - @property - def is_alive(self) -> bool: ... - @property def url(self) -> str: ... @@ -122,9 +118,6 @@ class ChromiumBase(BasePage): @property def tab_id(self) -> str: ... - @property - def ready_state(self) -> Union[str, None]: ... - @property def size(self) -> Tuple[int, int]: ... @@ -132,7 +125,7 @@ class ChromiumBase(BasePage): def active_ele(self) -> ChromiumElement: ... @property - def page_load_strategy(self) -> str: ... + def load_mode(self) -> str: ... @property def user_agent(self) -> str: ... @@ -161,6 +154,9 @@ class ChromiumBase(BasePage): @property def listen(self) -> NetworkListener: ... + @property + def states(self) -> PageStates: ... + @property def has_alert(self) -> bool: ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 5e21623..c0e15d1 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -13,6 +13,7 @@ from .._units.ids import FrameIds from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter +from .._units.states import FrameStates from .._units.waiter import FrameWaiter from ..errors import ContextLossError, ElementLossError, GetDocumentError, PageClosedError @@ -57,6 +58,7 @@ class ChromiumFrame(ChromiumBase): end_time = perf_counter() + 5 while perf_counter() < end_time and self.url == 'about:blank': sleep(.1) + self._rect = None def __call__(self, loc_or_str, timeout=None): """在内部查找元素 @@ -78,7 +80,7 @@ class ChromiumFrame(ChromiumBase): self._timeouts = copy(self._target_page.timeouts) self.retry_times = self._target_page.retry_times self.retry_interval = self._target_page.retry_interval - self._page_load_strategy = self._target_page.page_load_strategy if not self._is_diff_domain else 'normal' + self._load_mode = self._target_page._load_mode if not self._is_diff_domain else 'normal' self._download_path = self._target_page.download_path def _driver_init(self, tab_id, is_init=True): @@ -261,6 +263,46 @@ class ChromiumFrame(ChromiumBase): if self._debug: print(f'{self._frame_id}执行FrameDetached完毕') + # ----------挂件---------- + + @property + def scroll(self): + """返回用于等待的对象""" + self.wait.load_complete() + if self._scroll is None: + self._scroll = FrameScroller(self) + return self._scroll + + @property + def set(self): + """返回用于等待的对象""" + if self._set is None: + self._set = ChromiumFrameSetter(self) + return self._set + + @property + def states(self): + """返回用于获取状态信息的对象""" + if self._states is None: + self._states = FrameStates(self) + return self._states + + @property + def wait(self): + """返回用于等待的对象""" + if self._wait is None: + self._wait = FrameWaiter(self) + return self._wait + + @property + def rect(self): + """返回获取坐标和大小的对象""" + if self._rect is None: + self._rect = FrameRect(self) + return self._rect + + # ----------挂件---------- + @property def page(self): return self._page @@ -326,11 +368,6 @@ class ChromiumFrame(ChromiumBase): """返回frame元素大小""" return self.frame_ele.size - @property - def rect(self): - """返回获取坐标和大小的对象""" - return FrameRect(self) - @property def active_ele(self): """返回当前焦点所在元素""" @@ -356,59 +393,6 @@ class ChromiumFrame(ChromiumBase): """返回frame的css selector绝对路径""" return self.frame_ele.css_path - @property - def ready_state(self): - """返回当前页面加载状态，'loading' 'interactive' 'complete'""" - if self._is_diff_domain: - try: - return super().ready_state - except: - return 'complete' - - else: - end_time = perf_counter() + 3 - while self.is_alive and perf_counter() < end_time: - try: - return self.doc_ele.run_js('return this.readyState;') - except ContextLossError: - try: - node = self.run_cdp('DOM.describeNode', backendNodeId=self.frame_ele.ids.backend_id)['node'] - doc = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) - return doc.run_js('return this.readyState;') - except: - pass - - sleep(.1) - - @property - def is_alive(self): - """返回是否仍可用""" - return self.states.is_alive - - @property - def scroll(self): - """返回用于等待的对象""" - return FrameScroller(self) - - @property - def set(self): - """返回用于等待的对象""" - if self._set is None: - self._set = ChromiumFrameSetter(self) - return self._set - - @property - def states(self): - """返回用于获取状态信息的对象""" - return self.frame_ele.states - - @property - def wait(self): - """返回用于等待的对象""" - if self._wait is None: - self._wait = FrameWaiter(self) - return self._wait - @property def tab_id(self): """返回frame所在tab的id""" @@ -418,6 +402,23 @@ class ChromiumFrame(ChromiumBase): def download_path(self): return self._download_path + @property + def _js_ready_state(self): + """返回当前页面加载状态，'loading' 'interactive' 'complete'""" + if self._is_diff_domain: + return super()._js_ready_state + + else: + try: + return self.doc_ele.run_js('return this.readyState;') + except ContextLossError: + try: + node = self.run_cdp('DOM.describeNode', backendNodeId=self.frame_ele.ids.backend_id)['node'] + doc = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) + return doc.run_js('return this.readyState;') + except: + return None + def refresh(self): """刷新frame页面""" self.doc_ele.run_js('this.location.reload();') @@ -644,3 +645,10 @@ class ChromiumFrame(ChromiumBase): def _is_inner_frame(self): """返回当前frame是否同域""" return self._frame_id in str(self._target_page.run_cdp('Page.getFrameTree')['frameTree']) + + # ----------------即将废弃----------------- + + @property + def is_alive(self): + """返回是否仍可用""" + return self.states.is_alive diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index facbb5f..cfaba9e 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -11,7 +11,7 @@ from .chromium_page import ChromiumPage from .chromium_tab import ChromiumTab from .web_page import WebPage from .._elements.chromium_element import ChromiumElement -from .._units.element_states import ElementStates +from .._units.states import FrameStates from .._units.ids import FrameIds from .._units.locations import Locations from .._units.rect import FrameRect @@ -32,9 +32,10 @@ class ChromiumFrame(ChromiumBase): self._doc_ele: ChromiumElement = ... self._is_diff_domain: bool = ... self.doc_ele: ChromiumElement = ... - self._states: ElementStates = ... + self._states: FrameStates = ... self._ids: FrameIds = ... self._is_init_get_doc: bool = ... + self._rect: FrameRect = ... def __call__(self, loc_or_str: Union[Tuple[str, str], str], @@ -112,12 +113,6 @@ class ChromiumFrame(ChromiumBase): @property def css_path(self) -> str: ... - @property - def ready_state(self) -> str: ... - - @property - def is_alive(self) -> bool: ... - @property def scroll(self) -> FrameScroller: ... @@ -125,7 +120,7 @@ class ChromiumFrame(ChromiumBase): def set(self) -> ChromiumFrameSetter: ... @property - def states(self) -> ElementStates: ... + def states(self) -> FrameStates: ... @property def wait(self) -> FrameWaiter: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 87c7c10..bd2cdd5 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -87,7 +87,7 @@ class ChromiumPage(ChromiumBase): implicit=self._driver_options.timeouts['implicit']) if self._driver_options.timeouts['implicit'] is not None: self._timeout = self._driver_options.timeouts['implicit'] - self._page_load_strategy = self._driver_options.page_load_strategy + self._load_mode = self._driver_options.load_mode self._download_path = str(Path(self._driver_options.download_path).absolute()) def _page_init(self): @@ -95,6 +95,32 @@ class ChromiumPage(ChromiumBase): self._rect = None self._browser.connect_to_page() + # ----------挂件---------- + + @property + def set(self): + """返回用于等待的对象""" + if self._set is None: + self._set = ChromiumPageSetter(self) + return self._set + + @property + def rect(self): + """返回保存窗口方位信息的对象""" + self.wait.load_complete() + if self._rect is None: + self._rect = TabRect(self) + return self._rect + + @property + def wait(self): + """返回用于等待的对象""" + if self._wait is None: + self._wait = PageWaiter(self) + return self._wait + + # ----------挂件---------- + @property def browser(self): """返回用于控制浏览器cdp的driver""" @@ -120,28 +146,6 @@ class ChromiumPage(ChromiumBase): """返回浏览器进程id""" return self.browser.process_id - @property - def set(self): - """返回用于等待的对象""" - if self._set is None: - self._set = ChromiumPageSetter(self) - return self._set - - @property - def rect(self): - """返回保存窗口方位信息的对象""" - self.wait.load_complete() - if self._rect is None: - self._rect = TabRect(self) - return self._rect - - @property - def wait(self): - """返回用于等待的对象""" - if self._wait is None: - self._wait = PageWaiter(self) - return self._wait - def get_tab(self, tab_id=None): """获取一个标签页对象 :param tab_id: 要获取的标签页id，为None时获取当前tab diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 66edc32..8f8cc7c 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -32,7 +32,7 @@ class ChromiumTab(ChromiumBase): self._timeouts = copy(self.page.timeouts) self.retry_times = self.page.retry_times self.retry_interval = self.page.retry_interval - self._page_load_strategy = self.page.page_load_strategy + self._load_mode = self.page._load_mode self._download_path = self.page.download_path def close(self): @@ -90,6 +90,13 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): elif self._mode == 's': return super().__call__(loc_or_str) + @property + def set(self): + """返回用于等待的对象""" + if self._set is None: + self._set = WebPageTabSetter(self) + return self._set + @property def url(self): """返回当前url""" @@ -175,13 +182,6 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): """ self.set.timeouts(implicit=second) - @property - def set(self): - """返回用于等待的对象""" - if self._set is None: - self._set = WebPageTabSetter(self) - return self._set - def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None, **kwargs): """跳转到一个url :param url: 目标url diff --git a/DrissionPage/_units/ids.pyi b/DrissionPage/_units/ids.pyi index 20d4271..907005d 100644 --- a/DrissionPage/_units/ids.pyi +++ b/DrissionPage/_units/ids.pyi @@ -5,8 +5,8 @@ """ from typing import Union -from .._pages.chromium_frame import ChromiumFrame from .._elements.chromium_element import ChromiumElement, ChromiumShadowRoot +from .._pages.chromium_frame import ChromiumFrame class ShadowRootIds(object): diff --git a/DrissionPage/_units/rect.py b/DrissionPage/_units/rect.py index 8ebf0fe..1b31ebb 100644 --- a/DrissionPage/_units/rect.py +++ b/DrissionPage/_units/rect.py @@ -92,6 +92,11 @@ class FrameRect(object): """返回页面总宽高，格式：(宽, 高)""" return self._frame.page_size + @property + def size(self): + """返回页面总宽高，格式：(宽, 高)""" + return self._frame.size + @property def viewport_size(self): """返回视口宽高，不包括滚动条，格式：(宽, 高)""" diff --git a/DrissionPage/_units/rect.pyi b/DrissionPage/_units/rect.pyi index 678b34f..4c507fb 100644 --- a/DrissionPage/_units/rect.pyi +++ b/DrissionPage/_units/rect.pyi @@ -51,6 +51,9 @@ class FrameRect(object): @property def viewport_location(self) -> Tuple[float, float]: ... + @property + def size(self) -> Tuple[float, float]: ... + @property def page_size(self) -> Tuple[float, float]: ... diff --git a/DrissionPage/_units/scroller.py b/DrissionPage/_units/scroller.py index 1771468..1a67c4c 100644 --- a/DrissionPage/_units/scroller.py +++ b/DrissionPage/_units/scroller.py @@ -157,9 +157,8 @@ class FrameScroller(PageScroller): """ :param frame: ChromiumFrame对象 """ - self._driver = frame.doc_ele + super().__init__(frame.doc_ele) self.t1 = self.t2 = 'this.documentElement' - self._wait_complete = False def to_see(self, loc_or_ele, center=None): """滚动页面直到元素可见 diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index 78b20c3..a1ee590 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -16,9 +16,9 @@ class ChromiumBaseSetter(object): self._page = page @property - def load_strategy(self): + def load_mode(self): """返回用于设置页面加载策略的对象""" - return PageLoadStrategy(self._page) + return LoadMode(self._page) @property def scroll(self): @@ -117,6 +117,13 @@ class ChromiumBaseSetter(object): self._page.run_cdp('Network.enable') self._page.run_cdp('Network.setExtraHTTPHeaders', headers=headers) + # --------------即将废弃--------------- + + @property + def load_strategy(self): + """返回用于设置页面加载策略的对象""" + return LoadMode(self._page) + class TabSetter(ChromiumBaseSetter): def __init__(self, page): @@ -432,11 +439,10 @@ class ChromiumFrameSetter(ChromiumBaseSetter): :param value: 属性值 :return: None """ - self._page._check_ok() self._page.frame_ele.set.attr(attr, value) -class PageLoadStrategy(object): +class LoadMode(object): """用于设置页面加载策略的类""" def __init__(self, page): @@ -452,19 +458,19 @@ class PageLoadStrategy(object): """ if value.lower() not in ('normal', 'eager', 'none'): raise ValueError("只能选择 'normal', 'eager', 'none'。") - self._page._page_load_strategy = value + self._page._load_mode = value def normal(self): """设置页面加载策略为normal""" - self._page._page_load_strategy = 'normal' + self._page._load_mode = 'normal' def eager(self): """设置页面加载策略为eager""" - self._page._page_load_strategy = 'eager' + self._page._load_mode = 'eager' def none(self): """设置页面加载策略为none""" - self._page._page_load_strategy = 'none' + self._page._load_mode = 'none' class PageScrollSetter(object): diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index e337d3b..cf2f73e 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -11,8 +11,9 @@ from requests.adapters import HTTPAdapter from requests.auth import HTTPBasicAuth from requests.cookies import RequestsCookieJar +from .scroller import PageScroller from .._elements.chromium_element import ChromiumElement -from .._pages.chromium_base import ChromiumBase, ChromiumPageScroll +from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage from .._pages.chromium_tab import ChromiumTab @@ -27,7 +28,7 @@ class ChromiumBaseSetter(object): self._page: ChromiumBase = ... @property - def load_strategy(self) -> PageLoadStrategy: ... + def load_mode(self) -> LoadMode: ... @property def scroll(self) -> PageScrollSetter: ... @@ -163,7 +164,7 @@ class ChromiumFrameSetter(ChromiumBaseSetter): def attr(self, attr: str, value: str) -> None: ... -class PageLoadStrategy(object): +class LoadMode(object): def __init__(self, page: ChromiumBase): self._page: ChromiumBase = ... @@ -177,8 +178,8 @@ class PageLoadStrategy(object): class PageScrollSetter(object): - def __init__(self, scroll: ChromiumPageScroll): - self._scroll: ChromiumPageScroll = ... + def __init__(self, scroll: PageScroller): + self._scroll: PageScroller = ... def wait_complete(self, on_off: bool = True): ... diff --git a/DrissionPage/_units/element_states.py b/DrissionPage/_units/states.py similarity index 66% rename from DrissionPage/_units/element_states.py rename to DrissionPage/_units/states.py index 8f3b524..2b7c79f 100644 --- a/DrissionPage/_units/element_states.py +++ b/DrissionPage/_units/states.py @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from .._commons.web import location_in_viewport -from ..errors import CDPError, NoRectError +from ..errors import CDPError, NoRectError, PageClosedError, ElementLossError class ElementStates(object): @@ -40,7 +40,7 @@ class ElementStates(object): def is_alive(self): """返回元素是否仍在DOM中""" try: - d = self._ele.attrs + self._ele.attrs return True except Exception: return False @@ -102,3 +102,54 @@ class ShadowRootStates(object): return True except Exception: return False + + +class PageStates(object): + """Page对象、Tab对象使用""" + + def __init__(self, page): + """ + :param page: ChromiumBase对象 + """ + self._page = page + + @property + def is_loading(self): + """返回页面是否在加载状态""" + return self._page._is_loading + + @property + def is_alive(self): + """返回页面对象是否仍然可用""" + try: + self._page.run_cdp('Page.getLayoutMetrics') + return True + except PageClosedError: + return False + + @property + def ready_state(self): + """返回当前页面加载状态，'loading' 'interactive' 'complete'""" + return self._page._ready_state + + +class FrameStates(object): + def __init__(self, frame): + """ + :param frame: ChromiumFrame对象 + """ + self._frame = frame + + @property + def is_alive(self): + """返回frame元素是否可用，且里面仍挂载有frame""" + try: + node = self._frame._target_page.run_cdp('DOM.describeNode', + backendNodeId=self._frame._frame_ele.ids.backend_id)['node'] + except (ElementLossError, PageClosedError): + return False + return 'frameId' in node + + @property + def ready_state(self): + return self._frame._ready_state diff --git a/DrissionPage/_units/element_states.pyi b/DrissionPage/_units/states.pyi similarity index 62% rename from DrissionPage/_units/element_states.pyi rename to DrissionPage/_units/states.pyi index 5c2485f..e3c303c 100644 --- a/DrissionPage/_units/element_states.pyi +++ b/DrissionPage/_units/states.pyi @@ -3,9 +3,11 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union, Tuple, List +from typing import Union, Tuple, List, Optional from .._elements.chromium_element import ChromiumShadowRoot, ChromiumElement +from .._pages.chromium_base import ChromiumBase +from .._pages.chromium_frame import ChromiumFrame class ElementStates(object): @@ -52,3 +54,28 @@ class ShadowRootStates(object): @property def is_alive(self) -> bool: ... + + +class PageStates(object): + def __init__(self, page: ChromiumBase): + self._page: ChromiumBase = ... + + @property + def is_loading(self) -> bool: ... + + @property + def is_alive(self) -> bool: ... + + @property + def ready_state(self) -> Optional[str]: ... + + +class FrameStates(object): + def __init__(self, frame: ChromiumFrame): + self._frame: ChromiumFrame = ... + + @property + def is_alive(self) -> bool: ... + + @property + def ready_state(self) -> str: ... diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 56de34e..0361c56 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -180,7 +180,7 @@ class BaseWaiter(object): timeout = self._driver.timeout end_time = perf_counter() + timeout while perf_counter() < end_time: - if self._driver.is_loading == start: + if self._driver._is_loading == start: return True sleep(gap) diff --git a/DrissionPage/easy_set.py b/DrissionPage/easy_set.py index c70cb68..2bf6df5 100644 --- a/DrissionPage/easy_set.py +++ b/DrissionPage/easy_set.py @@ -52,7 +52,7 @@ def set_paths(browser_path=None, return str(path) if path else '' if browser_path is not None: - om.set_item('chrome_options', 'binary_location', format_path(browser_path)) + om.set_item('chrome_options', 'browser_path', format_path(browser_path)) if local_port is not None: om.set_item('chrome_options', 'debugger_address', f'127.0.0.1:{local_port}') @@ -185,7 +185,7 @@ def get_chrome_path(ini_path=None, # -----------从ini文件中获取-------------- if ini_path and from_ini: try: - path = OptionsManager(ini_path).chrome_options['binary_location'] + path = OptionsManager(ini_path).chrome_options['browser_path'] except KeyError: path = None else: From 006bd07167dea17cda527f5e3c5fe7a6ca352326 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 16 Nov 2023 22:33:21 +0800 Subject: [PATCH 087/182] =?UTF-8?q?maximized()=E5=92=8Cminimized()?= =?UTF-8?q?=E6=94=B9=E6=88=90max()=E5=92=8Cmini()=EF=BC=9Bsize=E3=80=81loc?= =?UTF-8?q?ations=E7=AD=89=E5=90=88=E5=B9=B6=E5=88=B0rect=E5=B1=9E?= =?UTF-8?q?=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 3 +- DrissionPage/_base/browser.py | 5 +- DrissionPage/_elements/chromium_element.py | 43 +++--- DrissionPage/_elements/chromium_element.pyi | 12 +- DrissionPage/_pages/chromium_base.py | 23 +++- DrissionPage/_pages/chromium_base.pyi | 12 +- DrissionPage/_pages/chromium_frame.py | 46 +++---- DrissionPage/_pages/chromium_frame.pyi | 13 -- DrissionPage/_pages/chromium_page.py | 9 -- DrissionPage/_pages/chromium_page.pyi | 3 - DrissionPage/_pages/chromium_tab.py | 8 -- DrissionPage/_pages/chromium_tab.pyi | 3 - DrissionPage/_pages/web_page.py | 14 +- DrissionPage/_units/action_chains.py | 6 +- DrissionPage/_units/clicker.py | 16 +-- DrissionPage/_units/locations.py | 103 -------------- DrissionPage/_units/locations.pyi | 50 ------- DrissionPage/_units/rect.py | 143 ++++++++++++++++++-- DrissionPage/_units/rect.pyi | 67 ++++++++- DrissionPage/_units/states.py | 10 +- DrissionPage/_units/waiter.py | 8 +- 21 files changed, 297 insertions(+), 300 deletions(-) delete mode 100644 DrissionPage/_units/locations.py delete mode 100644 DrissionPage/_units/locations.pyi diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index ad912d6..5b6d740 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -12,4 +12,5 @@ from ._pages.web_page import WebPage from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions -__all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage'] +__all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] +__version__ = '4.0.0b8' diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 05d12c9..9b9ebdf 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -170,7 +170,10 @@ class Browser(object): end_time = perf_counter() + timeout while perf_counter() < end_time: p = popen(txt) - if f' {self.process_id} ' not in p.read(): + try: + if f' {self.process_id} ' not in p.read(): + return + except TypeError: return sleep(.2) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index bf052c7..d28e9e7 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -16,7 +16,7 @@ from .._commons.tools import get_usable_path from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, offset_scroll from .._units.clicker import Clicker from .._units.ids import ShadowRootIds, ElementIds -from .._units.locations import Locations +from .._units.rect import ElementRect from .._units.scroller import ElementScroller from .._units.select_element import SelectElement from .._units.setter import ChromiumElementSetter @@ -39,7 +39,7 @@ class ChromiumElement(DrissionElement): super().__init__(page) self._select = None self._scroll = None - self._locations = None + self._rect = None self._set = None self._states = None self._pseudo = None @@ -126,12 +126,6 @@ class ChromiumElement(DrissionElement): """返回获取内置id的对象""" return self._ids - @property - def size(self): - """返回元素宽和高组成的元组""" - border = self.page.run_cdp('DOM.getBoxModel', backendNodeId=self._backend_id)['model']['border'] - return border[2] - border[0], border[5] - border[1] - @property def set(self): """返回用于设置元素属性的对象""" @@ -154,16 +148,11 @@ class ChromiumElement(DrissionElement): return self._pseudo @property - def location(self): - """返回元素左上角的绝对坐标""" - return self.locations.location - - @property - def locations(self): + def rect(self): """返回用于获取元素位置的对象""" - if self._locations is None: - self._locations = Locations(self) - return self._locations + if self._rect is None: + self._rect = ElementRect(self) + return self._rect @property def shadow_root(self): @@ -565,8 +554,8 @@ class ChromiumElement(DrissionElement): if scroll_to_center: self.scroll.to_see(center=True) - left, top = self.location - width, height = self.size + left, top = self.rect.location + width, height = self.rect.size left_top = (left, top) right_bottom = (left + width, top + height) if not name: @@ -657,7 +646,7 @@ class ChromiumElement(DrissionElement): :param duration: 拖动用时，传入0即瞬间到j达 :return: None """ - curr_x, curr_y = self.locations.midpoint + curr_x, curr_y = self.rect.midpoint offset_x += curr_x offset_y += curr_y self.drag_to((offset_x, offset_y), duration) @@ -669,7 +658,7 @@ class ChromiumElement(DrissionElement): :return: None """ if isinstance(ele_or_loc, ChromiumElement): - ele_or_loc = ele_or_loc.locations.midpoint + ele_or_loc = ele_or_loc.rect.midpoint elif not isinstance(ele_or_loc, (list, tuple)): raise TypeError('需要ChromiumElement对象或坐标。') @@ -754,6 +743,18 @@ class ChromiumElement(DrissionElement): files = [str(Path(i).absolute()) for i in files] self.page.run_cdp('DOM.setFileInputFiles', files=files, backendNodeId=self._backend_id) + # -------------即将废弃------------- + + @property + def location(self): + """返回元素左上角的绝对坐标""" + return self.rect.location + + @property + def size(self): + """返回元素宽和高组成的元组""" + return self.rect.size + class ChromiumShadowRoot(BaseElement): """ChromiumShadowRoot是用于处理ShadowRoot的类，使用方法和ChromiumElement基本一致""" diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 90f3caa..7b261fa 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -15,7 +15,7 @@ from .._pages.chromium_page import ChromiumPage from .._pages.web_page import WebPage from .._units.clicker import Clicker from .._units.ids import ElementIds, ShadowRootIds -from .._units.locations import Locations +from .._units.rect import ElementRect from .._units.scroller import ElementScroller from .._units.select_element import SelectElement from .._units.setter import ChromiumElementSetter @@ -37,7 +37,7 @@ class ChromiumElement(DrissionElement): self._clicker: Clicker = ... self._select: SelectElement = ... self._wait: ElementWaiter = ... - self._locations: Locations = ... + self._rect: ElementRect = ... self._set: ChromiumElementSetter = ... self._states: ElementStates = ... self._pseudo: Pseudo = ... @@ -69,9 +69,6 @@ class ChromiumElement(DrissionElement): @property def ids(self) -> ElementIds: ... - @property - def size(self) -> Tuple[float, float]: ... - @property def set(self) -> ChromiumElementSetter: ... @@ -79,10 +76,7 @@ class ChromiumElement(DrissionElement): def states(self) -> ElementStates: ... @property - def location(self) -> Tuple[float, float]: ... - - @property - def locations(self) -> Locations: ... + def rect(self) -> ElementRect: ... @property def pseudo(self) -> Pseudo: ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 92c8c56..2158ce0 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -10,6 +10,7 @@ from re import findall from threading import Thread from time import perf_counter, sleep +from .._units.rect import TabRect from .._base.base import BasePage from .._commons.constants import ERROR, NoneElement from .._commons.locator import get_loc @@ -48,6 +49,7 @@ class ChromiumBase(BasePage): self._states = None self._has_alert = False self._ready_state = None + self._rect = None self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc self._download_path = str(Path('.').absolute()) @@ -316,6 +318,14 @@ class ChromiumBase(BasePage): self._scroll = PageScroller(self) return self._scroll + @property + def rect(self): + """返回获取窗口坐标和大小的对象""" + # self.wait.load_complete() + if self._rect is None: + self._rect = TabRect(self) + return self._rect + @property def timeouts(self): """返回timeouts设置""" @@ -373,12 +383,6 @@ class ChromiumBase(BasePage): """返回当前标签页id""" return self.driver.id if not self.driver._stopped.is_set() else '' - @property - def size(self): - """返回页面总宽高，格式：(宽, 高)""" - r = self.run_cdp_loaded('Page.getLayoutMetrics')['contentSize'] - return r['width'], r['height'] - @property def active_ele(self): """返回当前焦点所在元素""" @@ -973,7 +977,7 @@ class ChromiumBase(BasePage): pic_type = path.suffix.lower() pic_type = 'jpeg' if pic_type == '.jpg' else pic_type[1:] - width, height = self.size + width, height = self.rect.size if full_page: vp = {'x': 0, 'y': 0, 'width': width, 'height': height, 'scale': 1} png = self.run_cdp_loaded('Page.captureScreenshot', format=pic_type, @@ -1030,6 +1034,11 @@ class ChromiumBase(BasePage): def ready_state(self): return self._ready_state + @property + def size(self): + """返回页面总宽高，格式：(宽, 高)""" + return self.rect.size + class Timeout(object): """用于保存d模式timeout信息的类""" diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 41caec9..37f9225 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -6,6 +6,7 @@ from pathlib import Path from typing import Union, Tuple, List, Any, Optional +from .._units.rect import TabRect from .._base.base import BasePage from .._base.browser import Browser from .._base.chromium_driver import ChromiumDriver @@ -54,6 +55,7 @@ class ChromiumBase(BasePage): self._has_alert: bool = ... self._doc_got: bool = ... self._ready_state: Optional[str] = ... + self._rect: TabRect = ... def _connect_browser(self, tab_id: str = None) -> None: ... @@ -118,9 +120,6 @@ class ChromiumBase(BasePage): @property def tab_id(self) -> str: ... - @property - def size(self) -> Tuple[int, int]: ... - @property def active_ele(self) -> ChromiumElement: ... @@ -133,6 +132,9 @@ class ChromiumBase(BasePage): @property def scroll(self) -> PageScroller: ... + @property + def rect(self) -> TabRect: ... + @property def timeouts(self) -> Timeout: ... @@ -216,8 +218,8 @@ class ChromiumBase(BasePage): left_top: Tuple[int, int] = None, right_bottom: Tuple[int, int] = None) -> Union[str, bytes]: ... def _get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, - as_base64: [bool, str] = None, full_page: bool = False, left_top: Tuple[int, int] = None, - right_bottom: Tuple[int, int] = None, ele: ChromiumElement = None) -> Union[str, bytes]: ... + as_base64: [bool, str] = None, full_page: bool = False, left_top: Tuple[float, float] = None, + right_bottom: Tuple[float, float] = None, ele: ChromiumElement = None) -> Union[str, bytes]: ... def clear_cache(self, session_storage: bool = True, local_storage: bool = True, cache: bool = True, cookies: bool = True) -> None: ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index c0e15d1..7be33ba 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -356,33 +356,11 @@ class ChromiumFrame(ChromiumBase): """返回frame元素所有attribute属性""" return self.frame_ele.attrs - @property - def page_size(self): - """返回frame内页面尺寸，格式：(长, 高)""" - w = self.doc_ele.run_js('return this.body.scrollWidth') - h = self.doc_ele.run_js('return this.body.scrollHeight') - return w, h - - @property - def size(self): - """返回frame元素大小""" - return self.frame_ele.size - @property def active_ele(self): """返回当前焦点所在元素""" return self.doc_ele.run_js('return this.activeElement;') - @property - def location(self): - """返回frame元素左上角的绝对坐标""" - return self.frame_ele.location - - @property - def locations(self): - """返回用于获取元素位置的对象""" - return self.frame_ele.locations - @property def xpath(self): """返回frame的xpath绝对路径""" @@ -597,8 +575,8 @@ class ChromiumFrame(ChromiumBase): self.frame_ele.scroll.to_see(center=True) self.scroll.to_see(ele, center=True) - cx, cy = ele.locations.viewport_location - w, h = ele.size + cx, cy = ele.rect.viewport_location + w, h = ele.rect.size img_data = f'data:image/{pic_type};base64,{self.frame_ele.get_screenshot(as_base64=True)}' body = self.tab('t:body') first_child = body('c::first-child') @@ -652,3 +630,23 @@ class ChromiumFrame(ChromiumBase): def is_alive(self): """返回是否仍可用""" return self.states.is_alive + + @property + def page_size(self): + """返回frame内页面尺寸，格式：(宽,, 高)""" + return self.rect.size + + @property + def size(self): + """返回frame元素大小""" + return self.frame_ele.rect.size + + @property + def location(self): + """返回frame元素左上角的绝对坐标""" + return self.frame_ele.rect.location + + @property + def locations(self): + """返回用于获取元素位置的对象""" + return self.frame_ele.rect diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index cfaba9e..7c93fd0 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -13,7 +13,6 @@ from .web_page import WebPage from .._elements.chromium_element import ChromiumElement from .._units.states import FrameStates from .._units.ids import FrameIds -from .._units.locations import Locations from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter @@ -89,24 +88,12 @@ class ChromiumFrame(ChromiumBase): @property def attrs(self) -> dict: ... - @property - def page_size(self) -> Tuple[int, int]: ... - - @property - def size(self) -> Tuple[int, int]: ... - @property def rect(self) -> FrameRect: ... @property def active_ele(self) -> ChromiumElement: ... - @property - def location(self) -> Tuple[int, int]: ... - - @property - def locations(self) -> Locations: ... - @property def xpath(self) -> str: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index bd2cdd5..6a8b60c 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -13,7 +13,6 @@ from .._commons.browser import connect_browser from .._configs.chromium_options import ChromiumOptions from .._pages.chromium_base import ChromiumBase, Timeout from .._pages.chromium_tab import ChromiumTab -from .._units.rect import TabRect from .._units.setter import ChromiumPageSetter from .._units.waiter import PageWaiter from ..errors import BrowserConnectError @@ -104,14 +103,6 @@ class ChromiumPage(ChromiumBase): self._set = ChromiumPageSetter(self) return self._set - @property - def rect(self): - """返回保存窗口方位信息的对象""" - self.wait.load_complete() - if self._rect is None: - self._rect = TabRect(self) - return self._rect - @property def wait(self): """返回用于等待的对象""" diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index 62f7e95..c834405 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -39,9 +39,6 @@ class ChromiumPage(ChromiumBase): @property def tabs(self) -> List[str]: ... - @property - def rect(self) -> TabRect: ... - @property def wait(self) -> PageWaiter: ... diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 8f8cc7c..c67600a 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -9,7 +9,6 @@ from .._base.base import BasePage from .._commons.web import set_session_cookies, set_browser_cookies from .._pages.chromium_base import ChromiumBase from .._pages.session_page import SessionPage -from .._units.rect import TabRect from .._units.setter import TabSetter, WebPageTabSetter from .._units.waiter import TabWaiter @@ -44,13 +43,6 @@ class ChromiumTab(ChromiumBase): """返回总体page对象""" return self._page - @property - def rect(self): - """返回获取窗口坐标和大小的对象""" - if self._rect is None: - self._rect = TabRect(self) - return self._rect - @property def set(self): """返回用于等待的对象""" diff --git a/DrissionPage/_pages/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi index 5adb85f..687a88c 100644 --- a/DrissionPage/_pages/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -34,9 +34,6 @@ class ChromiumTab(ChromiumBase): @property def page(self) -> ChromiumPage: ... - @property - def rect(self) -> TabRect: ... - @property def set(self) -> TabSetter: ... diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 679745a..a873b3a 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -49,6 +49,13 @@ class WebPage(SessionPage, ChromiumPage, BasePage): elif self._mode == 's': return super().__call__(loc_or_str) + @property + def set(self): + """返回用于等待的对象""" + if self._set is None: + self._set = WebPageSetter(self) + return self._set + @property def url(self): """返回当前url""" @@ -134,13 +141,6 @@ class WebPage(SessionPage, ChromiumPage, BasePage): """ self.set.timeouts(implicit=second) - @property - def set(self): - """返回用于等待的对象""" - if self._set is None: - self._set = WebPageSetter(self) - return self._set - def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None, **kwargs): """跳转到一个url :param url: 目标url diff --git a/DrissionPage/_units/action_chains.py b/DrissionPage/_units/action_chains.py index 8b177d1..df74134 100644 --- a/DrissionPage/_units/action_chains.py +++ b/DrissionPage/_units/action_chains.py @@ -39,7 +39,7 @@ class ActionChains: elif isinstance(ele_or_loc, str) or 'ChromiumElement' in str(type(ele_or_loc)): ele_or_loc = self.page(ele_or_loc) self.page.scroll.to_see(ele_or_loc) - x, y = ele_or_loc.location if offset_x or offset_y else ele_or_loc.locations.midpoint + x, y = ele_or_loc.rect.location if offset_x or offset_y else ele_or_loc.rect.midpoint lx = x + offset_x ly = y + offset_y else: @@ -55,8 +55,8 @@ class ActionChains: if is_loc: cx, cy = location_to_client(self.page, lx, ly) else: - x, y = ele_or_loc.locations.viewport_location if offset_x or offset_y \ - else ele_or_loc.locations.viewport_midpoint + x, y = ele_or_loc.rect.viewport_location if offset_x or offset_y \ + else ele_or_loc.rect.viewport_midpoint cx = x + offset_x cy = y + offset_y diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index 0ab9234..aae2461 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -42,7 +42,7 @@ class Clicker(object): try: self._ele.scroll.to_see() if self._ele.states.is_enabled and self._ele.states.is_displayed: - rect = self._ele.locations.viewport_rect + rect = self._ele.rect.viewport_corners can_click = True except NoRectError: if by_js is False: @@ -59,7 +59,7 @@ class Clicker(object): self._ele.wait.stop_moving(timeout=end_time - perf_counter()) if rect: self._ele.scroll.to_see() - rect = self._ele.locations.rect + rect = self._ele.rect.corners while perf_counter() < end_time: if self._ele.states.is_enabled and self._ele.states.is_displayed: can_click = True @@ -79,12 +79,12 @@ class Clicker(object): r = self._ele.page.run_cdp('DOM.getNodeForLocation', x=x, y=y, includeUserAgentShadowDOM=True, ignorePointerEventsNone=True) if r['backendNodeId'] != self._ele.ids.backend_id: - vx, vy = self._ele.locations.viewport_midpoint + vx, vy = self._ele.rect.viewport_midpoint else: - vx, vy = self._ele.locations.viewport_click_point + vx, vy = self._ele.rect.viewport_click_point except CDPError: - vx, vy = self._ele.locations.viewport_midpoint + vx, vy = self._ele.rect.viewport_midpoint self._click(vx, vy) return True @@ -99,13 +99,13 @@ class Clicker(object): def right(self): """右键单击""" self._ele.page.scroll.to_see(self._ele) - x, y = self._ele.locations.viewport_click_point + x, y = self._ele.rect.viewport_click_point self._click(x, y, 'right') def middle(self): """中键单击""" self._ele.page.scroll.to_see(self._ele) - x, y = self._ele.locations.viewport_click_point + x, y = self._ele.rect.viewport_click_point self._click(x, y, 'middle') def at(self, offset_x=None, offset_y=None, button='left', count=1): @@ -118,7 +118,7 @@ class Clicker(object): """ self._ele.page.scroll.to_see(self._ele) if offset_x is None and offset_y is None: - w, h = self._ele.size + w, h = self._ele.rect.size offset_x = w // 2 offset_y = h // 2 x, y = offset_scroll(self._ele, offset_x, offset_y) diff --git a/DrissionPage/_units/locations.py b/DrissionPage/_units/locations.py deleted file mode 100644 index 010a962..0000000 --- a/DrissionPage/_units/locations.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding:utf-8 -*- -""" -@Author : g1879 -@Contact : g1879@qq.com -""" - - -class Locations(object): - def __init__(self, ele): - """ - :param ele: ChromiumElement - """ - self._ele = ele - - @property - def location(self): - """返回元素左上角的绝对坐标""" - cl = self.viewport_location - return self._get_page_coord(cl[0], cl[1]) - - @property - def midpoint(self): - """返回元素中间点的绝对坐标""" - cl = self.viewport_midpoint - return self._get_page_coord(cl[0], cl[1]) - - @property - def click_point(self): - """返回元素接受点击的点的绝对坐标""" - cl = self.viewport_click_point - return self._get_page_coord(cl[0], cl[1]) - - @property - def viewport_location(self): - """返回元素左上角在视口中的坐标""" - m = self._get_viewport_rect('border') - return m[0], m[1] - - @property - def viewport_midpoint(self): - """返回元素中间点在视口中的坐标""" - m = self._get_viewport_rect('border') - return m[0] + (m[2] - m[0]) // 2, m[3] + (m[5] - m[3]) // 2 - - @property - def viewport_click_point(self): - """返回元素接受点击的点视口坐标""" - m = self._get_viewport_rect('padding') - return self.viewport_midpoint[0], m[1] + 3 - - @property - def screen_location(self): - """返回元素左上角在屏幕上坐标，左上角为(0, 0)""" - vx, vy = self._ele.page.rect.viewport_location - ex, ey = self.viewport_location - pr = self._ele.page.run_js('return window.devicePixelRatio;') - return (vx + ex) * pr, (ey + vy) * pr - - @property - def screen_midpoint(self): - """返回元素中点在屏幕上坐标，左上角为(0, 0)""" - vx, vy = self._ele.page.rect.viewport_location - ex, ey = self.viewport_midpoint - pr = self._ele.page.run_js('return window.devicePixelRatio;') - return (vx + ex) * pr, (ey + vy) * pr - - @property - def screen_click_point(self): - """返回元素中点在屏幕上坐标，左上角为(0, 0)""" - vx, vy = self._ele.page.rect.viewport_location - ex, ey = self.viewport_click_point - pr = self._ele.page.run_js('return window.devicePixelRatio;') - return (vx + ex) * pr, (ey + vy) * pr - - @property - def rect(self): - """返回元素四个角坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" - vr = self._get_viewport_rect('border') - r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] - sx = r['pageX'] - sy = r['pageY'] - return [(vr[0] + sx, vr[1] + sy), (vr[2] + sx, vr[3] + sy), - (vr[4] + sx, vr[5] + sy), (vr[6] + sx, vr[7] + sy)] - - @property - def viewport_rect(self): - """返回元素四个角视口坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" - r = self._get_viewport_rect('border') - return [(r[0], r[1]), (r[2], r[3]), (r[4], r[5]), (r[6], r[7])] - - def _get_viewport_rect(self, quad): - """按照类型返回在可视窗口中的范围 - :param quad: 方框类型，margin border padding - :return: 四个角坐标 - """ - return self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele.ids.backend_id)['model'][quad] - - def _get_page_coord(self, x, y): - """根据视口坐标获取绝对坐标""" - r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] - sx = r['pageX'] - sy = r['pageY'] - return x + sx, y + sy diff --git a/DrissionPage/_units/locations.pyi b/DrissionPage/_units/locations.pyi deleted file mode 100644 index 5e7ca31..0000000 --- a/DrissionPage/_units/locations.pyi +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding:utf-8 -*- -""" -@Author : g1879 -@Contact : g1879@qq.com -""" -from typing import Tuple, Union, List - -from .._elements.chromium_element import ChromiumElement - - -class Locations(object): - def __init__(self, ele: ChromiumElement): - self._ele: ChromiumElement = ... - - @property - def location(self) -> Tuple[float, float]: ... - - @property - def midpoint(self) -> Tuple[float, float]: ... - - @property - def click_point(self) -> Tuple[float, float]: ... - - @property - def viewport_location(self) -> Tuple[float, float]: ... - - @property - def viewport_midpoint(self) -> Tuple[float, float]: ... - - @property - def viewport_click_point(self) -> Tuple[float, float]: ... - - @property - def screen_location(self) -> Tuple[float, float]: ... - - @property - def screen_midpoint(self) -> Tuple[float, float]: ... - - @property - def screen_click_point(self) -> Tuple[float, float]: ... - - @property - def rect(self) -> List[Tuple[float, float], ...]: ... - - @property - def viewport_rect(self) -> List[Tuple[float, float], ...]: ... - - def _get_viewport_rect(self, quad: str) -> Union[list, None]: ... - - def _get_page_coord(self, x: float, y: float) -> Tuple[float, float]: ... diff --git a/DrissionPage/_units/rect.py b/DrissionPage/_units/rect.py index 1b31ebb..4180b14 100644 --- a/DrissionPage/_units/rect.py +++ b/DrissionPage/_units/rect.py @@ -5,6 +5,110 @@ """ +class ElementRect(object): + def __init__(self, ele): + """ + :param ele: ChromiumElement + """ + self._ele = ele + + @property + def corners(self): + """返回元素四个角坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" + vr = self._get_viewport_rect('border') + r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] + sx = r['pageX'] + sy = r['pageY'] + return [(vr[0] + sx, vr[1] + sy), (vr[2] + sx, vr[3] + sy), + (vr[4] + sx, vr[5] + sy), (vr[6] + sx, vr[7] + sy)] + + @property + def viewport_corners(self): + """返回元素四个角视口坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" + r = self._get_viewport_rect('border') + return [(r[0], r[1]), (r[2], r[3]), (r[4], r[5]), (r[6], r[7])] + + @property + def size(self): + """返回元素大小，格式(宽, 高)""" + border = self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele._backend_id)['model']['border'] + return border[2] - border[0], border[5] - border[1] + + @property + def location(self): + """返回元素左上角的绝对坐标""" + cl = self.viewport_location + return self._get_page_coord(cl[0], cl[1]) + + @property + def midpoint(self): + """返回元素中间点的绝对坐标""" + cl = self.viewport_midpoint + return self._get_page_coord(cl[0], cl[1]) + + @property + def click_point(self): + """返回元素接受点击的点的绝对坐标""" + cl = self.viewport_click_point + return self._get_page_coord(cl[0], cl[1]) + + @property + def viewport_location(self): + """返回元素左上角在视口中的坐标""" + m = self._get_viewport_rect('border') + return m[0], m[1] + + @property + def viewport_midpoint(self): + """返回元素中间点在视口中的坐标""" + m = self._get_viewport_rect('border') + return m[0] + (m[2] - m[0]) // 2, m[3] + (m[5] - m[3]) // 2 + + @property + def viewport_click_point(self): + """返回元素接受点击的点视口坐标""" + m = self._get_viewport_rect('padding') + return self.viewport_midpoint[0], m[1] + 3 + + @property + def screen_location(self): + """返回元素左上角在屏幕上坐标，左上角为(0, 0)""" + vx, vy = self._ele.page.rect.viewport_location + ex, ey = self.viewport_location + pr = self._ele.page.run_js('return window.devicePixelRatio;') + return (vx + ex) * pr, (ey + vy) * pr + + @property + def screen_midpoint(self): + """返回元素中点在屏幕上坐标，左上角为(0, 0)""" + vx, vy = self._ele.page.rect.viewport_location + ex, ey = self.viewport_midpoint + pr = self._ele.page.run_js('return window.devicePixelRatio;') + return (vx + ex) * pr, (ey + vy) * pr + + @property + def screen_click_point(self): + """返回元素中点在屏幕上坐标，左上角为(0, 0)""" + vx, vy = self._ele.page.rect.viewport_location + ex, ey = self.viewport_click_point + pr = self._ele.page.run_js('return window.devicePixelRatio;') + return (vx + ex) * pr, (ey + vy) * pr + + def _get_viewport_rect(self, quad): + """按照类型返回在可视窗口中的范围 + :param quad: 方框类型，margin border padding + :return: 四个角坐标 + """ + return self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele.ids.backend_id)['model'][quad] + + def _get_page_coord(self, x, y): + """根据视口坐标获取绝对坐标""" + r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] + sx = r['pageX'] + sy = r['pageY'] + return x + sx, y + sy + + class TabRect(object): def __init__(self, page): self._page = page @@ -49,7 +153,7 @@ class TabRect(object): return w_bl + w_bs - w_vs, h_bl + h_bs - h_vs @property - def page_size(self): + def size(self): """返回页面总宽高，格式：(宽, 高)""" r = self._get_page_rect()['contentSize'] return r['width'], r['height'] @@ -83,21 +187,38 @@ class FrameRect(object): self._frame = frame @property - def viewport_location(self): - """返回视口在屏幕中坐标，左上角为(0, 0)""" - return self._frame.frame_ele.locations.screen_location + def location(self): + """返回元素左上角的绝对坐标""" + return self._frame.frame_ele.rect.location @property - def page_size(self): - """返回页面总宽高，格式：(宽, 高)""" - return self._frame.page_size + def viewport_location(self): + """返回视口在屏幕中坐标，左上角为(0, 0)""" + return self._frame.frame_ele.rect.viewport_location + + @property + def screen_location(self): + """返回元素左上角在屏幕上坐标，左上角为(0, 0)""" + return self._frame.frame_ele.rect.screen_location @property def size(self): - """返回页面总宽高，格式：(宽, 高)""" - return self._frame.size + """返回frame内页面尺寸，格式：(宽, 高)""" + w = self._frame.doc_ele.run_js('return this.body.scrollWidth') + h = self._frame.doc_ele.run_js('return this.body.scrollHeight') + return w, h @property def viewport_size(self): - """返回视口宽高，不包括滚动条，格式：(宽, 高)""" - return self._frame.frame_ele.size + """返回视口宽高，格式：(宽, 高)""" + return self._frame.frame_ele.rect.size + + @property + def corners(self): + """返回元素四个角坐标，顺序：坐上、右上、右下、左下""" + return self._frame.frame_ele.rect.corners + + @property + def viewport_corners(self): + """返回元素四个角视口坐标，顺序：坐上、右上、右下、左下""" + return self._frame.frame_ele.rect.viewport_corners diff --git a/DrissionPage/_units/rect.pyi b/DrissionPage/_units/rect.pyi index 4c507fb..b45ef98 100644 --- a/DrissionPage/_units/rect.pyi +++ b/DrissionPage/_units/rect.pyi @@ -3,16 +3,64 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Tuple, Union +from typing import Tuple, Union, List + +from .._elements.chromium_element import ChromiumElement +from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage from .._pages.chromium_tab import ChromiumTab, WebPageTab from .._pages.web_page import WebPage +class ElementRect(object): + def __init__(self, ele: ChromiumElement): + self._ele: ChromiumElement = ... + + @property + def size(self) -> Tuple[float, float]: ... + + @property + def location(self) -> Tuple[float, float]: ... + + @property + def midpoint(self) -> Tuple[float, float]: ... + + @property + def click_point(self) -> Tuple[float, float]: ... + + @property + def viewport_location(self) -> Tuple[float, float]: ... + + @property + def viewport_midpoint(self) -> Tuple[float, float]: ... + + @property + def viewport_click_point(self) -> Tuple[float, float]: ... + + @property + def screen_location(self) -> Tuple[float, float]: ... + + @property + def screen_midpoint(self) -> Tuple[float, float]: ... + + @property + def screen_click_point(self) -> Tuple[float, float]: ... + + @property + def corners(self) -> List[Tuple[float, float], ...]: ... + + @property + def viewport_corners(self) -> List[Tuple[float, float], ...]: ... + + def _get_viewport_rect(self, quad: str) -> Union[list, None]: ... + + def _get_page_coord(self, x: float, y: float) -> Tuple[float, float]: ... + + class TabRect(object): - def __init__(self, page: Union[ChromiumPage, ChromiumTab, WebPage, WebPageTab]): + def __init__(self, page: ChromiumBase): self._page: Union[ChromiumPage, ChromiumTab, WebPage, WebPageTab] = ... @property @@ -31,7 +79,7 @@ class TabRect(object): def window_size(self) -> Tuple[int, int]: ... @property - def page_size(self) -> Tuple[int, int]: ... + def size(self) -> Tuple[int, int]: ... @property def viewport_size(self) -> Tuple[int, int]: ... @@ -48,14 +96,23 @@ class FrameRect(object): def __init__(self, frame: ChromiumFrame): self._frame: ChromiumFrame = ... + @property + def location(self) -> Tuple[float, float]: ... + @property def viewport_location(self) -> Tuple[float, float]: ... + @property + def screen_location(self) -> Tuple[float, float]: ... + @property def size(self) -> Tuple[float, float]: ... @property - def page_size(self) -> Tuple[float, float]: ... + def viewport_size(self) -> Tuple[float, float]: ... @property - def viewport_size(self) -> Tuple[float, float]: ... + def corners(self) -> List[Tuple[float, float], ...]: ... + + @property + def viewport_corners(self) -> List[Tuple[float, float], ...]: ... diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index 2b7c79f..0bf571b 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -48,21 +48,21 @@ class ElementStates(object): @property def is_in_viewport(self): """返回元素是否出现在视口中，以元素click_point为判断""" - x, y = self._ele.locations.click_point + x, y = self._ele.rect.click_point return location_in_viewport(self._ele.page, x, y) if x else False @property def is_whole_in_viewport(self): """返回元素是否整个都在视口内""" - x1, y1 = self._ele.location - w, h = self._ele.size + x1, y1 = self._ele.rect.location + w, h = self._ele.rect.size x2, y2 = x1 + w, y1 + h return location_in_viewport(self._ele.page, x1, y1) and location_in_viewport(self._ele.page, x2, y2) @property def is_covered(self): """返回元素是否被覆盖，与是否在视口中无关""" - lx, ly = self._ele.locations.click_point + lx, ly = self._ele.rect.click_point try: r = self._ele.page.run_cdp('DOM.getNodeForLocation', x=lx, y=ly) except CDPError: @@ -77,7 +77,7 @@ class ElementStates(object): def has_rect(self): """返回元素是否拥有位置和大小，没有返回False，有返回四个角在页面中坐标组成的列表""" try: - return self._ele.locations.rect + return self._ele.rect.corners except NoRectError: return False diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 0361c56..cb265d4 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -376,7 +376,7 @@ class ElementWaiter(object): while perf_counter() < end_time: try: size = self._ele.states.has_rect - location = self._ele.location + location = self._ele.rect.location break except NoRectError: pass @@ -385,10 +385,10 @@ class ElementWaiter(object): while perf_counter() < end_time: sleep(gap) - if self._ele.size == size and location == self._ele.location: + if self._ele.rect.size == size and location == self._ele.rect.location: return True - size = self._ele.size - location = self._ele.location + size = self._ele.rect.size + location = self._ele.rect.location if raise_err is True or Settings.raise_when_wait_failed is True: raise WaitTimeoutError('等待元素停止运动失败。') From 7cfac54349503de76ffe57da4c736f8113d4cd19 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 16 Nov 2023 23:18:34 +0800 Subject: [PATCH 088/182] =?UTF-8?q?maximized()=E5=92=8Cminimized()?= =?UTF-8?q?=E6=94=B9=E6=88=90max()=E5=92=8Cmini()=EF=BC=9B=E5=88=A0?= =?UTF-8?q?=E9=99=A4ids=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 26 +++------ DrissionPage/_elements/chromium_element.pyi | 8 --- DrissionPage/_elements/session_element.py | 4 +- DrissionPage/_pages/chromium_base.py | 8 +-- DrissionPage/_pages/chromium_frame.py | 28 ++++++---- DrissionPage/_pages/chromium_frame.pyi | 62 ++++++++------------- DrissionPage/_units/clicker.py | 2 +- DrissionPage/_units/ids.py | 57 ------------------- DrissionPage/_units/ids.pyi | 45 --------------- DrissionPage/_units/rect.py | 2 +- DrissionPage/_units/setter.py | 16 +++++- DrissionPage/_units/setter.pyi | 4 +- DrissionPage/_units/states.py | 6 +- 13 files changed, 72 insertions(+), 196 deletions(-) delete mode 100644 DrissionPage/_units/ids.py delete mode 100644 DrissionPage/_units/ids.pyi diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index d28e9e7..e70bb8d 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -15,7 +15,6 @@ from .._commons.locator import get_loc from .._commons.tools import get_usable_path from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, offset_scroll from .._units.clicker import Clicker -from .._units.ids import ShadowRootIds, ElementIds from .._units.rect import ElementRect from .._units.scroller import ElementScroller from .._units.select_element import SelectElement @@ -66,7 +65,6 @@ class ChromiumElement(DrissionElement): else: raise ElementLossError - self._ids = ElementIds(self) doc = self.run_js('return this.ownerDocument;') self._doc_id = doc['objectId'] if doc else None @@ -121,10 +119,6 @@ class ChromiumElement(DrissionElement): return self.prop('innerText') # -----------------d模式独有属性------------------- - @property - def ids(self): - """返回获取内置id的对象""" - return self._ids @property def set(self): @@ -775,7 +769,6 @@ class ChromiumShadowRoot(BaseElement): self._obj_id = obj_id self._node_id = self._get_node_id(obj_id) self._backend_id = self._get_backend_id(self._node_id) - self._ids = ShadowRootIds(self) self._states = None def __repr__(self): @@ -805,11 +798,6 @@ class ChromiumShadowRoot(BaseElement): """返回内部的html文本""" return self.run_js('return this.innerHTML;') - @property - def ids(self): - """返回获取内置id的对象""" - return self._ids - @property def states(self): """返回用于获取元素状态的对象""" @@ -1104,7 +1092,7 @@ def find_by_xpath(ele, xpath, single, timeout, relative=True): type_txt = '9' if single else '7' node_txt = 'this.contentDocument' if ele.tag in FRAME_ELEMENT and not relative else 'this' js = make_js_for_find_ele_by_xpath(xpath, type_txt, node_txt) - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele.ids.obj_id, + r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, returnByValue=False, awaitPromise=True, userGesture=True) if r['result']['type'] == 'string': return r['result']['value'] @@ -1112,7 +1100,7 @@ def find_by_xpath(ele, xpath, single, timeout, relative=True): if 'exceptionDetails' in r: if 'The result is not a node set' in r['result']['description']: js = make_js_for_find_ele_by_xpath(xpath, '1', node_txt) - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele.ids.obj_id, + r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, returnByValue=False, awaitPromise=True, userGesture=True) return r['result']['value'] else: @@ -1121,7 +1109,7 @@ def find_by_xpath(ele, xpath, single, timeout, relative=True): end_time = perf_counter() + timeout while (r['result']['subtype'] == 'null' or r['result']['description'] == 'NodeList(0)') and perf_counter() < end_time: - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele.ids.obj_id, + r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, returnByValue=False, awaitPromise=True, userGesture=True) if single: @@ -1150,13 +1138,13 @@ def find_by_css(ele, selector, single, timeout): find_all = '' if single else 'All' node_txt = 'this.contentDocument' if ele.tag in ('iframe', 'frame', 'shadow-root') else 'this' js = f'function(){{return {node_txt}.querySelector{find_all}("{selector}");}}' - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele.ids.obj_id, + r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, returnByValue=False, awaitPromise=True, userGesture=True) end_time = perf_counter() + timeout while ('exceptionDetails' in r or r['result']['subtype'] == 'null' or r['result']['description'] == 'NodeList(0)') and perf_counter() < end_time: - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele.ids.obj_id, + r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, returnByValue=False, awaitPromise=True, userGesture=True) if 'exceptionDetails' in r: @@ -1259,7 +1247,7 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): """ if isinstance(page_or_ele, (ChromiumElement, ChromiumShadowRoot)): page = page_or_ele.page - obj_id = page_or_ele.ids.obj_id + obj_id = page_or_ele._obj_id is_page = False else: page = page_or_ele @@ -1345,7 +1333,7 @@ def parse_js_result(page, ele, result): def convert_argument(arg): """把参数转换成js能够接收的形式""" if isinstance(arg, ChromiumElement): - return {'objectId': arg.ids.obj_id} + return {'objectId': arg._obj_id} elif isinstance(arg, (int, float, str, bool)): return {'value': arg} diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 7b261fa..04bd049 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -14,7 +14,6 @@ from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage from .._pages.web_page import WebPage from .._units.clicker import Clicker -from .._units.ids import ElementIds, ShadowRootIds from .._units.rect import ElementRect from .._units.scroller import ElementScroller from .._units.select_element import SelectElement @@ -32,7 +31,6 @@ class ChromiumElement(DrissionElement): self._obj_id: str = ... self._backend_id: str = ... self._doc_id: str = ... - self._ids: ElementIds = ... self._scroll: ElementScroller = ... self._clicker: Clicker = ... self._select: SelectElement = ... @@ -66,8 +64,6 @@ class ChromiumElement(DrissionElement): def raw_text(self) -> str: ... # -----------------d模式独有属性------------------- - @property - def ids(self) -> ElementIds: ... @property def set(self) -> ChromiumElementSetter: ... @@ -211,7 +207,6 @@ class ChromiumShadowRoot(BaseElement): def __init__(self, parent_ele: ChromiumElement, obj_id: str = None, backend_id: str = None): self._obj_id: str = ... - self._ids: ShadowRootIds = ... self._node_id: str = ... self._backend_id: str = ... self.page: ChromiumPage = ... @@ -223,9 +218,6 @@ class ChromiumShadowRoot(BaseElement): def __call__(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> ChromiumElement: ... - @property - def ids(self) -> ShadowRootIds: ... - @property def states(self) -> ShadowRootStates: ... diff --git a/DrissionPage/_elements/session_element.py b/DrissionPage/_elements/session_element.py index 5fd09f4..a3d74f5 100644 --- a/DrissionPage/_elements/session_element.py +++ b/DrissionPage/_elements/session_element.py @@ -334,8 +334,8 @@ def make_session_ele(html_or_ele, loc=None, single=True): page = html_or_ele.page xpath = html_or_ele.xpath # ChromiumElement，兼容传入的元素在iframe内的情况 - html = html_or_ele.page.run_cdp('DOM.getOuterHTML', objectId=html_or_ele.ids.doc_id)['outerHTML'] \ - if html_or_ele.ids.doc_id else html_or_ele.page.html + html = html_or_ele.page.run_cdp('DOM.getOuterHTML', objectId=html_or_ele._doc_id)['outerHTML'] \ + if html_or_ele._doc_id else html_or_ele.page.html html_or_ele = fromstring(html) html_or_ele = html_or_ele.xpath(xpath)[0] diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 2158ce0..d60e0b2 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -50,6 +50,9 @@ class ChromiumBase(BasePage): self._has_alert = False self._ready_state = None self._rect = None + self._wait = None + self._scroll = None + self._upload_list = None self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc self._download_path = str(Path('.').absolute()) @@ -81,9 +84,6 @@ class ChromiumBase(BasePage): :return: None """ self._is_reading = False - self._upload_list = None - self._wait = None - self._scroll = None if not tab_id: tabs = self.browser.driver.get(f'http://{self.address}/json').json() @@ -697,7 +697,7 @@ class ChromiumBase(BasePage): return ele = self._ele(loc_or_ele, raise_err=False) if ele: - self.run_cdp('DOM.removeNode', nodeId=ele.ids.node_id) + self.run_cdp('DOM.removeNode', nodeId=ele._node_id) def get_frame(self, loc_ind_ele, timeout=None): """获取页面中一个frame对象，可传入定位符、iframe序号、ChromiumFrame对象，序号从1开始 diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 7be33ba..ec9cea9 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -9,7 +9,6 @@ from time import sleep, perf_counter from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase -from .._units.ids import FrameIds from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter @@ -35,12 +34,11 @@ class ChromiumFrame(ChromiumBase): self.tab = page.tab if 'ChromiumFrame' in page_type else page self.address = page.address - node = page.run_cdp('DOM.describeNode', backendNodeId=ele.ids.backend_id)['node'] + node = page.run_cdp('DOM.describeNode', backendNodeId=ele._backend_id)['node'] self._tab_id = page.tab_id - self._backend_id = ele.ids.backend_id + self._backend_id = ele._backend_id self._frame_ele = ele self._states = None - self._ids = FrameIds(self) self._is_init_get_doc = True self._frame_id = node['frameId'] @@ -114,7 +112,7 @@ class ChromiumFrame(ChromiumBase): end_time = perf_counter() + 2 while perf_counter() < end_time: node = self._target_page.run_cdp('DOM.describeNode', - backendNodeId=self._frame_ele.ids.backend_id)['node'] + backendNodeId=self._frame_ele._backend_id)['node'] if 'frameId' in node: break @@ -189,7 +187,7 @@ class ChromiumFrame(ChromiumBase): while perf_counter() < end_time: try: if self._is_diff_domain is False: - node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self.ids.backend_id)['node'] + node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self._backend_id)['node'] self.doc_ele = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) @@ -304,12 +302,18 @@ class ChromiumFrame(ChromiumBase): # ----------挂件---------- @property - def page(self): - return self._page + def _obj_id(self): + """返回frame元素的object id""" + return self.frame_ele._obj_id @property - def ids(self): - return self._ids + def _node_id(self): + """返回cdp中的node id""" + return self.frame_ele._node_id + + @property + def page(self): + return self._page @property def frame_ele(self): @@ -330,7 +334,7 @@ class ChromiumFrame(ChromiumBase): def html(self): """返回元素outerHTML文本""" tag = self.tag - out_html = self._target_page.run_cdp('DOM.getOuterHTML', backendNodeId=self.frame_ele.ids.backend_id)[ + out_html = self._target_page.run_cdp('DOM.getOuterHTML', backendNodeId=self.frame_ele._backend_id)[ 'outerHTML'] sign = search(rf'<{tag}.*?>', out_html).group(0) return f'{sign}{self.inner_html}' @@ -391,7 +395,7 @@ class ChromiumFrame(ChromiumBase): return self.doc_ele.run_js('return this.readyState;') except ContextLossError: try: - node = self.run_cdp('DOM.describeNode', backendNodeId=self.frame_ele.ids.backend_id)['node'] + node = self.run_cdp('DOM.describeNode', backendNodeId=self.frame_ele._backend_id)['node'] doc = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) return doc.run_js('return this.readyState;') except: diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 7c93fd0..465f6e7 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -12,7 +12,6 @@ from .chromium_tab import ChromiumTab from .web_page import WebPage from .._elements.chromium_element import ChromiumElement from .._units.states import FrameStates -from .._units.ids import FrameIds from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter @@ -32,12 +31,10 @@ class ChromiumFrame(ChromiumBase): self._is_diff_domain: bool = ... self.doc_ele: ChromiumElement = ... self._states: FrameStates = ... - self._ids: FrameIds = ... self._is_init_get_doc: bool = ... self._rect: FrameRect = ... - def __call__(self, - loc_or_str: Union[Tuple[str, str], str], + def __call__(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> Union[ChromiumElement, str]: ... def _check_alive(self) -> None: ... @@ -61,9 +58,6 @@ class ChromiumFrame(ChromiumBase): @property def page(self) -> Union[ChromiumPage, WebPage]: ... - @property - def ids(self) -> FrameIds: ... - @property def frame_ele(self) -> ChromiumElement: ... @@ -91,6 +85,12 @@ class ChromiumFrame(ChromiumBase): @property def rect(self) -> FrameRect: ... + @property + def _obj_id(self) -> str: ... + + @property + def _node_id(self) -> str: ... + @property def active_ele(self) -> ChromiumElement: ... @@ -128,52 +128,36 @@ class ChromiumFrame(ChromiumBase): def parent(self, level_or_loc: Union[tuple, str, int] = 1, index: int = 1) -> Union[ChromiumElement, None]: ... - def prev(self, filter_loc: Union[tuple, str, int] = '', - index: int = 1, - timeout: float = 0, - ele_only: bool = True) -> Union[ChromiumElement, str]: ... + def prev(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, + timeout: float = 0, ele_only: bool = True) -> Union[ChromiumElement, str]: ... - def next(self, filter_loc: Union[tuple, str, int] = '', - index: int = 1, - timeout: float = 0, - ele_only: bool = True) -> Union[ChromiumElement, str]: ... + def next(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, + timeout: float = 0, ele_only: bool = True) -> Union[ChromiumElement, str]: ... - def before(self, filter_loc: Union[tuple, str, int] = '', - index: int = 1, - timeout: float = None, - ele_only: bool = True) -> Union[ChromiumElement, str]: ... + def before(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, + timeout: float = None, ele_only: bool = True) -> Union[ChromiumElement, str]: ... - def after(self, filter_loc: Union[tuple, str, int] = '', - index: int = 1, - timeout: float = None, - ele_only: bool = True) -> Union[ChromiumElement, str]: ... + def after(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, + timeout: float = None, ele_only: bool = True) -> Union[ChromiumElement, str]: ... - def prevs(self, filter_loc: Union[tuple, str] = '', - timeout: float = 0, + def prevs(self, filter_loc: Union[tuple, str] = '', timeout: float = 0, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def nexts(self, filter_loc: Union[tuple, str] = '', - timeout: float = 0, + def nexts(self, filter_loc: Union[tuple, str] = '', timeout: float = 0, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def befores(self, filter_loc: Union[tuple, str] = '', - timeout: float = None, + def befores(self, filter_loc: Union[tuple, str] = '', timeout: float = None, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def afters(self, filter_loc: Union[tuple, str] = '', - timeout: float = None, + def afters(self, filter_loc: Union[tuple, str] = '', timeout: float = None, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def get_screenshot(self, path: [str, Path] = None, name: str = None, - as_bytes: [bool, str] = None, + def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, as_base64: [bool, str] = None) -> Union[str, bytes]: ... - def _get_screenshot(self, path: [str, Path] = None, name: str = None, - as_bytes: [bool, str] = None, as_base64: [bool, str] = None, - full_page: bool = False, - left_top: Tuple[int, int] = None, - right_bottom: Tuple[int, int] = None, - ele: ChromiumElement = None) -> Union[str, bytes]: ... + def _get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, + as_base64: [bool, str] = None, full_page: bool = False, left_top: Tuple[int, int] = None, + right_bottom: Tuple[int, int] = None, ele: ChromiumElement = None) -> Union[str, bytes]: ... def _find_elements(self, loc_or_ele: Union[Tuple[str, str], str, ChromiumElement, ChromiumFrame], timeout: float = None, single: bool = True, relative: bool = False, raise_err: bool = None) \ diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index aae2461..643da4c 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -78,7 +78,7 @@ class Clicker(object): try: r = self._ele.page.run_cdp('DOM.getNodeForLocation', x=x, y=y, includeUserAgentShadowDOM=True, ignorePointerEventsNone=True) - if r['backendNodeId'] != self._ele.ids.backend_id: + if r['backendNodeId'] != self._ele._backend_id: vx, vy = self._ele.rect.viewport_midpoint else: vx, vy = self._ele.rect.viewport_click_point diff --git a/DrissionPage/_units/ids.py b/DrissionPage/_units/ids.py deleted file mode 100644 index f5fb8d8..0000000 --- a/DrissionPage/_units/ids.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding:utf-8 -*- -""" -@Author : g1879 -@Contact : g1879@qq.com -""" - - -class ShadowRootIds(object): - def __init__(self, ele): - self._ele = ele - - @property - def node_id(self): - """返回元素cdp中的node id""" - return self._ele._node_id - - @property - def obj_id(self): - """返回元素js中的object id""" - return self._ele._obj_id - - @property - def backend_id(self): - """返回backend id""" - return self._ele._backend_id - - -class ElementIds(ShadowRootIds): - @property - def doc_id(self): - """返回所在document的object id""" - return self._ele._doc_id - - -class FrameIds(object): - def __init__(self, frame): - self._frame = frame - - @property - def tab_id(self): - """返回当前标签页id""" - return self._frame._tab_id - - @property - def backend_id(self): - """返回cdp中的node id""" - return self._frame._backend_id - - @property - def obj_id(self): - """返回frame元素的object id""" - return self._frame.frame_ele.ids.obj_id - - @property - def node_id(self): - """返回cdp中的node id""" - return self._frame.frame_ele.ids.node_id diff --git a/DrissionPage/_units/ids.pyi b/DrissionPage/_units/ids.pyi deleted file mode 100644 index 907005d..0000000 --- a/DrissionPage/_units/ids.pyi +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding:utf-8 -*- -""" -@Author : g1879 -@Contact : g1879@qq.com -""" -from typing import Union - -from .._elements.chromium_element import ChromiumElement, ChromiumShadowRoot -from .._pages.chromium_frame import ChromiumFrame - - -class ShadowRootIds(object): - def __init__(self, ele: Union[ChromiumElement, ChromiumShadowRoot]): - self._ele: Union[ChromiumElement, ChromiumShadowRoot] = ... - - @property - def node_id(self) -> str: ... - - @property - def obj_id(self) -> str: ... - - @property - def backend_id(self) -> str: ... - - -class ElementIds(ShadowRootIds): - @property - def doc_id(self) -> str: ... - - -class FrameIds(object): - def __init__(self, frame: ChromiumFrame): - self._frame: ChromiumFrame = ... - - @property - def tab_id(self) -> str: ... - - @property - def backend_id(self) -> str: ... - - @property - def obj_id(self) -> str: ... - - @property - def node_id(self) -> str: ... diff --git a/DrissionPage/_units/rect.py b/DrissionPage/_units/rect.py index 4180b14..0b9f0f0 100644 --- a/DrissionPage/_units/rect.py +++ b/DrissionPage/_units/rect.py @@ -99,7 +99,7 @@ class ElementRect(object): :param quad: 方框类型，margin border padding :return: 四个角坐标 """ - return self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele.ids.backend_id)['model'][quad] + return self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele._backend_id)['model'][quad] def _get_page_coord(self, x, y): """根据视口坐标获取绝对坐标""" diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index a1ee590..af58a07 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -413,7 +413,7 @@ class ChromiumElementSetter(object): :param value: 属性值 :return: None """ - self._ele.page.run_cdp('DOM.setAttributeValue', nodeId=self._ele.ids.node_id, name=attr, value=str(value)) + self._ele.page.run_cdp('DOM.setAttributeValue', nodeId=self._ele._node_id, name=attr, value=str(value)) def prop(self, prop, value): """设置元素property属性 @@ -508,14 +508,14 @@ class WindowSetter(object): self._page = page self._window_id = self._get_info()['windowId'] - def maximized(self): + def max(self): """窗口最大化""" s = self._get_info()['bounds']['windowState'] if s in ('fullscreen', 'minimized'): self._perform({'windowState': 'normal'}) self._perform({'windowState': 'maximized'}) - def minimized(self): + def mini(self): """窗口最小化""" s = self._get_info()['bounds']['windowState'] if s == 'fullscreen': @@ -575,6 +575,16 @@ class WindowSetter(object): """ self._page.run_cdp('Browser.setWindowBounds', windowId=self._window_id, bounds=bounds) + # ------------即将废除---------- + + def maximized(self): + """窗口最大化""" + self.max() + + def minimized(self): + """窗口最小化""" + self.mini() + class PageWindowSetter(WindowSetter): def hide(self): diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index cf2f73e..15db2b7 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -191,9 +191,9 @@ class WindowSetter(object): self._page: ChromiumBase = ... self._window_id: str = ... - def maximized(self) -> None: ... + def max(self) -> None: ... - def minimized(self) -> None: ... + def mini(self) -> None: ... def fullscreen(self) -> None: ... diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index 0bf571b..2e3214f 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -68,7 +68,7 @@ class ElementStates(object): except CDPError: return False - if r.get('backendNodeId') != self._ele.ids.backend_id: + if r.get('backendNodeId') != self._ele._backend_id: return True return False @@ -98,7 +98,7 @@ class ShadowRootStates(object): def is_alive(self): """返回元素是否仍在DOM中""" try: - self._ele.page.run_cdp('DOM.describeNode', backendNodeId=self._ele.ids.backend_id) + self._ele.page.run_cdp('DOM.describeNode', backendNodeId=self._ele._backend_id) return True except Exception: return False @@ -145,7 +145,7 @@ class FrameStates(object): """返回frame元素是否可用，且里面仍挂载有frame""" try: node = self._frame._target_page.run_cdp('DOM.describeNode', - backendNodeId=self._frame._frame_ele.ids.backend_id)['node'] + backendNodeId=self._frame._frame_ele._backend_id)['node'] except (ElementLossError, PageClosedError): return False return 'frameId' in node From dbdb4528ab802f27cb84d33f78f8f131860f6205 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 16 Nov 2023 22:33:21 +0800 Subject: [PATCH 089/182] =?UTF-8?q?size=E3=80=81locations=E7=AD=89?= =?UTF-8?q?=E5=90=88=E5=B9=B6=E5=88=B0rect=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 3 +- DrissionPage/_base/browser.py | 5 +- DrissionPage/_elements/chromium_element.py | 43 +++--- DrissionPage/_elements/chromium_element.pyi | 12 +- DrissionPage/_pages/chromium_base.py | 23 +++- DrissionPage/_pages/chromium_base.pyi | 12 +- DrissionPage/_pages/chromium_frame.py | 46 +++---- DrissionPage/_pages/chromium_frame.pyi | 13 -- DrissionPage/_pages/chromium_page.py | 9 -- DrissionPage/_pages/chromium_page.pyi | 3 - DrissionPage/_pages/chromium_tab.py | 8 -- DrissionPage/_pages/chromium_tab.pyi | 3 - DrissionPage/_pages/web_page.py | 14 +- DrissionPage/_units/action_chains.py | 6 +- DrissionPage/_units/clicker.py | 16 +-- DrissionPage/_units/locations.py | 103 -------------- DrissionPage/_units/locations.pyi | 50 ------- DrissionPage/_units/rect.py | 143 ++++++++++++++++++-- DrissionPage/_units/rect.pyi | 67 ++++++++- DrissionPage/_units/states.py | 10 +- DrissionPage/_units/waiter.py | 8 +- 21 files changed, 297 insertions(+), 300 deletions(-) delete mode 100644 DrissionPage/_units/locations.py delete mode 100644 DrissionPage/_units/locations.pyi diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index ad912d6..5b6d740 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -12,4 +12,5 @@ from ._pages.web_page import WebPage from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions -__all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage'] +__all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] +__version__ = '4.0.0b8' diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 05d12c9..9b9ebdf 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -170,7 +170,10 @@ class Browser(object): end_time = perf_counter() + timeout while perf_counter() < end_time: p = popen(txt) - if f' {self.process_id} ' not in p.read(): + try: + if f' {self.process_id} ' not in p.read(): + return + except TypeError: return sleep(.2) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index bf052c7..d28e9e7 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -16,7 +16,7 @@ from .._commons.tools import get_usable_path from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, offset_scroll from .._units.clicker import Clicker from .._units.ids import ShadowRootIds, ElementIds -from .._units.locations import Locations +from .._units.rect import ElementRect from .._units.scroller import ElementScroller from .._units.select_element import SelectElement from .._units.setter import ChromiumElementSetter @@ -39,7 +39,7 @@ class ChromiumElement(DrissionElement): super().__init__(page) self._select = None self._scroll = None - self._locations = None + self._rect = None self._set = None self._states = None self._pseudo = None @@ -126,12 +126,6 @@ class ChromiumElement(DrissionElement): """返回获取内置id的对象""" return self._ids - @property - def size(self): - """返回元素宽和高组成的元组""" - border = self.page.run_cdp('DOM.getBoxModel', backendNodeId=self._backend_id)['model']['border'] - return border[2] - border[0], border[5] - border[1] - @property def set(self): """返回用于设置元素属性的对象""" @@ -154,16 +148,11 @@ class ChromiumElement(DrissionElement): return self._pseudo @property - def location(self): - """返回元素左上角的绝对坐标""" - return self.locations.location - - @property - def locations(self): + def rect(self): """返回用于获取元素位置的对象""" - if self._locations is None: - self._locations = Locations(self) - return self._locations + if self._rect is None: + self._rect = ElementRect(self) + return self._rect @property def shadow_root(self): @@ -565,8 +554,8 @@ class ChromiumElement(DrissionElement): if scroll_to_center: self.scroll.to_see(center=True) - left, top = self.location - width, height = self.size + left, top = self.rect.location + width, height = self.rect.size left_top = (left, top) right_bottom = (left + width, top + height) if not name: @@ -657,7 +646,7 @@ class ChromiumElement(DrissionElement): :param duration: 拖动用时，传入0即瞬间到j达 :return: None """ - curr_x, curr_y = self.locations.midpoint + curr_x, curr_y = self.rect.midpoint offset_x += curr_x offset_y += curr_y self.drag_to((offset_x, offset_y), duration) @@ -669,7 +658,7 @@ class ChromiumElement(DrissionElement): :return: None """ if isinstance(ele_or_loc, ChromiumElement): - ele_or_loc = ele_or_loc.locations.midpoint + ele_or_loc = ele_or_loc.rect.midpoint elif not isinstance(ele_or_loc, (list, tuple)): raise TypeError('需要ChromiumElement对象或坐标。') @@ -754,6 +743,18 @@ class ChromiumElement(DrissionElement): files = [str(Path(i).absolute()) for i in files] self.page.run_cdp('DOM.setFileInputFiles', files=files, backendNodeId=self._backend_id) + # -------------即将废弃------------- + + @property + def location(self): + """返回元素左上角的绝对坐标""" + return self.rect.location + + @property + def size(self): + """返回元素宽和高组成的元组""" + return self.rect.size + class ChromiumShadowRoot(BaseElement): """ChromiumShadowRoot是用于处理ShadowRoot的类，使用方法和ChromiumElement基本一致""" diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 90f3caa..7b261fa 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -15,7 +15,7 @@ from .._pages.chromium_page import ChromiumPage from .._pages.web_page import WebPage from .._units.clicker import Clicker from .._units.ids import ElementIds, ShadowRootIds -from .._units.locations import Locations +from .._units.rect import ElementRect from .._units.scroller import ElementScroller from .._units.select_element import SelectElement from .._units.setter import ChromiumElementSetter @@ -37,7 +37,7 @@ class ChromiumElement(DrissionElement): self._clicker: Clicker = ... self._select: SelectElement = ... self._wait: ElementWaiter = ... - self._locations: Locations = ... + self._rect: ElementRect = ... self._set: ChromiumElementSetter = ... self._states: ElementStates = ... self._pseudo: Pseudo = ... @@ -69,9 +69,6 @@ class ChromiumElement(DrissionElement): @property def ids(self) -> ElementIds: ... - @property - def size(self) -> Tuple[float, float]: ... - @property def set(self) -> ChromiumElementSetter: ... @@ -79,10 +76,7 @@ class ChromiumElement(DrissionElement): def states(self) -> ElementStates: ... @property - def location(self) -> Tuple[float, float]: ... - - @property - def locations(self) -> Locations: ... + def rect(self) -> ElementRect: ... @property def pseudo(self) -> Pseudo: ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 92c8c56..2158ce0 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -10,6 +10,7 @@ from re import findall from threading import Thread from time import perf_counter, sleep +from .._units.rect import TabRect from .._base.base import BasePage from .._commons.constants import ERROR, NoneElement from .._commons.locator import get_loc @@ -48,6 +49,7 @@ class ChromiumBase(BasePage): self._states = None self._has_alert = False self._ready_state = None + self._rect = None self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc self._download_path = str(Path('.').absolute()) @@ -316,6 +318,14 @@ class ChromiumBase(BasePage): self._scroll = PageScroller(self) return self._scroll + @property + def rect(self): + """返回获取窗口坐标和大小的对象""" + # self.wait.load_complete() + if self._rect is None: + self._rect = TabRect(self) + return self._rect + @property def timeouts(self): """返回timeouts设置""" @@ -373,12 +383,6 @@ class ChromiumBase(BasePage): """返回当前标签页id""" return self.driver.id if not self.driver._stopped.is_set() else '' - @property - def size(self): - """返回页面总宽高，格式：(宽, 高)""" - r = self.run_cdp_loaded('Page.getLayoutMetrics')['contentSize'] - return r['width'], r['height'] - @property def active_ele(self): """返回当前焦点所在元素""" @@ -973,7 +977,7 @@ class ChromiumBase(BasePage): pic_type = path.suffix.lower() pic_type = 'jpeg' if pic_type == '.jpg' else pic_type[1:] - width, height = self.size + width, height = self.rect.size if full_page: vp = {'x': 0, 'y': 0, 'width': width, 'height': height, 'scale': 1} png = self.run_cdp_loaded('Page.captureScreenshot', format=pic_type, @@ -1030,6 +1034,11 @@ class ChromiumBase(BasePage): def ready_state(self): return self._ready_state + @property + def size(self): + """返回页面总宽高，格式：(宽, 高)""" + return self.rect.size + class Timeout(object): """用于保存d模式timeout信息的类""" diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 41caec9..37f9225 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -6,6 +6,7 @@ from pathlib import Path from typing import Union, Tuple, List, Any, Optional +from .._units.rect import TabRect from .._base.base import BasePage from .._base.browser import Browser from .._base.chromium_driver import ChromiumDriver @@ -54,6 +55,7 @@ class ChromiumBase(BasePage): self._has_alert: bool = ... self._doc_got: bool = ... self._ready_state: Optional[str] = ... + self._rect: TabRect = ... def _connect_browser(self, tab_id: str = None) -> None: ... @@ -118,9 +120,6 @@ class ChromiumBase(BasePage): @property def tab_id(self) -> str: ... - @property - def size(self) -> Tuple[int, int]: ... - @property def active_ele(self) -> ChromiumElement: ... @@ -133,6 +132,9 @@ class ChromiumBase(BasePage): @property def scroll(self) -> PageScroller: ... + @property + def rect(self) -> TabRect: ... + @property def timeouts(self) -> Timeout: ... @@ -216,8 +218,8 @@ class ChromiumBase(BasePage): left_top: Tuple[int, int] = None, right_bottom: Tuple[int, int] = None) -> Union[str, bytes]: ... def _get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, - as_base64: [bool, str] = None, full_page: bool = False, left_top: Tuple[int, int] = None, - right_bottom: Tuple[int, int] = None, ele: ChromiumElement = None) -> Union[str, bytes]: ... + as_base64: [bool, str] = None, full_page: bool = False, left_top: Tuple[float, float] = None, + right_bottom: Tuple[float, float] = None, ele: ChromiumElement = None) -> Union[str, bytes]: ... def clear_cache(self, session_storage: bool = True, local_storage: bool = True, cache: bool = True, cookies: bool = True) -> None: ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index c0e15d1..7be33ba 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -356,33 +356,11 @@ class ChromiumFrame(ChromiumBase): """返回frame元素所有attribute属性""" return self.frame_ele.attrs - @property - def page_size(self): - """返回frame内页面尺寸，格式：(长, 高)""" - w = self.doc_ele.run_js('return this.body.scrollWidth') - h = self.doc_ele.run_js('return this.body.scrollHeight') - return w, h - - @property - def size(self): - """返回frame元素大小""" - return self.frame_ele.size - @property def active_ele(self): """返回当前焦点所在元素""" return self.doc_ele.run_js('return this.activeElement;') - @property - def location(self): - """返回frame元素左上角的绝对坐标""" - return self.frame_ele.location - - @property - def locations(self): - """返回用于获取元素位置的对象""" - return self.frame_ele.locations - @property def xpath(self): """返回frame的xpath绝对路径""" @@ -597,8 +575,8 @@ class ChromiumFrame(ChromiumBase): self.frame_ele.scroll.to_see(center=True) self.scroll.to_see(ele, center=True) - cx, cy = ele.locations.viewport_location - w, h = ele.size + cx, cy = ele.rect.viewport_location + w, h = ele.rect.size img_data = f'data:image/{pic_type};base64,{self.frame_ele.get_screenshot(as_base64=True)}' body = self.tab('t:body') first_child = body('c::first-child') @@ -652,3 +630,23 @@ class ChromiumFrame(ChromiumBase): def is_alive(self): """返回是否仍可用""" return self.states.is_alive + + @property + def page_size(self): + """返回frame内页面尺寸，格式：(宽,, 高)""" + return self.rect.size + + @property + def size(self): + """返回frame元素大小""" + return self.frame_ele.rect.size + + @property + def location(self): + """返回frame元素左上角的绝对坐标""" + return self.frame_ele.rect.location + + @property + def locations(self): + """返回用于获取元素位置的对象""" + return self.frame_ele.rect diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index cfaba9e..7c93fd0 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -13,7 +13,6 @@ from .web_page import WebPage from .._elements.chromium_element import ChromiumElement from .._units.states import FrameStates from .._units.ids import FrameIds -from .._units.locations import Locations from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter @@ -89,24 +88,12 @@ class ChromiumFrame(ChromiumBase): @property def attrs(self) -> dict: ... - @property - def page_size(self) -> Tuple[int, int]: ... - - @property - def size(self) -> Tuple[int, int]: ... - @property def rect(self) -> FrameRect: ... @property def active_ele(self) -> ChromiumElement: ... - @property - def location(self) -> Tuple[int, int]: ... - - @property - def locations(self) -> Locations: ... - @property def xpath(self) -> str: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index bd2cdd5..6a8b60c 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -13,7 +13,6 @@ from .._commons.browser import connect_browser from .._configs.chromium_options import ChromiumOptions from .._pages.chromium_base import ChromiumBase, Timeout from .._pages.chromium_tab import ChromiumTab -from .._units.rect import TabRect from .._units.setter import ChromiumPageSetter from .._units.waiter import PageWaiter from ..errors import BrowserConnectError @@ -104,14 +103,6 @@ class ChromiumPage(ChromiumBase): self._set = ChromiumPageSetter(self) return self._set - @property - def rect(self): - """返回保存窗口方位信息的对象""" - self.wait.load_complete() - if self._rect is None: - self._rect = TabRect(self) - return self._rect - @property def wait(self): """返回用于等待的对象""" diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index 62f7e95..c834405 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -39,9 +39,6 @@ class ChromiumPage(ChromiumBase): @property def tabs(self) -> List[str]: ... - @property - def rect(self) -> TabRect: ... - @property def wait(self) -> PageWaiter: ... diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 8f8cc7c..c67600a 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -9,7 +9,6 @@ from .._base.base import BasePage from .._commons.web import set_session_cookies, set_browser_cookies from .._pages.chromium_base import ChromiumBase from .._pages.session_page import SessionPage -from .._units.rect import TabRect from .._units.setter import TabSetter, WebPageTabSetter from .._units.waiter import TabWaiter @@ -44,13 +43,6 @@ class ChromiumTab(ChromiumBase): """返回总体page对象""" return self._page - @property - def rect(self): - """返回获取窗口坐标和大小的对象""" - if self._rect is None: - self._rect = TabRect(self) - return self._rect - @property def set(self): """返回用于等待的对象""" diff --git a/DrissionPage/_pages/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi index 5adb85f..687a88c 100644 --- a/DrissionPage/_pages/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -34,9 +34,6 @@ class ChromiumTab(ChromiumBase): @property def page(self) -> ChromiumPage: ... - @property - def rect(self) -> TabRect: ... - @property def set(self) -> TabSetter: ... diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 679745a..a873b3a 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -49,6 +49,13 @@ class WebPage(SessionPage, ChromiumPage, BasePage): elif self._mode == 's': return super().__call__(loc_or_str) + @property + def set(self): + """返回用于等待的对象""" + if self._set is None: + self._set = WebPageSetter(self) + return self._set + @property def url(self): """返回当前url""" @@ -134,13 +141,6 @@ class WebPage(SessionPage, ChromiumPage, BasePage): """ self.set.timeouts(implicit=second) - @property - def set(self): - """返回用于等待的对象""" - if self._set is None: - self._set = WebPageSetter(self) - return self._set - def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None, **kwargs): """跳转到一个url :param url: 目标url diff --git a/DrissionPage/_units/action_chains.py b/DrissionPage/_units/action_chains.py index 8b177d1..df74134 100644 --- a/DrissionPage/_units/action_chains.py +++ b/DrissionPage/_units/action_chains.py @@ -39,7 +39,7 @@ class ActionChains: elif isinstance(ele_or_loc, str) or 'ChromiumElement' in str(type(ele_or_loc)): ele_or_loc = self.page(ele_or_loc) self.page.scroll.to_see(ele_or_loc) - x, y = ele_or_loc.location if offset_x or offset_y else ele_or_loc.locations.midpoint + x, y = ele_or_loc.rect.location if offset_x or offset_y else ele_or_loc.rect.midpoint lx = x + offset_x ly = y + offset_y else: @@ -55,8 +55,8 @@ class ActionChains: if is_loc: cx, cy = location_to_client(self.page, lx, ly) else: - x, y = ele_or_loc.locations.viewport_location if offset_x or offset_y \ - else ele_or_loc.locations.viewport_midpoint + x, y = ele_or_loc.rect.viewport_location if offset_x or offset_y \ + else ele_or_loc.rect.viewport_midpoint cx = x + offset_x cy = y + offset_y diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index 0ab9234..aae2461 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -42,7 +42,7 @@ class Clicker(object): try: self._ele.scroll.to_see() if self._ele.states.is_enabled and self._ele.states.is_displayed: - rect = self._ele.locations.viewport_rect + rect = self._ele.rect.viewport_corners can_click = True except NoRectError: if by_js is False: @@ -59,7 +59,7 @@ class Clicker(object): self._ele.wait.stop_moving(timeout=end_time - perf_counter()) if rect: self._ele.scroll.to_see() - rect = self._ele.locations.rect + rect = self._ele.rect.corners while perf_counter() < end_time: if self._ele.states.is_enabled and self._ele.states.is_displayed: can_click = True @@ -79,12 +79,12 @@ class Clicker(object): r = self._ele.page.run_cdp('DOM.getNodeForLocation', x=x, y=y, includeUserAgentShadowDOM=True, ignorePointerEventsNone=True) if r['backendNodeId'] != self._ele.ids.backend_id: - vx, vy = self._ele.locations.viewport_midpoint + vx, vy = self._ele.rect.viewport_midpoint else: - vx, vy = self._ele.locations.viewport_click_point + vx, vy = self._ele.rect.viewport_click_point except CDPError: - vx, vy = self._ele.locations.viewport_midpoint + vx, vy = self._ele.rect.viewport_midpoint self._click(vx, vy) return True @@ -99,13 +99,13 @@ class Clicker(object): def right(self): """右键单击""" self._ele.page.scroll.to_see(self._ele) - x, y = self._ele.locations.viewport_click_point + x, y = self._ele.rect.viewport_click_point self._click(x, y, 'right') def middle(self): """中键单击""" self._ele.page.scroll.to_see(self._ele) - x, y = self._ele.locations.viewport_click_point + x, y = self._ele.rect.viewport_click_point self._click(x, y, 'middle') def at(self, offset_x=None, offset_y=None, button='left', count=1): @@ -118,7 +118,7 @@ class Clicker(object): """ self._ele.page.scroll.to_see(self._ele) if offset_x is None and offset_y is None: - w, h = self._ele.size + w, h = self._ele.rect.size offset_x = w // 2 offset_y = h // 2 x, y = offset_scroll(self._ele, offset_x, offset_y) diff --git a/DrissionPage/_units/locations.py b/DrissionPage/_units/locations.py deleted file mode 100644 index 010a962..0000000 --- a/DrissionPage/_units/locations.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding:utf-8 -*- -""" -@Author : g1879 -@Contact : g1879@qq.com -""" - - -class Locations(object): - def __init__(self, ele): - """ - :param ele: ChromiumElement - """ - self._ele = ele - - @property - def location(self): - """返回元素左上角的绝对坐标""" - cl = self.viewport_location - return self._get_page_coord(cl[0], cl[1]) - - @property - def midpoint(self): - """返回元素中间点的绝对坐标""" - cl = self.viewport_midpoint - return self._get_page_coord(cl[0], cl[1]) - - @property - def click_point(self): - """返回元素接受点击的点的绝对坐标""" - cl = self.viewport_click_point - return self._get_page_coord(cl[0], cl[1]) - - @property - def viewport_location(self): - """返回元素左上角在视口中的坐标""" - m = self._get_viewport_rect('border') - return m[0], m[1] - - @property - def viewport_midpoint(self): - """返回元素中间点在视口中的坐标""" - m = self._get_viewport_rect('border') - return m[0] + (m[2] - m[0]) // 2, m[3] + (m[5] - m[3]) // 2 - - @property - def viewport_click_point(self): - """返回元素接受点击的点视口坐标""" - m = self._get_viewport_rect('padding') - return self.viewport_midpoint[0], m[1] + 3 - - @property - def screen_location(self): - """返回元素左上角在屏幕上坐标，左上角为(0, 0)""" - vx, vy = self._ele.page.rect.viewport_location - ex, ey = self.viewport_location - pr = self._ele.page.run_js('return window.devicePixelRatio;') - return (vx + ex) * pr, (ey + vy) * pr - - @property - def screen_midpoint(self): - """返回元素中点在屏幕上坐标，左上角为(0, 0)""" - vx, vy = self._ele.page.rect.viewport_location - ex, ey = self.viewport_midpoint - pr = self._ele.page.run_js('return window.devicePixelRatio;') - return (vx + ex) * pr, (ey + vy) * pr - - @property - def screen_click_point(self): - """返回元素中点在屏幕上坐标，左上角为(0, 0)""" - vx, vy = self._ele.page.rect.viewport_location - ex, ey = self.viewport_click_point - pr = self._ele.page.run_js('return window.devicePixelRatio;') - return (vx + ex) * pr, (ey + vy) * pr - - @property - def rect(self): - """返回元素四个角坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" - vr = self._get_viewport_rect('border') - r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] - sx = r['pageX'] - sy = r['pageY'] - return [(vr[0] + sx, vr[1] + sy), (vr[2] + sx, vr[3] + sy), - (vr[4] + sx, vr[5] + sy), (vr[6] + sx, vr[7] + sy)] - - @property - def viewport_rect(self): - """返回元素四个角视口坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" - r = self._get_viewport_rect('border') - return [(r[0], r[1]), (r[2], r[3]), (r[4], r[5]), (r[6], r[7])] - - def _get_viewport_rect(self, quad): - """按照类型返回在可视窗口中的范围 - :param quad: 方框类型，margin border padding - :return: 四个角坐标 - """ - return self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele.ids.backend_id)['model'][quad] - - def _get_page_coord(self, x, y): - """根据视口坐标获取绝对坐标""" - r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] - sx = r['pageX'] - sy = r['pageY'] - return x + sx, y + sy diff --git a/DrissionPage/_units/locations.pyi b/DrissionPage/_units/locations.pyi deleted file mode 100644 index 5e7ca31..0000000 --- a/DrissionPage/_units/locations.pyi +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding:utf-8 -*- -""" -@Author : g1879 -@Contact : g1879@qq.com -""" -from typing import Tuple, Union, List - -from .._elements.chromium_element import ChromiumElement - - -class Locations(object): - def __init__(self, ele: ChromiumElement): - self._ele: ChromiumElement = ... - - @property - def location(self) -> Tuple[float, float]: ... - - @property - def midpoint(self) -> Tuple[float, float]: ... - - @property - def click_point(self) -> Tuple[float, float]: ... - - @property - def viewport_location(self) -> Tuple[float, float]: ... - - @property - def viewport_midpoint(self) -> Tuple[float, float]: ... - - @property - def viewport_click_point(self) -> Tuple[float, float]: ... - - @property - def screen_location(self) -> Tuple[float, float]: ... - - @property - def screen_midpoint(self) -> Tuple[float, float]: ... - - @property - def screen_click_point(self) -> Tuple[float, float]: ... - - @property - def rect(self) -> List[Tuple[float, float], ...]: ... - - @property - def viewport_rect(self) -> List[Tuple[float, float], ...]: ... - - def _get_viewport_rect(self, quad: str) -> Union[list, None]: ... - - def _get_page_coord(self, x: float, y: float) -> Tuple[float, float]: ... diff --git a/DrissionPage/_units/rect.py b/DrissionPage/_units/rect.py index 1b31ebb..4180b14 100644 --- a/DrissionPage/_units/rect.py +++ b/DrissionPage/_units/rect.py @@ -5,6 +5,110 @@ """ +class ElementRect(object): + def __init__(self, ele): + """ + :param ele: ChromiumElement + """ + self._ele = ele + + @property + def corners(self): + """返回元素四个角坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" + vr = self._get_viewport_rect('border') + r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] + sx = r['pageX'] + sy = r['pageY'] + return [(vr[0] + sx, vr[1] + sy), (vr[2] + sx, vr[3] + sy), + (vr[4] + sx, vr[5] + sy), (vr[6] + sx, vr[7] + sy)] + + @property + def viewport_corners(self): + """返回元素四个角视口坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" + r = self._get_viewport_rect('border') + return [(r[0], r[1]), (r[2], r[3]), (r[4], r[5]), (r[6], r[7])] + + @property + def size(self): + """返回元素大小，格式(宽, 高)""" + border = self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele._backend_id)['model']['border'] + return border[2] - border[0], border[5] - border[1] + + @property + def location(self): + """返回元素左上角的绝对坐标""" + cl = self.viewport_location + return self._get_page_coord(cl[0], cl[1]) + + @property + def midpoint(self): + """返回元素中间点的绝对坐标""" + cl = self.viewport_midpoint + return self._get_page_coord(cl[0], cl[1]) + + @property + def click_point(self): + """返回元素接受点击的点的绝对坐标""" + cl = self.viewport_click_point + return self._get_page_coord(cl[0], cl[1]) + + @property + def viewport_location(self): + """返回元素左上角在视口中的坐标""" + m = self._get_viewport_rect('border') + return m[0], m[1] + + @property + def viewport_midpoint(self): + """返回元素中间点在视口中的坐标""" + m = self._get_viewport_rect('border') + return m[0] + (m[2] - m[0]) // 2, m[3] + (m[5] - m[3]) // 2 + + @property + def viewport_click_point(self): + """返回元素接受点击的点视口坐标""" + m = self._get_viewport_rect('padding') + return self.viewport_midpoint[0], m[1] + 3 + + @property + def screen_location(self): + """返回元素左上角在屏幕上坐标，左上角为(0, 0)""" + vx, vy = self._ele.page.rect.viewport_location + ex, ey = self.viewport_location + pr = self._ele.page.run_js('return window.devicePixelRatio;') + return (vx + ex) * pr, (ey + vy) * pr + + @property + def screen_midpoint(self): + """返回元素中点在屏幕上坐标，左上角为(0, 0)""" + vx, vy = self._ele.page.rect.viewport_location + ex, ey = self.viewport_midpoint + pr = self._ele.page.run_js('return window.devicePixelRatio;') + return (vx + ex) * pr, (ey + vy) * pr + + @property + def screen_click_point(self): + """返回元素中点在屏幕上坐标，左上角为(0, 0)""" + vx, vy = self._ele.page.rect.viewport_location + ex, ey = self.viewport_click_point + pr = self._ele.page.run_js('return window.devicePixelRatio;') + return (vx + ex) * pr, (ey + vy) * pr + + def _get_viewport_rect(self, quad): + """按照类型返回在可视窗口中的范围 + :param quad: 方框类型，margin border padding + :return: 四个角坐标 + """ + return self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele.ids.backend_id)['model'][quad] + + def _get_page_coord(self, x, y): + """根据视口坐标获取绝对坐标""" + r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] + sx = r['pageX'] + sy = r['pageY'] + return x + sx, y + sy + + class TabRect(object): def __init__(self, page): self._page = page @@ -49,7 +153,7 @@ class TabRect(object): return w_bl + w_bs - w_vs, h_bl + h_bs - h_vs @property - def page_size(self): + def size(self): """返回页面总宽高，格式：(宽, 高)""" r = self._get_page_rect()['contentSize'] return r['width'], r['height'] @@ -83,21 +187,38 @@ class FrameRect(object): self._frame = frame @property - def viewport_location(self): - """返回视口在屏幕中坐标，左上角为(0, 0)""" - return self._frame.frame_ele.locations.screen_location + def location(self): + """返回元素左上角的绝对坐标""" + return self._frame.frame_ele.rect.location @property - def page_size(self): - """返回页面总宽高，格式：(宽, 高)""" - return self._frame.page_size + def viewport_location(self): + """返回视口在屏幕中坐标，左上角为(0, 0)""" + return self._frame.frame_ele.rect.viewport_location + + @property + def screen_location(self): + """返回元素左上角在屏幕上坐标，左上角为(0, 0)""" + return self._frame.frame_ele.rect.screen_location @property def size(self): - """返回页面总宽高，格式：(宽, 高)""" - return self._frame.size + """返回frame内页面尺寸，格式：(宽, 高)""" + w = self._frame.doc_ele.run_js('return this.body.scrollWidth') + h = self._frame.doc_ele.run_js('return this.body.scrollHeight') + return w, h @property def viewport_size(self): - """返回视口宽高，不包括滚动条，格式：(宽, 高)""" - return self._frame.frame_ele.size + """返回视口宽高，格式：(宽, 高)""" + return self._frame.frame_ele.rect.size + + @property + def corners(self): + """返回元素四个角坐标，顺序：坐上、右上、右下、左下""" + return self._frame.frame_ele.rect.corners + + @property + def viewport_corners(self): + """返回元素四个角视口坐标，顺序：坐上、右上、右下、左下""" + return self._frame.frame_ele.rect.viewport_corners diff --git a/DrissionPage/_units/rect.pyi b/DrissionPage/_units/rect.pyi index 4c507fb..b45ef98 100644 --- a/DrissionPage/_units/rect.pyi +++ b/DrissionPage/_units/rect.pyi @@ -3,16 +3,64 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Tuple, Union +from typing import Tuple, Union, List + +from .._elements.chromium_element import ChromiumElement +from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage from .._pages.chromium_tab import ChromiumTab, WebPageTab from .._pages.web_page import WebPage +class ElementRect(object): + def __init__(self, ele: ChromiumElement): + self._ele: ChromiumElement = ... + + @property + def size(self) -> Tuple[float, float]: ... + + @property + def location(self) -> Tuple[float, float]: ... + + @property + def midpoint(self) -> Tuple[float, float]: ... + + @property + def click_point(self) -> Tuple[float, float]: ... + + @property + def viewport_location(self) -> Tuple[float, float]: ... + + @property + def viewport_midpoint(self) -> Tuple[float, float]: ... + + @property + def viewport_click_point(self) -> Tuple[float, float]: ... + + @property + def screen_location(self) -> Tuple[float, float]: ... + + @property + def screen_midpoint(self) -> Tuple[float, float]: ... + + @property + def screen_click_point(self) -> Tuple[float, float]: ... + + @property + def corners(self) -> List[Tuple[float, float], ...]: ... + + @property + def viewport_corners(self) -> List[Tuple[float, float], ...]: ... + + def _get_viewport_rect(self, quad: str) -> Union[list, None]: ... + + def _get_page_coord(self, x: float, y: float) -> Tuple[float, float]: ... + + class TabRect(object): - def __init__(self, page: Union[ChromiumPage, ChromiumTab, WebPage, WebPageTab]): + def __init__(self, page: ChromiumBase): self._page: Union[ChromiumPage, ChromiumTab, WebPage, WebPageTab] = ... @property @@ -31,7 +79,7 @@ class TabRect(object): def window_size(self) -> Tuple[int, int]: ... @property - def page_size(self) -> Tuple[int, int]: ... + def size(self) -> Tuple[int, int]: ... @property def viewport_size(self) -> Tuple[int, int]: ... @@ -48,14 +96,23 @@ class FrameRect(object): def __init__(self, frame: ChromiumFrame): self._frame: ChromiumFrame = ... + @property + def location(self) -> Tuple[float, float]: ... + @property def viewport_location(self) -> Tuple[float, float]: ... + @property + def screen_location(self) -> Tuple[float, float]: ... + @property def size(self) -> Tuple[float, float]: ... @property - def page_size(self) -> Tuple[float, float]: ... + def viewport_size(self) -> Tuple[float, float]: ... @property - def viewport_size(self) -> Tuple[float, float]: ... + def corners(self) -> List[Tuple[float, float], ...]: ... + + @property + def viewport_corners(self) -> List[Tuple[float, float], ...]: ... diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index 2b7c79f..0bf571b 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -48,21 +48,21 @@ class ElementStates(object): @property def is_in_viewport(self): """返回元素是否出现在视口中，以元素click_point为判断""" - x, y = self._ele.locations.click_point + x, y = self._ele.rect.click_point return location_in_viewport(self._ele.page, x, y) if x else False @property def is_whole_in_viewport(self): """返回元素是否整个都在视口内""" - x1, y1 = self._ele.location - w, h = self._ele.size + x1, y1 = self._ele.rect.location + w, h = self._ele.rect.size x2, y2 = x1 + w, y1 + h return location_in_viewport(self._ele.page, x1, y1) and location_in_viewport(self._ele.page, x2, y2) @property def is_covered(self): """返回元素是否被覆盖，与是否在视口中无关""" - lx, ly = self._ele.locations.click_point + lx, ly = self._ele.rect.click_point try: r = self._ele.page.run_cdp('DOM.getNodeForLocation', x=lx, y=ly) except CDPError: @@ -77,7 +77,7 @@ class ElementStates(object): def has_rect(self): """返回元素是否拥有位置和大小，没有返回False，有返回四个角在页面中坐标组成的列表""" try: - return self._ele.locations.rect + return self._ele.rect.corners except NoRectError: return False diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 0361c56..cb265d4 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -376,7 +376,7 @@ class ElementWaiter(object): while perf_counter() < end_time: try: size = self._ele.states.has_rect - location = self._ele.location + location = self._ele.rect.location break except NoRectError: pass @@ -385,10 +385,10 @@ class ElementWaiter(object): while perf_counter() < end_time: sleep(gap) - if self._ele.size == size and location == self._ele.location: + if self._ele.rect.size == size and location == self._ele.rect.location: return True - size = self._ele.size - location = self._ele.location + size = self._ele.rect.size + location = self._ele.rect.location if raise_err is True or Settings.raise_when_wait_failed is True: raise WaitTimeoutError('等待元素停止运动失败。') From bde7fecab3f2d0290addda84b43cc4f658217420 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 16 Nov 2023 23:18:34 +0800 Subject: [PATCH 090/182] =?UTF-8?q?maximized()=E5=92=8Cminimized()?= =?UTF-8?q?=E6=94=B9=E6=88=90max()=E5=92=8Cmini()=EF=BC=9B=E5=88=A0?= =?UTF-8?q?=E9=99=A4ids=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 26 +++------ DrissionPage/_elements/chromium_element.pyi | 8 --- DrissionPage/_elements/session_element.py | 4 +- DrissionPage/_pages/chromium_base.py | 8 +-- DrissionPage/_pages/chromium_frame.py | 28 ++++++---- DrissionPage/_pages/chromium_frame.pyi | 62 ++++++++------------- DrissionPage/_units/clicker.py | 2 +- DrissionPage/_units/ids.py | 57 ------------------- DrissionPage/_units/ids.pyi | 45 --------------- DrissionPage/_units/rect.py | 2 +- DrissionPage/_units/setter.py | 16 +++++- DrissionPage/_units/setter.pyi | 4 +- DrissionPage/_units/states.py | 6 +- 13 files changed, 72 insertions(+), 196 deletions(-) delete mode 100644 DrissionPage/_units/ids.py delete mode 100644 DrissionPage/_units/ids.pyi diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index d28e9e7..e70bb8d 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -15,7 +15,6 @@ from .._commons.locator import get_loc from .._commons.tools import get_usable_path from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, offset_scroll from .._units.clicker import Clicker -from .._units.ids import ShadowRootIds, ElementIds from .._units.rect import ElementRect from .._units.scroller import ElementScroller from .._units.select_element import SelectElement @@ -66,7 +65,6 @@ class ChromiumElement(DrissionElement): else: raise ElementLossError - self._ids = ElementIds(self) doc = self.run_js('return this.ownerDocument;') self._doc_id = doc['objectId'] if doc else None @@ -121,10 +119,6 @@ class ChromiumElement(DrissionElement): return self.prop('innerText') # -----------------d模式独有属性------------------- - @property - def ids(self): - """返回获取内置id的对象""" - return self._ids @property def set(self): @@ -775,7 +769,6 @@ class ChromiumShadowRoot(BaseElement): self._obj_id = obj_id self._node_id = self._get_node_id(obj_id) self._backend_id = self._get_backend_id(self._node_id) - self._ids = ShadowRootIds(self) self._states = None def __repr__(self): @@ -805,11 +798,6 @@ class ChromiumShadowRoot(BaseElement): """返回内部的html文本""" return self.run_js('return this.innerHTML;') - @property - def ids(self): - """返回获取内置id的对象""" - return self._ids - @property def states(self): """返回用于获取元素状态的对象""" @@ -1104,7 +1092,7 @@ def find_by_xpath(ele, xpath, single, timeout, relative=True): type_txt = '9' if single else '7' node_txt = 'this.contentDocument' if ele.tag in FRAME_ELEMENT and not relative else 'this' js = make_js_for_find_ele_by_xpath(xpath, type_txt, node_txt) - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele.ids.obj_id, + r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, returnByValue=False, awaitPromise=True, userGesture=True) if r['result']['type'] == 'string': return r['result']['value'] @@ -1112,7 +1100,7 @@ def find_by_xpath(ele, xpath, single, timeout, relative=True): if 'exceptionDetails' in r: if 'The result is not a node set' in r['result']['description']: js = make_js_for_find_ele_by_xpath(xpath, '1', node_txt) - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele.ids.obj_id, + r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, returnByValue=False, awaitPromise=True, userGesture=True) return r['result']['value'] else: @@ -1121,7 +1109,7 @@ def find_by_xpath(ele, xpath, single, timeout, relative=True): end_time = perf_counter() + timeout while (r['result']['subtype'] == 'null' or r['result']['description'] == 'NodeList(0)') and perf_counter() < end_time: - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele.ids.obj_id, + r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, returnByValue=False, awaitPromise=True, userGesture=True) if single: @@ -1150,13 +1138,13 @@ def find_by_css(ele, selector, single, timeout): find_all = '' if single else 'All' node_txt = 'this.contentDocument' if ele.tag in ('iframe', 'frame', 'shadow-root') else 'this' js = f'function(){{return {node_txt}.querySelector{find_all}("{selector}");}}' - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele.ids.obj_id, + r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, returnByValue=False, awaitPromise=True, userGesture=True) end_time = perf_counter() + timeout while ('exceptionDetails' in r or r['result']['subtype'] == 'null' or r['result']['description'] == 'NodeList(0)') and perf_counter() < end_time: - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele.ids.obj_id, + r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, returnByValue=False, awaitPromise=True, userGesture=True) if 'exceptionDetails' in r: @@ -1259,7 +1247,7 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): """ if isinstance(page_or_ele, (ChromiumElement, ChromiumShadowRoot)): page = page_or_ele.page - obj_id = page_or_ele.ids.obj_id + obj_id = page_or_ele._obj_id is_page = False else: page = page_or_ele @@ -1345,7 +1333,7 @@ def parse_js_result(page, ele, result): def convert_argument(arg): """把参数转换成js能够接收的形式""" if isinstance(arg, ChromiumElement): - return {'objectId': arg.ids.obj_id} + return {'objectId': arg._obj_id} elif isinstance(arg, (int, float, str, bool)): return {'value': arg} diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 7b261fa..04bd049 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -14,7 +14,6 @@ from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage from .._pages.web_page import WebPage from .._units.clicker import Clicker -from .._units.ids import ElementIds, ShadowRootIds from .._units.rect import ElementRect from .._units.scroller import ElementScroller from .._units.select_element import SelectElement @@ -32,7 +31,6 @@ class ChromiumElement(DrissionElement): self._obj_id: str = ... self._backend_id: str = ... self._doc_id: str = ... - self._ids: ElementIds = ... self._scroll: ElementScroller = ... self._clicker: Clicker = ... self._select: SelectElement = ... @@ -66,8 +64,6 @@ class ChromiumElement(DrissionElement): def raw_text(self) -> str: ... # -----------------d模式独有属性------------------- - @property - def ids(self) -> ElementIds: ... @property def set(self) -> ChromiumElementSetter: ... @@ -211,7 +207,6 @@ class ChromiumShadowRoot(BaseElement): def __init__(self, parent_ele: ChromiumElement, obj_id: str = None, backend_id: str = None): self._obj_id: str = ... - self._ids: ShadowRootIds = ... self._node_id: str = ... self._backend_id: str = ... self.page: ChromiumPage = ... @@ -223,9 +218,6 @@ class ChromiumShadowRoot(BaseElement): def __call__(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> ChromiumElement: ... - @property - def ids(self) -> ShadowRootIds: ... - @property def states(self) -> ShadowRootStates: ... diff --git a/DrissionPage/_elements/session_element.py b/DrissionPage/_elements/session_element.py index 5fd09f4..a3d74f5 100644 --- a/DrissionPage/_elements/session_element.py +++ b/DrissionPage/_elements/session_element.py @@ -334,8 +334,8 @@ def make_session_ele(html_or_ele, loc=None, single=True): page = html_or_ele.page xpath = html_or_ele.xpath # ChromiumElement，兼容传入的元素在iframe内的情况 - html = html_or_ele.page.run_cdp('DOM.getOuterHTML', objectId=html_or_ele.ids.doc_id)['outerHTML'] \ - if html_or_ele.ids.doc_id else html_or_ele.page.html + html = html_or_ele.page.run_cdp('DOM.getOuterHTML', objectId=html_or_ele._doc_id)['outerHTML'] \ + if html_or_ele._doc_id else html_or_ele.page.html html_or_ele = fromstring(html) html_or_ele = html_or_ele.xpath(xpath)[0] diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 2158ce0..d60e0b2 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -50,6 +50,9 @@ class ChromiumBase(BasePage): self._has_alert = False self._ready_state = None self._rect = None + self._wait = None + self._scroll = None + self._upload_list = None self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc self._download_path = str(Path('.').absolute()) @@ -81,9 +84,6 @@ class ChromiumBase(BasePage): :return: None """ self._is_reading = False - self._upload_list = None - self._wait = None - self._scroll = None if not tab_id: tabs = self.browser.driver.get(f'http://{self.address}/json').json() @@ -697,7 +697,7 @@ class ChromiumBase(BasePage): return ele = self._ele(loc_or_ele, raise_err=False) if ele: - self.run_cdp('DOM.removeNode', nodeId=ele.ids.node_id) + self.run_cdp('DOM.removeNode', nodeId=ele._node_id) def get_frame(self, loc_ind_ele, timeout=None): """获取页面中一个frame对象，可传入定位符、iframe序号、ChromiumFrame对象，序号从1开始 diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 7be33ba..ec9cea9 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -9,7 +9,6 @@ from time import sleep, perf_counter from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase -from .._units.ids import FrameIds from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter @@ -35,12 +34,11 @@ class ChromiumFrame(ChromiumBase): self.tab = page.tab if 'ChromiumFrame' in page_type else page self.address = page.address - node = page.run_cdp('DOM.describeNode', backendNodeId=ele.ids.backend_id)['node'] + node = page.run_cdp('DOM.describeNode', backendNodeId=ele._backend_id)['node'] self._tab_id = page.tab_id - self._backend_id = ele.ids.backend_id + self._backend_id = ele._backend_id self._frame_ele = ele self._states = None - self._ids = FrameIds(self) self._is_init_get_doc = True self._frame_id = node['frameId'] @@ -114,7 +112,7 @@ class ChromiumFrame(ChromiumBase): end_time = perf_counter() + 2 while perf_counter() < end_time: node = self._target_page.run_cdp('DOM.describeNode', - backendNodeId=self._frame_ele.ids.backend_id)['node'] + backendNodeId=self._frame_ele._backend_id)['node'] if 'frameId' in node: break @@ -189,7 +187,7 @@ class ChromiumFrame(ChromiumBase): while perf_counter() < end_time: try: if self._is_diff_domain is False: - node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self.ids.backend_id)['node'] + node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self._backend_id)['node'] self.doc_ele = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) @@ -304,12 +302,18 @@ class ChromiumFrame(ChromiumBase): # ----------挂件---------- @property - def page(self): - return self._page + def _obj_id(self): + """返回frame元素的object id""" + return self.frame_ele._obj_id @property - def ids(self): - return self._ids + def _node_id(self): + """返回cdp中的node id""" + return self.frame_ele._node_id + + @property + def page(self): + return self._page @property def frame_ele(self): @@ -330,7 +334,7 @@ class ChromiumFrame(ChromiumBase): def html(self): """返回元素outerHTML文本""" tag = self.tag - out_html = self._target_page.run_cdp('DOM.getOuterHTML', backendNodeId=self.frame_ele.ids.backend_id)[ + out_html = self._target_page.run_cdp('DOM.getOuterHTML', backendNodeId=self.frame_ele._backend_id)[ 'outerHTML'] sign = search(rf'<{tag}.*?>', out_html).group(0) return f'{sign}{self.inner_html}' @@ -391,7 +395,7 @@ class ChromiumFrame(ChromiumBase): return self.doc_ele.run_js('return this.readyState;') except ContextLossError: try: - node = self.run_cdp('DOM.describeNode', backendNodeId=self.frame_ele.ids.backend_id)['node'] + node = self.run_cdp('DOM.describeNode', backendNodeId=self.frame_ele._backend_id)['node'] doc = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) return doc.run_js('return this.readyState;') except: diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 7c93fd0..465f6e7 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -12,7 +12,6 @@ from .chromium_tab import ChromiumTab from .web_page import WebPage from .._elements.chromium_element import ChromiumElement from .._units.states import FrameStates -from .._units.ids import FrameIds from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter @@ -32,12 +31,10 @@ class ChromiumFrame(ChromiumBase): self._is_diff_domain: bool = ... self.doc_ele: ChromiumElement = ... self._states: FrameStates = ... - self._ids: FrameIds = ... self._is_init_get_doc: bool = ... self._rect: FrameRect = ... - def __call__(self, - loc_or_str: Union[Tuple[str, str], str], + def __call__(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> Union[ChromiumElement, str]: ... def _check_alive(self) -> None: ... @@ -61,9 +58,6 @@ class ChromiumFrame(ChromiumBase): @property def page(self) -> Union[ChromiumPage, WebPage]: ... - @property - def ids(self) -> FrameIds: ... - @property def frame_ele(self) -> ChromiumElement: ... @@ -91,6 +85,12 @@ class ChromiumFrame(ChromiumBase): @property def rect(self) -> FrameRect: ... + @property + def _obj_id(self) -> str: ... + + @property + def _node_id(self) -> str: ... + @property def active_ele(self) -> ChromiumElement: ... @@ -128,52 +128,36 @@ class ChromiumFrame(ChromiumBase): def parent(self, level_or_loc: Union[tuple, str, int] = 1, index: int = 1) -> Union[ChromiumElement, None]: ... - def prev(self, filter_loc: Union[tuple, str, int] = '', - index: int = 1, - timeout: float = 0, - ele_only: bool = True) -> Union[ChromiumElement, str]: ... + def prev(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, + timeout: float = 0, ele_only: bool = True) -> Union[ChromiumElement, str]: ... - def next(self, filter_loc: Union[tuple, str, int] = '', - index: int = 1, - timeout: float = 0, - ele_only: bool = True) -> Union[ChromiumElement, str]: ... + def next(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, + timeout: float = 0, ele_only: bool = True) -> Union[ChromiumElement, str]: ... - def before(self, filter_loc: Union[tuple, str, int] = '', - index: int = 1, - timeout: float = None, - ele_only: bool = True) -> Union[ChromiumElement, str]: ... + def before(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, + timeout: float = None, ele_only: bool = True) -> Union[ChromiumElement, str]: ... - def after(self, filter_loc: Union[tuple, str, int] = '', - index: int = 1, - timeout: float = None, - ele_only: bool = True) -> Union[ChromiumElement, str]: ... + def after(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, + timeout: float = None, ele_only: bool = True) -> Union[ChromiumElement, str]: ... - def prevs(self, filter_loc: Union[tuple, str] = '', - timeout: float = 0, + def prevs(self, filter_loc: Union[tuple, str] = '', timeout: float = 0, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def nexts(self, filter_loc: Union[tuple, str] = '', - timeout: float = 0, + def nexts(self, filter_loc: Union[tuple, str] = '', timeout: float = 0, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def befores(self, filter_loc: Union[tuple, str] = '', - timeout: float = None, + def befores(self, filter_loc: Union[tuple, str] = '', timeout: float = None, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def afters(self, filter_loc: Union[tuple, str] = '', - timeout: float = None, + def afters(self, filter_loc: Union[tuple, str] = '', timeout: float = None, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def get_screenshot(self, path: [str, Path] = None, name: str = None, - as_bytes: [bool, str] = None, + def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, as_base64: [bool, str] = None) -> Union[str, bytes]: ... - def _get_screenshot(self, path: [str, Path] = None, name: str = None, - as_bytes: [bool, str] = None, as_base64: [bool, str] = None, - full_page: bool = False, - left_top: Tuple[int, int] = None, - right_bottom: Tuple[int, int] = None, - ele: ChromiumElement = None) -> Union[str, bytes]: ... + def _get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, + as_base64: [bool, str] = None, full_page: bool = False, left_top: Tuple[int, int] = None, + right_bottom: Tuple[int, int] = None, ele: ChromiumElement = None) -> Union[str, bytes]: ... def _find_elements(self, loc_or_ele: Union[Tuple[str, str], str, ChromiumElement, ChromiumFrame], timeout: float = None, single: bool = True, relative: bool = False, raise_err: bool = None) \ diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index aae2461..643da4c 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -78,7 +78,7 @@ class Clicker(object): try: r = self._ele.page.run_cdp('DOM.getNodeForLocation', x=x, y=y, includeUserAgentShadowDOM=True, ignorePointerEventsNone=True) - if r['backendNodeId'] != self._ele.ids.backend_id: + if r['backendNodeId'] != self._ele._backend_id: vx, vy = self._ele.rect.viewport_midpoint else: vx, vy = self._ele.rect.viewport_click_point diff --git a/DrissionPage/_units/ids.py b/DrissionPage/_units/ids.py deleted file mode 100644 index f5fb8d8..0000000 --- a/DrissionPage/_units/ids.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding:utf-8 -*- -""" -@Author : g1879 -@Contact : g1879@qq.com -""" - - -class ShadowRootIds(object): - def __init__(self, ele): - self._ele = ele - - @property - def node_id(self): - """返回元素cdp中的node id""" - return self._ele._node_id - - @property - def obj_id(self): - """返回元素js中的object id""" - return self._ele._obj_id - - @property - def backend_id(self): - """返回backend id""" - return self._ele._backend_id - - -class ElementIds(ShadowRootIds): - @property - def doc_id(self): - """返回所在document的object id""" - return self._ele._doc_id - - -class FrameIds(object): - def __init__(self, frame): - self._frame = frame - - @property - def tab_id(self): - """返回当前标签页id""" - return self._frame._tab_id - - @property - def backend_id(self): - """返回cdp中的node id""" - return self._frame._backend_id - - @property - def obj_id(self): - """返回frame元素的object id""" - return self._frame.frame_ele.ids.obj_id - - @property - def node_id(self): - """返回cdp中的node id""" - return self._frame.frame_ele.ids.node_id diff --git a/DrissionPage/_units/ids.pyi b/DrissionPage/_units/ids.pyi deleted file mode 100644 index 907005d..0000000 --- a/DrissionPage/_units/ids.pyi +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding:utf-8 -*- -""" -@Author : g1879 -@Contact : g1879@qq.com -""" -from typing import Union - -from .._elements.chromium_element import ChromiumElement, ChromiumShadowRoot -from .._pages.chromium_frame import ChromiumFrame - - -class ShadowRootIds(object): - def __init__(self, ele: Union[ChromiumElement, ChromiumShadowRoot]): - self._ele: Union[ChromiumElement, ChromiumShadowRoot] = ... - - @property - def node_id(self) -> str: ... - - @property - def obj_id(self) -> str: ... - - @property - def backend_id(self) -> str: ... - - -class ElementIds(ShadowRootIds): - @property - def doc_id(self) -> str: ... - - -class FrameIds(object): - def __init__(self, frame: ChromiumFrame): - self._frame: ChromiumFrame = ... - - @property - def tab_id(self) -> str: ... - - @property - def backend_id(self) -> str: ... - - @property - def obj_id(self) -> str: ... - - @property - def node_id(self) -> str: ... diff --git a/DrissionPage/_units/rect.py b/DrissionPage/_units/rect.py index 4180b14..0b9f0f0 100644 --- a/DrissionPage/_units/rect.py +++ b/DrissionPage/_units/rect.py @@ -99,7 +99,7 @@ class ElementRect(object): :param quad: 方框类型，margin border padding :return: 四个角坐标 """ - return self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele.ids.backend_id)['model'][quad] + return self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele._backend_id)['model'][quad] def _get_page_coord(self, x, y): """根据视口坐标获取绝对坐标""" diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index a1ee590..af58a07 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -413,7 +413,7 @@ class ChromiumElementSetter(object): :param value: 属性值 :return: None """ - self._ele.page.run_cdp('DOM.setAttributeValue', nodeId=self._ele.ids.node_id, name=attr, value=str(value)) + self._ele.page.run_cdp('DOM.setAttributeValue', nodeId=self._ele._node_id, name=attr, value=str(value)) def prop(self, prop, value): """设置元素property属性 @@ -508,14 +508,14 @@ class WindowSetter(object): self._page = page self._window_id = self._get_info()['windowId'] - def maximized(self): + def max(self): """窗口最大化""" s = self._get_info()['bounds']['windowState'] if s in ('fullscreen', 'minimized'): self._perform({'windowState': 'normal'}) self._perform({'windowState': 'maximized'}) - def minimized(self): + def mini(self): """窗口最小化""" s = self._get_info()['bounds']['windowState'] if s == 'fullscreen': @@ -575,6 +575,16 @@ class WindowSetter(object): """ self._page.run_cdp('Browser.setWindowBounds', windowId=self._window_id, bounds=bounds) + # ------------即将废除---------- + + def maximized(self): + """窗口最大化""" + self.max() + + def minimized(self): + """窗口最小化""" + self.mini() + class PageWindowSetter(WindowSetter): def hide(self): diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index cf2f73e..15db2b7 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -191,9 +191,9 @@ class WindowSetter(object): self._page: ChromiumBase = ... self._window_id: str = ... - def maximized(self) -> None: ... + def max(self) -> None: ... - def minimized(self) -> None: ... + def mini(self) -> None: ... def fullscreen(self) -> None: ... diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index 0bf571b..2e3214f 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -68,7 +68,7 @@ class ElementStates(object): except CDPError: return False - if r.get('backendNodeId') != self._ele.ids.backend_id: + if r.get('backendNodeId') != self._ele._backend_id: return True return False @@ -98,7 +98,7 @@ class ShadowRootStates(object): def is_alive(self): """返回元素是否仍在DOM中""" try: - self._ele.page.run_cdp('DOM.describeNode', backendNodeId=self._ele.ids.backend_id) + self._ele.page.run_cdp('DOM.describeNode', backendNodeId=self._ele._backend_id) return True except Exception: return False @@ -145,7 +145,7 @@ class FrameStates(object): """返回frame元素是否可用，且里面仍挂载有frame""" try: node = self._frame._target_page.run_cdp('DOM.describeNode', - backendNodeId=self._frame._frame_ele.ids.backend_id)['node'] + backendNodeId=self._frame._frame_ele._backend_id)['node'] except (ElementLossError, PageClosedError): return False return 'frameId' in node From 727d850df3df520acfc5c3f2dee8631e116c5c0b Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 17 Nov 2023 17:56:02 +0800 Subject: [PATCH 091/182] =?UTF-8?q?=E5=BE=AE=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_commons/locator.py | 4 ++-- DrissionPage/_pages/chromium_base.py | 7 ++----- DrissionPage/_pages/chromium_tab.py | 4 ++-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/DrissionPage/_commons/locator.py b/DrissionPage/_commons/locator.py index 9daeb16..3e9662b 100644 --- a/DrissionPage/_commons/locator.py +++ b/DrissionPage/_commons/locator.py @@ -8,8 +8,8 @@ from .by import By def get_loc(loc, translate_css=False): - """接收selenium定位元组或本库定位语法，转换为标准定位元组，可翻译css selector为xpath - :param loc: selenium定位元组或本库定位语法 + """接收本库定位语法或selenium定位元组，转换为标准定位元组，可翻译css selector为xpath + :param loc: 本库定位语法或selenium定位元组 :param translate_css: 是否翻译css selector为xpath :return: DrissionPage定位元组 """ diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index d60e0b2..8d01ca5 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -501,11 +501,8 @@ class ChromiumBase(BasePage): :return: 目标url是否可用 """ retry, interval = self._before_connect(url, retry, interval) - self._url_available = self._d_connect(self._url, - times=retry, - interval=interval, - show_errmsg=show_errmsg, - timeout=timeout) + self._url_available = self._d_connect(self._url, times=retry, interval=interval, + show_errmsg=show_errmsg, timeout=timeout) return self._url_available def get_cookies(self, as_dict=False, all_domains=False, all_info=False): diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index c67600a..0e5a13d 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -299,8 +299,8 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): return if copy_user_agent: - selenium_user_agent = self.run_cdp('Runtime.evaluate', expression='navigator.userAgent;')['result']['value'] - self.session.headers.update({"User-Agent": selenium_user_agent}) + user_agent = self.run_cdp('Runtime.evaluate', expression='navigator.userAgent;')['result']['value'] + self.session.headers.update({"User-Agent": user_agent}) set_session_cookies(self.session, super(SessionPage, self).get_cookies()) From b62cb110c4de938a67c6d85e3e1459277058f5f7 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 17 Nov 2023 22:48:04 +0800 Subject: [PATCH 092/182] =?UTF-8?q?SessionOptions=E5=A2=9E=E5=8A=A0from=5F?= =?UTF-8?q?session()=EF=BC=9B=E4=BF=AE=E5=A4=8D=E6=97=A0ini=E6=97=B6WebPag?= =?UTF-8?q?e=E7=9A=84get=5Ftab()=E6=8A=A5=E9=94=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_configs/session_options.py | 77 ++++++++++++++--------- DrissionPage/_configs/session_options.pyi | 2 + DrissionPage/_pages/chromium_tab.py | 3 +- DrissionPage/_pages/session_page.py | 5 +- DrissionPage/_pages/session_page.pyi | 7 +-- 5 files changed, 56 insertions(+), 38 deletions(-) diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index d918f63..ed8f808 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -22,6 +22,9 @@ class SessionOptions(object): """ self.ini_path = None self._download_path = '' + self._timeout = 10 + self._del_set = set() # 记录要从ini文件删除的参数 + self._headers = None self._cookies = None self._auth = None @@ -34,46 +37,45 @@ class SessionOptions(object): self._stream = None self._trust_env = None self._max_redirects = None - self._timeout = 10 - self._del_set = set() # 记录要从ini文件删除的参数 + if read_file is False: + return - if read_file is not False: - ini_path = str(ini_path) if ini_path else None - om = OptionsManager(ini_path) - self.ini_path = om.ini_path - options_dict = om.session_options + ini_path = str(ini_path) if ini_path else None + om = OptionsManager(ini_path) + self.ini_path = om.ini_path + options_dict = om.session_options - if options_dict.get('headers', None) is not None: - self.set_headers(options_dict['headers']) + if options_dict.get('headers', None) is not None: + self.set_headers(options_dict['headers']) - if options_dict.get('cookies', None) is not None: - self.set_cookies(options_dict['cookies']) + if options_dict.get('cookies', None) is not None: + self.set_cookies(options_dict['cookies']) - if options_dict.get('auth', None) is not None: - self._auth = options_dict['auth'] + if options_dict.get('auth', None) is not None: + self._auth = options_dict['auth'] - if options_dict.get('params', None) is not None: - self._params = options_dict['params'] + if options_dict.get('params', None) is not None: + self._params = options_dict['params'] - if options_dict.get('verify', None) is not None: - self._verify = options_dict['verify'] + if options_dict.get('verify', None) is not None: + self._verify = options_dict['verify'] - if options_dict.get('cert', None) is not None: - self._cert = options_dict['cert'] + if options_dict.get('cert', None) is not None: + self._cert = options_dict['cert'] - if options_dict.get('stream', None) is not None: - self._stream = options_dict['stream'] + if options_dict.get('stream', None) is not None: + self._stream = options_dict['stream'] - if options_dict.get('trust_env', None) is not None: - self._trust_env = options_dict['trust_env'] + if options_dict.get('trust_env', None) is not None: + self._trust_env = options_dict['trust_env'] - if options_dict.get('max_redirects', None) is not None: - self._max_redirects = options_dict['max_redirects'] + if options_dict.get('max_redirects', None) is not None: + self._max_redirects = options_dict['max_redirects'] - self.set_proxies(om.proxies.get('http', None), om.proxies.get('https', None)) - self._timeout = om.timeouts.get('implicit', 10) - self._download_path = om.paths.get('download_path', '') + self.set_proxies(om.proxies.get('http', None), om.proxies.get('https', None)) + self._timeout = om.timeouts.get('implicit', 10) + self._download_path = om.paths.get('download_path', '') # ===========须独立处理的项开始============ @property @@ -397,6 +399,25 @@ class SessionOptions(object): return s + def from_session(self, session): + """从Session对象中读取配置 + :param session: Session对象 + :return: 当前对象 + """ + self._headers = session.headers + self._cookies = session.cookies + self._auth = session.auth + self._proxies = session.proxies + self._hooks = session.hooks + self._params = session.params + self._verify = session.verify + self._cert = session.cert + self._adapters = session.adapters + self._stream = session.stream + self._trust_env = session.trust_env + self._max_redirects = session.max_redirects + return self + def session_options_to_dict(options): """把session配置对象转换为字典 diff --git a/DrissionPage/_configs/session_options.pyi b/DrissionPage/_configs/session_options.pyi index 3c0ae72..9caf336 100644 --- a/DrissionPage/_configs/session_options.pyi +++ b/DrissionPage/_configs/session_options.pyi @@ -115,5 +115,7 @@ class SessionOptions(object): def make_session(self) -> Session: ... + def from_session(self, session: Session) -> SessionOptions: ... + def session_options_to_dict(options: Union[dict, SessionOptions, None]) -> Union[dict, None]: ... diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 0e5a13d..55227b8 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -7,6 +7,7 @@ from copy import copy from .._base.base import BasePage from .._commons.web import set_session_cookies, set_browser_cookies +from .._configs.session_options import SessionOptions from .._pages.chromium_base import ChromiumBase from .._pages.session_page import SessionPage from .._units.setter import TabSetter, WebPageTabSetter @@ -67,7 +68,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): self._mode = 'd' self._has_driver = True self._has_session = True - super().__init__(session_or_options=copy(page.session)) + super().__init__(session_or_options=SessionOptions(read_file=False).from_session(copy(page.session))) super(SessionPage, self).__init__(page=page, tab_id=tab_id) def __call__(self, loc_or_str, timeout=None): diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index 5a92319..e135e3c 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -30,16 +30,15 @@ class SessionPage(BasePage): self._response = None self._session = None self._set = None - self._s_set_start_options(session_or_options, None) + self._s_set_start_options(session_or_options) self._s_set_runtime_settings() self._create_session() if timeout is not None: self.timeout = timeout - def _s_set_start_options(self, session_or_options, none): + def _s_set_start_options(self, session_or_options): """启动配置 :param session_or_options: Session、SessionOptions - :param none: 用于后代继承 :return: None """ if not session_or_options or isinstance(session_or_options, SessionOptions): diff --git a/DrissionPage/_pages/session_page.pyi b/DrissionPage/_pages/session_page.pyi index 22f6963..98d4e91 100644 --- a/DrissionPage/_pages/session_page.pyi +++ b/DrissionPage/_pages/session_page.pyi @@ -23,15 +23,13 @@ class SessionPage(BasePage): self._session_options: SessionOptions = ... self._url: str = ... self._response: Response = ... - # self._download_path: str = ... - # self._DownloadKit: DownloadKit = ... self._url_available: bool = ... self.timeout: float = ... self.retry_times: int = ... self.retry_interval: float = ... self._set: SessionPageSetter = ... - def _s_set_start_options(self, session_or_options, none) -> None: ... + def _s_set_start_options(self, session_or_options: Union[Session, SessionOptions]) -> None: ... def _s_set_runtime_settings(self) -> None: ... @@ -114,9 +112,6 @@ class SessionPage(BasePage): @property def set(self) -> SessionPageSetter: ... - # @property - # def download(self) -> DownloadKit: ... - def post(self, url: str, data: Union[dict, str, None] = ..., From 066eadc7e045a1599fd4e6915de49bd7afe13733 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 19 Nov 2023 19:49:01 +0800 Subject: [PATCH 093/182] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E9=9A=90=E7=A7=81=E5=A3=B0=E6=98=8E=E9=80=BB=E8=BE=91=EF=BC=9B?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E8=AE=BE=E7=BD=AE=E5=AE=9E=E9=AA=8C=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_commons/browser.py | 46 +++++++++++++++++++--- DrissionPage/_commons/browser.pyi | 3 ++ DrissionPage/_configs/chromium_options.py | 27 ++++++++++++- DrissionPage/_configs/chromium_options.pyi | 9 +++++ DrissionPage/_configs/configs.ini | 1 + DrissionPage/_pages/chromium_base.py | 19 +++++++-- setup.py | 2 +- 8 files changed, 97 insertions(+), 12 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 5b6d740..46a2a1d 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b8' +__version__ = '4.0.0b9' diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_commons/browser.py index c0a5e80..8e3d6aa 100644 --- a/DrissionPage/_commons/browser.py +++ b/DrissionPage/_commons/browser.py @@ -37,6 +37,7 @@ def connect_browser(option): # ----------创建浏览器进程---------- args = get_launch_args(option) set_prefs(option) + set_flags(option) try: _run_browser(port, chrome_path, args) @@ -89,6 +90,7 @@ def get_launch_args(opt): port = opt.debugger_address.split(':')[-1] if opt.debugger_address else '0' path = Path(gettempdir()) / 'DrissionPage' / f'userData_{port}' path.mkdir(parents=True, exist_ok=True) + opt.set_user_data_path(path) result.add(f'--user-data-dir={path}') if not remote_allow: @@ -119,15 +121,13 @@ def set_prefs(opt): :param opt: ChromiumOptions :return: None """ + if not opt.user_data_path or (not opt.preferences and not opt._prefs_to_del): + return prefs = opt.preferences del_list = opt._prefs_to_del - if not opt.user_data_path: - return - - args = opt.arguments user = 'Default' - for arg in args: + for arg in opt.arguments: if arg.startswith('--profile-directory'): user = arg.split('=')[-1].strip() break @@ -158,6 +158,42 @@ def set_prefs(opt): dump(prefs_dict, f) +def set_flags(opt): + """处理启动配置中的prefs项，目前只能对已存在文件夹配置 + :param opt: ChromiumOptions + :return: None + """ + if not opt.user_data_path or (not opt.clear_file_flags and not opt.flags): + return + + state_file = Path(opt.user_data_path) / 'Local State' + + if not state_file.exists(): + state_file.parent.mkdir(parents=True, exist_ok=True) + with open(state_file, 'w') as f: + f.write('{}') + + with open(state_file, "r", encoding='utf-8') as f: + try: + states_dict = load(f) + except JSONDecodeError: + states_dict = {} + flags_list = [] if opt.clear_file_flags else states_dict.setdefault( + 'browser', {}).setdefault('enabled_labs_experiments', []) + flags_dict = {} + for i in flags_list: + f = str(i).split('@', 1) + flags_dict[f[0]] = None if len(f) == 1 else f[1] + + for k, i in opt.flags.items(): + flags_dict[k] = i + + states_dict['browser']['enabled_labs_experiments'] = [f'{k}@{i}' if i else k for k, i in flags_dict.items()] + + with open(state_file, 'w', encoding='utf-8') as f: + dump(states_dict, f) + + def test_connect(ip, port, timeout=30): """测试浏览器是否可用 :param ip: 浏览器ip diff --git a/DrissionPage/_commons/browser.pyi b/DrissionPage/_commons/browser.pyi index 9e74e87..76a2008 100644 --- a/DrissionPage/_commons/browser.pyi +++ b/DrissionPage/_commons/browser.pyi @@ -17,4 +17,7 @@ def get_launch_args(opt: ChromiumOptions) -> list: ... def set_prefs(opt: ChromiumOptions) -> None: ... +def set_flags(opt: ChromiumOptions) -> None: ... + + def test_connect(ip: str, port: Union[int, str], timeout: float = 30) -> None: ... diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index cdcf8c8..5993f7b 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -21,6 +21,7 @@ class ChromiumOptions(object): self._user_data_path = None self._user = 'Default' self._prefs_to_del = [] + self.clear_file_flags = False self._headless = None if read_file is not False: @@ -34,6 +35,7 @@ class ChromiumOptions(object): self._browser_path = options.get('browser_path', '') self._extensions = options.get('extensions', []) self._prefs = options.get('prefs', {}) + self._flags = options.get('flags', {}) self._debugger_address = options.get('debugger_address', None) self._load_mode = options.get('load_mode', 'normal') self._proxy = om.proxies.get('http', None) @@ -69,6 +71,7 @@ class ChromiumOptions(object): self._download_path = '' self._extensions = [] self._prefs = {} + self._flags = {} self._timeouts = {'implicit': 10, 'pageLoad': 30, 'script': 30} self._debugger_address = '127.0.0.1:9222' self._load_mode = 'normal' @@ -137,6 +140,11 @@ class ChromiumOptions(object): """返回用户首选项配置""" return self._prefs + @property + def flags(self): + """返回实验项配置""" + return self._flags + @property def system_user_path(self): """返回是否使用系统安装的浏览器所使用的用户数据文件夹""" @@ -221,6 +229,23 @@ class ChromiumOptions(object): self._prefs_to_del.append(arg) return self + def set_flag(self, flag, value=None): + """设置实验项 + :param flag: 设置项名称 + :param value: 设置项的值，为False则删除该项 + :return: 当前对象 + """ + if value is False: + self._flags.pop(flag, None) + else: + self._flags[flag] = value + return self + + def clear_flags_in_file(self): + """删除浏览器设置文件中已设置的实验项""" + self.clear_file_flags = True + return self + def set_timeouts(self, implicit=None, pageLoad=None, script=None): """设置超时时间，单位为秒 :param implicit: 默认超时时间 @@ -454,7 +479,7 @@ class ChromiumOptions(object): # 设置chrome_options attrs = ('debugger_address', 'browser_path', 'arguments', 'extensions', 'user', 'load_mode', - 'auto_port', 'system_user_path', 'existing_only') + 'auto_port', 'system_user_path', 'existing_only', 'flags') for i in attrs: om.set_item('chrome_options', i, self.__getattribute__(f'_{i}')) # 设置代理 diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index e42c6de..76f8624 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -22,7 +22,9 @@ class ChromiumOptions(object): self._debugger_address: str = ... self._extensions: list = ... self._prefs: dict = ... + self._flags: dict = ... self._prefs_to_del: list = ... + self.clear_file_flags: bool = ... self._auto_port: bool = ... self._system_user_path: bool = ... self._existing_only: bool = ... @@ -61,6 +63,9 @@ class ChromiumOptions(object): @property def preferences(self) -> dict: ... + @property + def flags(self) -> dict: ... + @property def system_user_path(self) -> bool: ... @@ -81,6 +86,10 @@ class ChromiumOptions(object): def remove_pref_from_file(self, arg: str) -> ChromiumOptions: ... + def set_flag(self, flag: str, value: Union[int, str, bool] = None) -> ChromiumOptions: ... + + def clear_flags_in_file(self) -> ChromiumOptions: ... + def set_timeouts(self, implicit: float = None, pageLoad: float = None, script: float = None) -> ChromiumOptions: ... diff --git a/DrissionPage/_configs/configs.ini b/DrissionPage/_configs/configs.ini index e6cb8f5..62e1746 100644 --- a/DrissionPage/_configs/configs.ini +++ b/DrissionPage/_configs/configs.ini @@ -7,6 +7,7 @@ browser_path = chrome arguments = ['--remote-allow-origins=*', '--no-first-run', '--disable-infobars', '--disable-popup-blocking'] extensions = [] prefs = {'profile.default_content_settings.popups': 0, 'profile.default_content_setting_values': {'notifications': 2}} +flags = {} load_mode = normal user = Default auto_port = False diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 8d01ca5..a345f4f 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -10,7 +10,6 @@ from re import findall from threading import Thread from time import perf_counter, sleep -from .._units.rect import TabRect from .._base.base import BasePage from .._commons.constants import ERROR, NoneElement from .._commons.locator import get_loc @@ -20,6 +19,7 @@ from .._elements.chromium_element import ChromiumElement, run_js, make_chromium_ from .._elements.session_element import make_session_ele from .._units.action_chains import ActionChains from .._units.network_listener import NetworkListener +from .._units.rect import TabRect from .._units.screencast import Screencast from .._units.scroller import PageScroller from .._units.setter import ChromiumBaseSetter @@ -88,9 +88,20 @@ class ChromiumBase(BasePage): if not tab_id: tabs = self.browser.driver.get(f'http://{self.address}/json').json() tabs = [(i['id'], i['url']) for i in tabs if i['type'] == 'page' and not i['url'].startswith('devtools://')] - if tabs[0][1] == 'chrome://privacy-sandbox-dialog/notice' and len(tabs) > 1: - tab_id = tabs[1][0] - close_privacy_dialog(self, tabs[0][0]) + dialog = None + if len(tabs) > 1: + for k, t in enumerate(tabs): + if t[1] == 'chrome://privacy-sandbox-dialog/notice': + dialog = k + elif not tab_id: + tab_id = t[0] + + if tab_id and dialog is not None: + break + + if dialog is not None: + close_privacy_dialog(self, tabs[dialog][0]) + else: tab_id = tabs[0][0] diff --git a/setup.py b/setup.py index 77ed58a..ee2e434 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b8", + version="4.0.0b9", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From e52d14a9621c79d9b6c089e6c4e8878780e24ae0 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 20 Nov 2023 00:21:54 +0800 Subject: [PATCH 094/182] =?UTF-8?q?=E6=89=BE=E4=B8=8D=E5=88=B0=E5=85=83?= =?UTF-8?q?=E7=B4=A0=E6=97=B6=E6=98=BE=E7=A4=BA=E6=96=B9=E6=B3=95=E5=92=8C?= =?UTF-8?q?=E5=8F=82=E6=95=B0=EF=BC=9B=E9=93=BE=E5=BC=8F=E6=9F=A5=E6=89=BE?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E6=97=B6=E5=8F=AF=E8=BF=94=E5=9B=9E=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/base.py | 69 +++++++++++++-------- DrissionPage/_base/base.pyi | 10 +-- DrissionPage/_commons/constants.py | 28 +-------- DrissionPage/_commons/locator.py | 6 ++ DrissionPage/_commons/locator.pyi | 3 + DrissionPage/_elements/chromium_element.py | 47 +++++++++----- DrissionPage/_elements/chromium_element.pyi | 2 +- DrissionPage/_elements/none_element.py | 42 +++++++++++++ DrissionPage/_elements/session_element.py | 6 +- DrissionPage/_elements/session_element.pyi | 2 +- DrissionPage/_pages/chromium_base.py | 41 +++++++----- DrissionPage/_pages/chromium_base.pyi | 4 +- DrissionPage/_pages/session_page.py | 4 +- DrissionPage/_pages/session_page.pyi | 2 +- DrissionPage/errors.py | 19 ++++-- 15 files changed, 181 insertions(+), 104 deletions(-) create mode 100644 DrissionPage/_elements/none_element.py diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index eea2c3e..993699e 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -10,9 +10,10 @@ from urllib.parse import quote from DownloadKit import DownloadKit -from .._commons.constants import Settings, NoneElement +from .._commons.constants import Settings from .._commons.locator import get_loc from .._commons.web import format_html +from .._elements.none_element import NoneElement from ..errors import ElementNotFoundError @@ -23,7 +24,7 @@ class BaseParser(object): return self.ele(loc_or_str) def ele(self, loc_or_ele, timeout=None): - return self._ele(loc_or_ele, timeout, True) + return self._ele(loc_or_ele, timeout, True, method='ele()') def eles(self, loc_or_str, timeout=None): return self._ele(loc_or_str, timeout, False) @@ -39,7 +40,7 @@ class BaseParser(object): def s_eles(self, loc_or_str): pass - def _ele(self, loc_or_ele, timeout=None, single=True, raise_err=None): + def _ele(self, loc_or_ele, timeout=None, single=True, raise_err=None, method=None): pass @abstractmethod @@ -67,12 +68,16 @@ class BaseElement(BaseParser): def nexts(self): pass - def _ele(self, loc_or_str, timeout=None, single=True, relative=False, raise_err=None): + def _ele(self, loc_or_str, timeout=None, single=True, relative=False, raise_err=None, method=None): r = self._find_elements(loc_or_str, timeout=timeout, single=single, relative=relative, raise_err=raise_err) - if not single or raise_err is False: + if not single: return r - if not r and (Settings.raise_when_ele_not_found or raise_err is True): - raise ElementNotFoundError + if isinstance(r, NoneElement): + if Settings.raise_when_ele_not_found or raise_err is True: + raise ElementNotFoundError(None, method, {'loc_or_str': loc_or_str}) + else: + r.method = method + r.args = {'loc_or_str': loc_or_str} return r @abstractmethod @@ -136,7 +141,7 @@ class DrissionElement(BaseElement): else: raise TypeError('level_or_loc参数只能是tuple、int或str。') - return self._ele(loc, timeout=0, relative=True, raise_err=False) + return self._ele(loc, timeout=0, relative=True, raise_err=False, method='parent()') def child(self, filter_loc='', index=1, timeout=None, ele_only=True): """返回直接子元素元素或节点组成的列表，可用查询语法筛选 @@ -152,17 +157,19 @@ class DrissionElement(BaseElement): nodes = self.children(filter_loc=filter_loc, timeout=timeout, ele_only=ele_only) if not nodes: if Settings.raise_when_ele_not_found: - raise ElementNotFoundError + raise ElementNotFoundError(None, 'child()', + {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) else: - return NoneElement() + return NoneElement('child()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) try: return nodes[index - 1] except IndexError: if Settings.raise_when_ele_not_found: - raise ElementNotFoundError + raise ElementNotFoundError(None, 'child()', + {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) else: - return NoneElement() + return NoneElement('child()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) def prev(self, filter_loc='', index=1, timeout=0, ele_only=True): """返回前面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -179,9 +186,10 @@ class DrissionElement(BaseElement): if nodes: return nodes[-1] if Settings.raise_when_ele_not_found: - raise ElementNotFoundError + raise ElementNotFoundError(None, 'prev()', + {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) else: - return NoneElement() + return NoneElement('prev()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) def next(self, filter_loc='', index=1, timeout=0, ele_only=True): """返回后面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -198,9 +206,10 @@ class DrissionElement(BaseElement): if nodes: return nodes[0] if Settings.raise_when_ele_not_found: - raise ElementNotFoundError + raise ElementNotFoundError(None, 'next()', + {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) else: - return NoneElement() + return NoneElement('next()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) def before(self, filter_loc='', index=1, timeout=None, ele_only=True): """返回前面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -217,9 +226,10 @@ class DrissionElement(BaseElement): if nodes: return nodes[-1] if Settings.raise_when_ele_not_found: - raise ElementNotFoundError + raise ElementNotFoundError(None, 'before()', + {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) else: - return NoneElement() + return NoneElement('before()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) def after(self, filter_loc='', index=1, timeout=None, ele_only=True): """返回后面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -236,9 +246,10 @@ class DrissionElement(BaseElement): if nodes: return nodes[0] if Settings.raise_when_ele_not_found: - raise ElementNotFoundError + raise ElementNotFoundError(None, 'after()', + {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) else: - return NoneElement() + return NoneElement('after()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) def children(self, filter_loc='', timeout=None, ele_only=True): """返回直接子元素元素或节点组成的列表，可用查询语法筛选 @@ -372,7 +383,7 @@ class BasePage(BaseParser): @property def title(self): """返回网页title""" - ele = self._ele('xpath://title', raise_err=False) + ele = self._ele('xpath://title', raise_err=False, method='title') return ele.text if ele else None @property @@ -430,7 +441,7 @@ class BasePage(BaseParser): @property def user_agent(self): - pass + return @abstractmethod def get_cookies(self, as_dict=False, all_info=False): @@ -440,16 +451,20 @@ class BasePage(BaseParser): def get(self, url, show_errmsg=False, retry=None, interval=None): pass - def _ele(self, loc_or_ele, timeout=None, single=True, raise_err=None): + def _ele(self, loc_or_ele, timeout=None, single=True, raise_err=None, method=None): if not loc_or_ele: - raise ElementNotFoundError + raise ElementNotFoundError(None, method, {'loc_or_str': loc_or_ele}) r = self._find_elements(loc_or_ele, timeout=timeout, single=single, raise_err=raise_err) - if not single or raise_err is False: + if not single: return r - if not r and (Settings.raise_when_ele_not_found is True or raise_err is True): - raise ElementNotFoundError + if isinstance(r, NoneElement): + if Settings.raise_when_ele_not_found or raise_err is True: + raise ElementNotFoundError(None, method, {'loc_or_str': loc_or_ele}) + else: + r.method = method + r.args = {'loc_or_str': loc_or_ele} return r @abstractmethod diff --git a/DrissionPage/_base/base.pyi b/DrissionPage/_base/base.pyi index 2b9228c..20b793e 100644 --- a/DrissionPage/_base/base.pyi +++ b/DrissionPage/_base/base.pyi @@ -8,7 +8,7 @@ from typing import Union, Tuple, List from DownloadKit import DownloadKit -from .._commons.constants import NoneElement +from .._elements.none_element import NoneElement class BaseParser(object): @@ -27,7 +27,8 @@ class BaseParser(object): def s_eles(self, loc_or_str: Union[Tuple[str, str], str]): ... - def _ele(self, loc_or_ele, timeout: float = None, single: bool = True, raise_err: bool = None): ... + def _ele(self, loc_or_ele, timeout: float = None, single: bool = True, + raise_err: bool = None, method: str = None): ... @abstractmethod def _find_elements(self, loc_or_ele, timeout: float = None, single: bool = True, raise_err: bool = None): ... @@ -43,7 +44,7 @@ class BaseElement(BaseParser): def tag(self) -> str: ... def _ele(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None, single: bool = True, - relative: bool = False, raise_err: bool = None): ... + relative: bool = False, raise_err: bool = None, method: str = None): ... @abstractmethod def _find_elements(self, loc_or_str, timeout: float = None, single: bool = True, relative: bool = False, @@ -208,7 +209,8 @@ class BasePage(BaseParser): retry: int = None, interval: float = None): ... - def _ele(self, loc_or_ele, timeout: float = None, single: bool = True, raise_err: bool = None): ... + def _ele(self, loc_or_ele, timeout: float = None, single: bool = True, + raise_err: bool = None, method: str = None): ... @abstractmethod def _find_elements(self, loc_or_ele, timeout: float = None, single: bool = True, raise_err: bool = None): ... diff --git a/DrissionPage/_commons/constants.py b/DrissionPage/_commons/constants.py index a1c8054..4208282 100644 --- a/DrissionPage/_commons/constants.py +++ b/DrissionPage/_commons/constants.py @@ -3,8 +3,6 @@ @Author : g1879 @Contact : g1879@qq.com """ -from ..errors import ElementNotFoundError - FRAME_ELEMENT = ('iframe', 'frame') ERROR = 'error' @@ -13,28 +11,4 @@ class Settings(object): raise_when_ele_not_found = False raise_when_click_failed = False raise_when_wait_failed = False - - -class NoneElement(object): - _instance = None - - def __new__(cls): - if not cls._instance: - cls._instance = super(NoneElement, cls).__new__(cls) - return cls._instance - - def __call__(self, *args, **kwargs): - raise ElementNotFoundError - - def __getattr__(self, item): - raise ElementNotFoundError - - def __eq__(self, other): - if other is None: - return True - - def __bool__(self): - return False - - def __repr__(self): - return 'None' + NoneElement_value = None diff --git a/DrissionPage/_commons/locator.py b/DrissionPage/_commons/locator.py index 3e9662b..393559b 100644 --- a/DrissionPage/_commons/locator.py +++ b/DrissionPage/_commons/locator.py @@ -7,6 +7,12 @@ from re import split from .by import By +def is_loc(text): + """返回text是否定位符""" + return text.startswith(('.', '#', '@', 't:', 't=', 'tag:', 'tag=', 'tx:', 'tx=', 'tx^', 'tx$', 'text:', 'text=', + 'text^', 'text$', 'xpath:', 'xpath=', 'x:', 'x=', 'css:', 'css=', 'c:', 'c=')) + + def get_loc(loc, translate_css=False): """接收本库定位语法或selenium定位元组，转换为标准定位元组，可翻译css selector为xpath :param loc: 本库定位语法或selenium定位元组 diff --git a/DrissionPage/_commons/locator.pyi b/DrissionPage/_commons/locator.pyi index 1038890..ec6a426 100644 --- a/DrissionPage/_commons/locator.pyi +++ b/DrissionPage/_commons/locator.pyi @@ -6,6 +6,9 @@ from typing import Union +def is_loc(text:str) -> bool: ... + + def get_loc(loc: Union[tuple, str], translate_css: bool = False) -> tuple: ... diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index e70bb8d..3e1bc4e 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -7,9 +7,10 @@ from os.path import basename, sep from pathlib import Path from time import perf_counter, sleep +from .none_element import NoneElement from .session_element import make_session_ele from .._base.base import DrissionElement, BaseElement -from .._commons.constants import FRAME_ELEMENT, NoneElement, Settings +from .._commons.constants import FRAME_ELEMENT, Settings from .._commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions from .._commons.locator import get_loc from .._commons.tools import get_usable_path @@ -386,7 +387,7 @@ class ChromiumElement(DrissionElement): :param timeout: 查找元素超时时间，默认与元素所在页面等待时间一致 :return: ChromiumElement对象或属性、文本 """ - return self._ele(loc_or_str, timeout) + return self._ele(loc_or_str, timeout, method='ele()') def eles(self, loc_or_str, timeout=None): """返回当前元素下级所有符合条件的子元素、属性或节点文本 @@ -402,8 +403,16 @@ class ChromiumElement(DrissionElement): :return: SessionElement对象或属性、文本 """ if self.tag in FRAME_ELEMENT: - return make_session_ele(self.inner_html, loc_or_str) - return make_session_ele(self, loc_or_str) + r = make_session_ele(self.inner_html, loc_or_str) + else: + r = make_session_ele(self, loc_or_str) + if isinstance(r, NoneElement): + if Settings.raise_when_ele_not_found: + raise ElementNotFoundError(None, 's_ele()', {'loc_or_str': loc_or_str}) + else: + r.method = 's_ele()' + r.args = {'loc_or_str': loc_or_str} + return r def s_eles(self, loc_or_str=None): """查找所有符合条件的元素，以SessionElement列表形式返回 @@ -847,7 +856,7 @@ class ChromiumShadowRoot(BaseElement): else: raise TypeError('level_or_loc参数只能是tuple、int或str。') - return self.parent_ele._ele(loc, timeout=0, relative=True, raise_err=False) + return self.parent_ele._ele(loc, timeout=0, relative=True, raise_err=False, method='parent()') def child(self, filter_loc='', index=1): """返回直接子元素元素或节点组成的列表，可用查询语法筛选 @@ -858,17 +867,17 @@ class ChromiumShadowRoot(BaseElement): nodes = self.children(filter_loc=filter_loc) if not nodes: if Settings.raise_when_ele_not_found: - raise ElementNotFoundError + raise ElementNotFoundError(None, 'child()', {'filter_loc': filter_loc, 'index': index}) else: - return NoneElement() + return NoneElement('child()', {'filter_loc': filter_loc, 'index': index}) try: return nodes[index - 1] except IndexError: if Settings.raise_when_ele_not_found: - raise ElementNotFoundError + raise ElementNotFoundError(None, 'child()', {'filter_loc': filter_loc, 'index': index}) else: - return NoneElement() + return NoneElement('child()', {'filter_loc': filter_loc, 'index': index}) def next(self, filter_loc='', index=1): """返回当前元素后面一个符合条件的同级元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -880,9 +889,9 @@ class ChromiumShadowRoot(BaseElement): if nodes: return nodes[index - 1] if Settings.raise_when_ele_not_found: - raise ElementNotFoundError + raise ElementNotFoundError(None, 'next()', {'filter_loc': filter_loc, 'index': index}) else: - return NoneElement() + return NoneElement('next()', {'filter_loc': filter_loc, 'index': index}) def before(self, filter_loc='', index=1): """返回文档中当前元素前面符合条件的第一个元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -895,9 +904,9 @@ class ChromiumShadowRoot(BaseElement): if nodes: return nodes[index - 1] if Settings.raise_when_ele_not_found: - raise ElementNotFoundError + raise ElementNotFoundError(None, 'before()', {'filter_loc': filter_loc, 'index': index}) else: - return NoneElement() + return NoneElement('before()', {'filter_loc': filter_loc, 'index': index}) def after(self, filter_loc='', index=1): """返回文档中此当前元素后面符合条件的第一个元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -910,9 +919,9 @@ class ChromiumShadowRoot(BaseElement): if nodes: return nodes[index - 1] if Settings.raise_when_ele_not_found: - raise ElementNotFoundError + raise ElementNotFoundError(None, 'after()', {'filter_loc': filter_loc, 'index': index}) else: - return NoneElement() + return NoneElement('after()', {'filter_loc': filter_loc, 'index': index}) def children(self, filter_loc=''): """返回当前元素符合条件的直接子元素或节点组成的列表，可用查询语法筛选 @@ -974,7 +983,7 @@ class ChromiumShadowRoot(BaseElement): :param timeout: 查找元素超时时间，默认与元素所在页面等待时间一致 :return: ChromiumElement对象 """ - return self._ele(loc_or_str, timeout) + return self._ele(loc_or_str, timeout, method='ele()') def eles(self, loc_or_str, timeout=None): """返回当前元素下级所有符合条件的子元素 @@ -989,7 +998,11 @@ class ChromiumShadowRoot(BaseElement): :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 :return: SessionElement对象或属性、文本 """ - return make_session_ele(self, loc_or_str) + r = make_session_ele(self, loc_or_str) + if isinstance(r, NoneElement): + r.method = 's_ele()' + r.args = {'loc_or_str': loc_or_str} + return r def s_eles(self, loc_or_str): """查找所有符合条件的元素以SessionElement列表形式返回，处理复杂页面时效率很高 diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 04bd049..9bb0cb4 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -6,8 +6,8 @@ from pathlib import Path from typing import Union, Tuple, List, Any +from .none_element import NoneElement from .._base.base import DrissionElement, BaseElement -from .._commons.constants import NoneElement from .._elements.session_element import SessionElement from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame diff --git a/DrissionPage/_elements/none_element.py b/DrissionPage/_elements/none_element.py new file mode 100644 index 0000000..ceb43ce --- /dev/null +++ b/DrissionPage/_elements/none_element.py @@ -0,0 +1,42 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" +from .._commons.constants import Settings +from errors import ElementNotFoundError + + +class NoneElement(object): + def __init__(self, method=None, args=None): + self.method = method + self.args = args + + def __call__(self, *args, **kwargs): + if Settings.NoneElement_value is None: + raise ElementNotFoundError(None, self.method, self.args) + else: + return self + + def __getattr__(self, item): + if Settings.NoneElement_value is None: + raise ElementNotFoundError(None, self.method, self.args) + elif item in ('ele', 's_ele', 'parent', 'child', 'next', 'prev', 'before', + 'after', 'get_frame', 'shadow_root', 'sr'): + return self + else: + if item in ('size', 'link', 'css_path', 'xpath', 'comments', 'texts', 'tag', 'html', 'inner_html', + 'attrs', 'text', 'raw_text'): + return Settings.NoneElement_value + else: + raise ElementNotFoundError(None, self.method, self.args) + + def __eq__(self, other): + if other is None: + return True + + def __bool__(self): + return False + + def __repr__(self): + return 'None' diff --git a/DrissionPage/_elements/session_element.py b/DrissionPage/_elements/session_element.py index a3d74f5..177ae96 100644 --- a/DrissionPage/_elements/session_element.py +++ b/DrissionPage/_elements/session_element.py @@ -9,8 +9,8 @@ from re import match, DOTALL from lxml.etree import tostring from lxml.html import HtmlElement, fromstring +from .none_element import NoneElement from .._base.base import DrissionElement, BasePage, BaseElement -from .._commons.constants import NoneElement from .._commons.locator import get_loc from .._commons.web import get_ele_txt, make_absolute_link @@ -221,7 +221,7 @@ class SessionElement(DrissionElement): :param timeout: 不起实际作用 :return: SessionElement对象或属性、文本 """ - return self._ele(loc_or_str) + return self._ele(loc_or_str, method='ele()') def eles(self, loc_or_str, timeout=None): """返回当前元素下级所有符合条件的子元素、属性或节点文本 @@ -236,7 +236,7 @@ class SessionElement(DrissionElement): :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 :return: SessionElement对象或属性、文本 """ - return self._ele(loc_or_str) + return self._ele(loc_or_str, method='s_ele()') def s_eles(self, loc_or_str): """返回当前元素下级所有符合条件的子元素、属性或节点文本 diff --git a/DrissionPage/_elements/session_element.pyi b/DrissionPage/_elements/session_element.pyi index 6d3efc9..40a8909 100644 --- a/DrissionPage/_elements/session_element.pyi +++ b/DrissionPage/_elements/session_element.pyi @@ -7,8 +7,8 @@ from typing import Union, List, Tuple from lxml.html import HtmlElement +from .none_element import NoneElement from .._base.base import DrissionElement, BaseElement -from .._commons.constants import NoneElement from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index a345f4f..34b6ac9 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -11,11 +11,12 @@ from threading import Thread from time import perf_counter, sleep from .._base.base import BasePage -from .._commons.constants import ERROR, NoneElement -from .._commons.locator import get_loc +from .._commons.constants import ERROR, Settings +from .._commons.locator import get_loc, is_loc from .._commons.tools import get_usable_path from .._commons.web import location_in_viewport from .._elements.chromium_element import ChromiumElement, run_js, make_chromium_ele +from .._elements.none_element import NoneElement from .._elements.session_element import make_session_ele from .._units.action_chains import ActionChains from .._units.network_listener import NetworkListener @@ -26,7 +27,7 @@ from .._units.setter import ChromiumBaseSetter from .._units.states import PageStates from .._units.waiter import BaseWaiter from ..errors import (ContextLossError, ElementLossError, CDPError, PageClosedError, NoRectError, AlertExistsError, - GetDocumentError) + GetDocumentError, ElementNotFoundError) class ChromiumBase(BasePage): @@ -540,7 +541,7 @@ class ChromiumBase(BasePage): :param timeout: 查找超时时间 :return: ChromiumElement对象 """ - return self._ele(loc_or_ele, timeout=timeout) + return self._ele(loc_or_ele, timeout=timeout, method='ele()') def eles(self, loc_or_str, timeout=None): """获取所有符合条件的元素对象 @@ -555,7 +556,14 @@ class ChromiumBase(BasePage): :param loc_or_ele: 元素的定位信息，可以是loc元组，或查询字符串 :return: SessionElement对象或属性、文本 """ - return make_session_ele(self, loc_or_ele) + r = make_session_ele(self, loc_or_ele) + if isinstance(r, NoneElement): + if Settings.raise_when_ele_not_found: + raise ElementNotFoundError(None, 's_ele()', {'loc_or_ele': loc_or_ele}) + else: + r.method = 's_ele()' + r.args = {'loc_or_ele': loc_or_ele} + return r def s_eles(self, loc_or_str): """查找所有符合条件的元素以SessionElement列表形式返回 @@ -714,34 +722,39 @@ class ChromiumBase(BasePage): :return: ChromiumFrame对象 """ if isinstance(loc_ind_ele, str): - if not loc_ind_ele.startswith(('.', '#', '@', 't:', 't=', 'tag:', 'tag=', 'tx:', 'tx=', 'tx^', 'tx$', - 'text:', 'text=', 'text^', 'text$', 'xpath:', 'xpath=', 'x:', 'x=', 'css:', - 'css=', 'c:', 'c=')): - loc_ind_ele = f'xpath://*[(name()="iframe" or name()="frame") and ' \ + if not is_loc(loc_ind_ele): + xpath = f'xpath://*[(name()="iframe" or name()="frame") and ' \ f'(@name="{loc_ind_ele}" or @id="{loc_ind_ele}")]' - ele = self._ele(loc_ind_ele, timeout=timeout) + else: + xpath = loc_ind_ele + ele = self._ele(xpath, timeout=timeout) if ele and not str(type(ele)).endswith(".ChromiumFrame'>"): raise TypeError('该定位符不是指向frame元素。') - return ele + r = ele elif isinstance(loc_ind_ele, tuple): ele = self._ele(loc_ind_ele, timeout=timeout) if ele and not str(type(ele)).endswith(".ChromiumFrame'>"): raise TypeError('该定位符不是指向frame元素。') - return ele + r = ele elif isinstance(loc_ind_ele, int): if loc_ind_ele < 1: raise ValueError('序号必须大于0。') xpath = f'xpath:(//*[name()="frame" or name()="iframe"])[{loc_ind_ele}]' - return self._ele(xpath, timeout=timeout) + r = self._ele(xpath, timeout=timeout) elif str(type(loc_ind_ele)).endswith(".ChromiumFrame'>"): - return loc_ind_ele + r = loc_ind_ele else: raise TypeError('必须传入定位符、iframe序号、id、name、ChromiumFrame对象其中之一。') + if isinstance(r, NoneElement): + r.method = 'get_frame()' + r.args = {'loc_ind_ele': loc_ind_ele} + return r + def get_frames(self, loc=None, timeout=None): """获取所有符合条件的frame对象 :param loc: 定位符，为None时返回所有 diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 37f9225..772d54b 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -6,17 +6,17 @@ from pathlib import Path from typing import Union, Tuple, List, Any, Optional -from .._units.rect import TabRect from .._base.base import BasePage from .._base.browser import Browser from .._base.chromium_driver import ChromiumDriver -from .._commons.constants import NoneElement from .._elements.chromium_element import ChromiumElement +from .._elements.none_element import NoneElement from .._elements.session_element import SessionElement from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage from .._units.action_chains import ActionChains from .._units.network_listener import NetworkListener +from .._units.rect import TabRect from .._units.screencast import Screencast from .._units.scroller import Scroller, PageScroller from .._units.setter import ChromiumBaseSetter diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index e135e3c..bd595d0 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -137,7 +137,7 @@ class SessionPage(BasePage): :param timeout: 不起实际作用，用于和ChromiumElement对应，便于无差别调用 :return: SessionElement对象或属性、文本 """ - return self._ele(loc_or_ele) + return self._ele(loc_or_ele, method='ele()') def eles(self, loc_or_str, timeout=None): """返回页面中所有符合条件的元素、属性或节点文本 @@ -152,7 +152,7 @@ class SessionPage(BasePage): :param loc_or_ele: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 :return: SessionElement对象或属性、文本 """ - return make_session_ele(self.html) if loc_or_ele is None else self._ele(loc_or_ele) + return make_session_ele(self.html) if loc_or_ele is None else self._ele(loc_or_ele, method='s_ele()') def s_eles(self, loc_or_str): """返回页面中符合条件的所有元素、属性或节点文本 diff --git a/DrissionPage/_pages/session_page.pyi b/DrissionPage/_pages/session_page.pyi index 98d4e91..4ae4164 100644 --- a/DrissionPage/_pages/session_page.pyi +++ b/DrissionPage/_pages/session_page.pyi @@ -9,8 +9,8 @@ from requests import Session, Response from requests.structures import CaseInsensitiveDict from .._base.base import BasePage -from .._commons.constants import NoneElement from .._configs.session_options import SessionOptions +from .._elements.none_element import NoneElement from .._elements.session_element import SessionElement from .._units.setter import SessionPageSetter diff --git a/DrissionPage/errors.py b/DrissionPage/errors.py index ba515cb..88bb001 100644 --- a/DrissionPage/errors.py +++ b/DrissionPage/errors.py @@ -9,13 +9,26 @@ class BaseError(Exception): _info = None def __init__(self, ErrorInfo=None): - super().__init__(self) # 初始化父类 self._info = ErrorInfo or self._info def __str__(self): return self._info +class ElementNotFoundError(BaseError): + _info = '\n没有找到元素。' + + def __init__(self, ErrorInfo=None, method=None, arguments=None): + super().__init__(ErrorInfo=ErrorInfo) + self.method = method + self.arguments = arguments + + def __str__(self): + method = f'\nmethod: {self.method}' if self.method else '' + arguments = f'\nargs: {self.arguments}' if self.arguments else '' + return f'{self._info}{method}{arguments}' + + class AlertExistsError(BaseError): _info = '存在未处理的提示框。' @@ -36,10 +49,6 @@ class PageClosedError(BaseError): _info = '页面已关闭。' -class ElementNotFoundError(BaseError): - _info = '没有找到元素。' - - class JavaScriptError(BaseError): _info = 'JavaScript运行错误。' From b3d1c54980f365a87e4ee8bd7cf7714c0cd89490 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 20 Nov 2023 14:59:14 +0800 Subject: [PATCH 095/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=87=A0=E4=B8=AA?= =?UTF-8?q?=E5=B0=8F=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_commons/web.py | 9 +++++---- DrissionPage/_commons/web.pyi | 2 +- DrissionPage/_units/network_listener.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/DrissionPage/_commons/web.py b/DrissionPage/_commons/web.py index 9d9c7b5..c2598c6 100644 --- a/DrissionPage/_commons/web.py +++ b/DrissionPage/_commons/web.py @@ -112,16 +112,16 @@ def offset_scroll(ele, offset_x, offset_y): :param offset_y: 偏移量y :return: 视口中的坐标 """ - loc_x, loc_y = ele.location - cp_x, cp_y = ele.locations.click_point + loc_x, loc_y = ele.rect.location + cp_x, cp_y = ele.rect.click_point lx = loc_x + offset_x if offset_x else cp_x ly = loc_y + offset_y if offset_y else cp_y if not location_in_viewport(ele.page, lx, ly): clientWidth = ele.page.run_js('return document.body.clientWidth;') clientHeight = ele.page.run_js('return document.body.clientHeight;') ele.page.scroll.to_location(lx - clientWidth // 2, ly - clientHeight // 2) - cl_x, cl_y = ele.locations.viewport_location - ccp_x, ccp_y = ele.locations.viewport_click_point + cl_x, cl_y = ele.rect.viewport_location + ccp_x, ccp_y = ele.rect.viewport_click_point cx = cl_x + offset_x if offset_x else ccp_x cy = cl_y + offset_y if offset_y else ccp_y return cx, cy @@ -136,6 +136,7 @@ def make_absolute_link(link, page=None): if not link: return link + link = link.strip() parsed = urlparse(link)._asdict() # 是相对路径，与页面url拼接并返回 diff --git a/DrissionPage/_commons/web.pyi b/DrissionPage/_commons/web.pyi index d6b9dc2..7bf9a5f 100644 --- a/DrissionPage/_commons/web.pyi +++ b/DrissionPage/_commons/web.pyi @@ -26,7 +26,7 @@ def location_in_viewport(page: ChromiumBase, loc_x: float, loc_y: float) -> bool def offset_scroll(ele: ChromiumElement, offset_x: float, offset_y: float) -> tuple: ... -def make_absolute_link(link, page: BasePage = None) -> str: ... +def make_absolute_link(link: str, page: BasePage = None) -> str: ... def is_js_func(func: str) -> bool: ... diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index 7ab0df9..a0bf2c0 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -190,7 +190,7 @@ class NetworkListener(object): def _requestWillBeSent(self, **kwargs): """接收到请求时的回调函数""" - if kwargs['frameId'] != self._page._frame_id: + if kwargs.get('frameId', self._page._frame_id) != self._page._frame_id: return p = None if not self._targets: @@ -222,7 +222,7 @@ class NetworkListener(object): def _response_received(self, **kwargs): """接收到返回信息时处理方法""" - if kwargs['frameId'] != self._page._frame_id: + if kwargs.get('frameId', self._page._frame_id) != self._page._frame_id: return request = self._request_ids.get(kwargs['requestId'], None) if request: From 18951def81a3c0ffed220a90999372265036300e Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 20 Nov 2023 22:35:42 +0800 Subject: [PATCH 096/182] =?UTF-8?q?s=E6=A8=A1=E5=BC=8F=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E7=A9=BA=E6=97=B6=E9=87=8D=E8=AF=95=EF=BC=9B=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E4=B8=8D=E5=90=AF=E5=8A=A8=E4=B8=8B=E8=BD=BD=E7=AE=A1=E7=90=86?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=EF=BC=9BSessionOptions=E7=9A=84set=5Fpaths()?= =?UTF-8?q?=E6=94=B9=E6=88=90set=5Fdownload=5Fpath()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/base.py | 3 +-- DrissionPage/_configs/chromium_options.py | 4 +-- DrissionPage/_configs/session_options.py | 22 ++++++++++++----- DrissionPage/_configs/session_options.pyi | 2 +- DrissionPage/_pages/chromium_base.py | 9 +++---- DrissionPage/_pages/chromium_base.pyi | 2 +- DrissionPage/_pages/chromium_page.py | 4 +-- DrissionPage/_pages/session_page.py | 6 +++-- DrissionPage/_units/download_manager.py | 30 ++++++++++++++++------- DrissionPage/_units/download_manager.pyi | 9 +++++-- DrissionPage/_units/setter.py | 6 +++-- 11 files changed, 62 insertions(+), 35 deletions(-) diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index 993699e..7735145 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -4,7 +4,6 @@ @Contact : g1879@qq.com """ from abc import abstractmethod -from pathlib import Path from re import sub from urllib.parse import quote @@ -378,7 +377,7 @@ class BasePage(BaseParser): self.retry_times = 3 self.retry_interval = 2 self._DownloadKit = None - self._download_path = str(Path('.').absolute()) + self._download_path = None @property def title(self): diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index 5993f7b..3f1e962 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -30,7 +30,7 @@ class ChromiumOptions(object): self.ini_path = om.ini_path options = om.chrome_options - self._download_path = om.paths.get('download_path', '') + self._download_path = om.paths.get('download_path', None) or None self._arguments = options.get('arguments', []) self._browser_path = options.get('browser_path', '') self._extensions = options.get('extensions', []) @@ -68,7 +68,7 @@ class ChromiumOptions(object): self.ini_path = None self._browser_path = "chrome" self._arguments = [] - self._download_path = '' + self._download_path = None self._extensions = [] self._prefs = {} self._flags = {} diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index ed8f808..4e7f422 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -21,7 +21,7 @@ class SessionOptions(object): :param ini_path: ini文件路径 """ self.ini_path = None - self._download_path = '' + self._download_path = None self._timeout = 10 self._del_set = set() # 记录要从ini文件删除的参数 @@ -75,7 +75,7 @@ class SessionOptions(object): self.set_proxies(om.proxies.get('http', None), om.proxies.get('https', None)) self._timeout = om.timeouts.get('implicit', 10) - self._download_path = om.paths.get('download_path', '') + self._download_path = om.paths.get('download_path', None) or None # ===========须独立处理的项开始============ @property @@ -83,13 +83,12 @@ class SessionOptions(object): """返回默认下载路径属性信息""" return self._download_path - def set_paths(self, download_path=None): + def set_download_path(self, path=None): """设置默认下载路径 - :param download_path: 下载路径 + :param path: 下载路径 :return: 返回当前对象 """ - if download_path is not None: - self._download_path = str(download_path) + self._download_path = str(path) return self @property @@ -418,6 +417,17 @@ class SessionOptions(object): self._max_redirects = session.max_redirects return self + # --------------即将废弃--------------- + + def set_paths(self, download_path=None): + """设置默认下载路径 + :param download_path: 下载路径 + :return: 返回当前对象 + """ + if download_path is not None: + self._download_path = str(download_path) + return self + def session_options_to_dict(options): """把session配置对象转换为字典 diff --git a/DrissionPage/_configs/session_options.pyi b/DrissionPage/_configs/session_options.pyi index 9caf336..ed3acf8 100644 --- a/DrissionPage/_configs/session_options.pyi +++ b/DrissionPage/_configs/session_options.pyi @@ -34,7 +34,7 @@ class SessionOptions(object): @property def download_path(self) -> str: ... - def set_paths(self, download_path: Union[str, Path]) -> SessionOptions: ... + def set_download_path(self, path: Union[str, Path]) -> SessionOptions: ... @property def timeout(self) -> float: ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 34b6ac9..58bf6ab 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -5,7 +5,6 @@ """ from json import loads, JSONDecodeError from os.path import sep -from pathlib import Path from re import findall from threading import Thread from time import perf_counter, sleep @@ -55,22 +54,20 @@ class ChromiumBase(BasePage): self._scroll = None self._upload_list = None self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc - - self._download_path = str(Path('.').absolute()) + self._download_path = None if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): address = f'127.0.0.1:{address}' - self._d_set_start_options(address, None) + self._d_set_start_options(address) self._d_set_runtime_settings() self._connect_browser(tab_id) if timeout is not None: self.timeout = timeout - def _d_set_start_options(self, address, none): + def _d_set_start_options(self, address): """设置浏览器启动属性 :param address: 'ip:port' - :param none: 用于后代继承 :return: None """ self.address = address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 772d54b..df43bbd 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -83,7 +83,7 @@ class ChromiumBase(BasePage): def _wait_to_stop(self): ... - def _d_set_start_options(self, address, none) -> None: ... + def _d_set_start_options(self, address) -> None: ... def _d_set_runtime_settings(self) -> None: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 6a8b60c..54e2be6 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -87,11 +87,11 @@ class ChromiumPage(ChromiumBase): if self._driver_options.timeouts['implicit'] is not None: self._timeout = self._driver_options.timeouts['implicit'] self._load_mode = self._driver_options.load_mode - self._download_path = str(Path(self._driver_options.download_path).absolute()) + self._download_path = None if self._driver_options.download_path is None \ + else str(Path(self._driver_options.download_path).absolute()) def _page_init(self): """浏览器相关设置""" - self._rect = None self._browser.connect_to_page() # ----------挂件---------- diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index bd595d0..b39637c 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -3,6 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ +from pathlib import Path from re import search from time import sleep from urllib.parse import urlparse @@ -51,7 +52,8 @@ class SessionPage(BasePage): def _s_set_runtime_settings(self): """设置运行时用到的属性""" self._timeout = self._session_options.timeout - self._download_path = self._session_options.download_path + self._download_path = None if self._session_options.download_path is None \ + else str(Path(self._session_options.download_path).absolute()) def _create_session(self): """创建内建Session对象""" @@ -277,7 +279,7 @@ class SessionPage(BasePage): elif mode == 'post': r = self.session.post(url, data=data, **kwargs) - if r: + if r and r.content: return set_charset(r), 'Success' except Exception as e: diff --git a/DrissionPage/_units/download_manager.py b/DrissionPage/_units/download_manager.py index 6a6a9c4..a81df80 100644 --- a/DrissionPage/_units/download_manager.py +++ b/DrissionPage/_units/download_manager.py @@ -27,10 +27,17 @@ class DownloadManager(object): self._tab_missions = {} # {tab_id: DownloadMission} self._flags = {} # {tab_id: [bool, DownloadMission]} - self._browser.driver.set_callback('Browser.downloadProgress', self._onDownloadProgress) - self._browser.driver.set_callback('Browser.downloadWillBegin', self._onDownloadWillBegin) - self._browser.run_cdp('Browser.setDownloadBehavior', downloadPath=self._page.download_path, - behavior='allowAndName', eventsEnabled=True) + if self._page.download_path: + self._browser.driver.set_callback('Browser.downloadProgress', self._onDownloadProgress) + self._browser.driver.set_callback('Browser.downloadWillBegin', self._onDownloadWillBegin) + r = self._browser.run_cdp('Browser.setDownloadBehavior', downloadPath=self._page.download_path, + behavior='allowAndName', eventsEnabled=True) + if 'error' in r: + print('浏览器版本太低无法使用下载管理功能。') + self._running = True + + else: + self._running = False @property def missions(self): @@ -40,13 +47,18 @@ class DownloadManager(object): def set_path(self, tab_id, path): """设置某个tab的下载路径 :param tab_id: tab id - :param path: 下载路径 + :param path: 下载路径（绝对路径str） :return: None """ - TabDownloadSettings(tab_id).path = str(Path(path).absolute()) - if tab_id == self._page.tab_id: - self._browser.run_cdp('Browser.setDownloadBehavior', downloadPath=str(Path(path).absolute()), - behavior='allowAndName', eventsEnabled=True) + TabDownloadSettings(tab_id).path = path + if tab_id == self._page.tab_id or not self._running: + self._browser.driver.set_callback('Browser.downloadProgress', self._onDownloadProgress) + self._browser.driver.set_callback('Browser.downloadWillBegin', self._onDownloadWillBegin) + r = self._browser.run_cdp('Browser.setDownloadBehavior', downloadPath=path, + behavior='allowAndName', eventsEnabled=True) + if 'error' in r: + print('浏览器版本太低无法使用下载管理功能。') + self._running = True def set_rename(self, tab_id, rename=None, suffix=None): """设置某个tab的重命名文件名 diff --git a/DrissionPage/_units/download_manager.pyi b/DrissionPage/_units/download_manager.pyi index 466270b..1617d56 100644 --- a/DrissionPage/_units/download_manager.pyi +++ b/DrissionPage/_units/download_manager.pyi @@ -1,4 +1,8 @@ -from pathlib import Path +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" from typing import Dict, Optional, Union, Literal from .._base.browser import Browser @@ -11,13 +15,14 @@ class DownloadManager(object): _missions: Dict[str, DownloadMission] = ... _tab_missions: dict = ... _flags: dict = ... + _running: bool = ... def __init__(self, browser: Browser): ... @property def missions(self) -> Dict[str, DownloadMission]: ... - def set_path(self, tab_id: str, path: Union[Path, str]) -> None: ... + def set_path(self, tab_id: str, path: str) -> None: ... def set_rename(self, tab_id: str, rename: str = None, suffix: str = None) -> None: ... diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index af58a07..644492c 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -139,7 +139,8 @@ class TabSetter(ChromiumBaseSetter): :param path: 下载路径 :return: None """ - self._page._download_path = str(Path(path).absolute()) + path = str(Path(path).absolute()) + self._page._download_path = path self._page.browser._dl_mgr.set_path(self._page.tab_id, path) if self._page._DownloadKit: self._page._DownloadKit.set.goal_path(path) @@ -209,7 +210,8 @@ class SessionPageSetter(object): :param path: 下载路径 :return: None """ - self._page._download_path = str(Path(path).absolute()) + path = str(Path(path).absolute()) + self._page._download_path = path if self._page._DownloadKit: self._page._DownloadKit.set.goal_path(path) From 62dc68057356d1d83764d3da7eebfdea1b288bbe Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 21 Nov 2023 10:20:02 +0800 Subject: [PATCH 097/182] =?UTF-8?q?4.0.0b9=E5=9C=A8interactive=E6=97=B6get?= =?UTF-8?q?doc=EF=BC=9B=E7=AE=80=E5=8C=96Frame=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 14 ++++-- DrissionPage/_pages/chromium_frame.py | 68 +++++--------------------- DrissionPage/_pages/chromium_frame.pyi | 4 +- DrissionPage/_units/waiter.py | 2 +- 4 files changed, 22 insertions(+), 66 deletions(-) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 58bf6ab..6557701 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -211,9 +211,11 @@ class ChromiumBase(BasePage): print(f'{self._frame_id}触发DomContentEventFired') print('在DomContentEventFired变成interactive') - self._ready_state = 'interactive' if self._load_mode == 'eager': self.run_cdp('Page.stopLoading') + self._get_document() + self._doc_got = True + self._ready_state = 'interactive' if self._debug: print(f'{self._frame_id}执行DomContentEventFired完毕') @@ -224,9 +226,10 @@ class ChromiumBase(BasePage): print(f'{self._frame_id}触发LoadEventFired') print('在LoadEventFired变成complete') + if self._doc_got is False: + self._get_document() + self._doc_got = True self._ready_state = 'complete' - self._get_document() - self._doc_got = True if self._debug: print(f'{self._frame_id}执行LoadEventFired完毕') @@ -234,13 +237,14 @@ class ChromiumBase(BasePage): def _onFrameStoppedLoading(self, **kwargs): """页面加载完成后执行""" self.browser._frames[kwargs['frameId']] = self.tab_id - if kwargs['frameId'] == self._frame_id and self._doc_got is False: + if kwargs['frameId'] == self._frame_id: if self._debug: print(f'{self._frame_id}触发FrameStoppedLoading') print('在FrameStoppedLoading变成complete') + if self._doc_got is False: + self._get_document() self._ready_state = 'complete' - self._get_document() if self._debug: print(f'{self._frame_id}执行FrameStoppedLoading完毕') diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index ec9cea9..681d143 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -34,13 +34,13 @@ class ChromiumFrame(ChromiumBase): self.tab = page.tab if 'ChromiumFrame' in page_type else page self.address = page.address - node = page.run_cdp('DOM.describeNode', backendNodeId=ele._backend_id)['node'] self._tab_id = page.tab_id self._backend_id = ele._backend_id self._frame_ele = ele self._states = None - self._is_init_get_doc = True + self._reloading = False + node = page.run_cdp('DOM.describeNode', backendNodeId=ele._backend_id)['node'] self._frame_id = node['frameId'] if self._is_inner_frame(): self._is_diff_domain = False @@ -78,8 +78,8 @@ class ChromiumFrame(ChromiumBase): self._timeouts = copy(self._target_page.timeouts) self.retry_times = self._target_page.retry_times self.retry_interval = self._target_page.retry_interval - self._load_mode = self._target_page._load_mode if not self._is_diff_domain else 'normal' self._download_path = self._target_page.download_path + self._load_mode = self._target_page._load_mode if not self._is_diff_domain else 'normal' def _driver_init(self, tab_id, is_init=True): """避免出现服务器500错误 @@ -98,7 +98,7 @@ class ChromiumFrame(ChromiumBase): self._is_loading = True debug = self._debug d_debug = self.driver._debug - self._is_init_get_doc = False + self._reloading = True self._doc_got = False if debug: print(f'{self._frame_id} reload 开始') @@ -147,34 +147,12 @@ class ChromiumFrame(ChromiumBase): self.driver._debug = d_debug self._is_loading = False + self._reloading = False if self._debug: print(f'{self._frame_id} reload 完毕') def _get_document(self): - if self._is_reading: - return - self._is_reading = True - end_time = perf_counter() + 10 - while perf_counter() < end_time: - try: - b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] - self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id)['object']['objectId'] - break - except: - continue - else: - raise GetDocumentError - - r = self.run_cdp('Page.getFrameTree') - for i in findall(r"'id': '(.*?)'", str(r)): - self.browser._frames[i] = self.tab_id - - if self._is_init_get_doc: # 阻止reload时标识 - self._is_loading = False - self._is_reading = False - - def _get_new_document(self): """刷新cdp使用的document数据""" if self._is_reading: return @@ -195,7 +173,10 @@ class ChromiumFrame(ChromiumBase): b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] self.doc_ele = ChromiumElement(self, backend_id=b_id) + self._root_id = self.doc_ele._obj_id + break + except: continue @@ -206,39 +187,13 @@ class ChromiumFrame(ChromiumBase): for i in findall(r"'id': '(.*?)'", str(r)): self.browser._frames[i] = self.tab_id - self._is_loading = False + if not self._reloading: # 阻止reload时标识 + self._is_loading = False self._is_reading = False if self._debug: print('>>> new doc got') - def _onLoadEventFired(self, **kwargs): - """在页面刷新、变化后重新读取页面内容""" - if self._debug: - print(f'{self._frame_id}触发在frame的LoadEventFired') - print('在frame的LoadEventFired变成complete') - - self._ready_state = 'complete' - self._get_new_document() - self._doc_got = True - - if self._debug: - print(f'{self._frame_id}执行frame的LoadEventFired完毕') - - def _onFrameStoppedLoading(self, **kwargs): - """页面加载完成后触发""" - self.browser._frames[kwargs['frameId']] = self.tab_id - if kwargs['frameId'] == self._frame_id and self._doc_got is False: - if self._debug: - print(f'{self._frame_id}触发frame的FrameStoppedLoading') - print('在frame的FrameStoppedLoading变成complete') - - self._ready_state = 'complete' - self._get_new_document() - - if self._debug: - print(f'{self._frame_id}执行frame的FrameStoppedLoading完毕') - def _onInspectorDetached(self, **kwargs): """异域转同域或退出""" if self._debug: @@ -334,8 +289,7 @@ class ChromiumFrame(ChromiumBase): def html(self): """返回元素outerHTML文本""" tag = self.tag - out_html = self._target_page.run_cdp('DOM.getOuterHTML', backendNodeId=self.frame_ele._backend_id)[ - 'outerHTML'] + out_html = self._target_page.run_cdp('DOM.getOuterHTML', backendNodeId=self.frame_ele._backend_id)['outerHTML'] sign = search(rf'<{tag}.*?>', out_html).group(0) return f'{sign}{self.inner_html}' diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 465f6e7..9c627b2 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -31,7 +31,7 @@ class ChromiumFrame(ChromiumBase): self._is_diff_domain: bool = ... self.doc_ele: ChromiumElement = ... self._states: FrameStates = ... - self._is_init_get_doc: bool = ... + self._reloading: bool = ... self._rect: FrameRect = ... def __call__(self, loc_or_str: Union[Tuple[str, str], str], @@ -49,8 +49,6 @@ class ChromiumFrame(ChromiumBase): def _get_document(self) -> None: ... - def _get_new_document(self) -> None: ... - def _onFrameStoppedLoading(self, **kwargs): ... def _onInspectorDetached(self, **kwargs): ... diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index cb265d4..9360ba9 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -58,7 +58,7 @@ class BaseWaiter(object): """ ele = self._driver._ele(loc, raise_err=False, timeout=timeout) if ele: - return True + return ele if raise_err is True or Settings.raise_when_wait_failed is True: raise WaitTimeoutError('等待元素加载失败。') else: From 1a0d3c8029a21bd006d76e2f8a19ede2f921da18 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 21 Nov 2023 10:43:50 +0800 Subject: [PATCH 098/182] 4.0.0b10 --- DrissionPage/__init__.py | 2 +- DrissionPage/_elements/none_element.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 46a2a1d..92f5c0d 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b9' +__version__ = '4.0.0b10' diff --git a/DrissionPage/_elements/none_element.py b/DrissionPage/_elements/none_element.py index ceb43ce..96c1555 100644 --- a/DrissionPage/_elements/none_element.py +++ b/DrissionPage/_elements/none_element.py @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from .._commons.constants import Settings -from errors import ElementNotFoundError +from ..errors import ElementNotFoundError class NoneElement(object): diff --git a/setup.py b/setup.py index ee2e434..d2ca9ee 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b9", + version="4.0.0b10", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From 9d51af46ab403b33f07b8d9b960e5083353c7b2f Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 21 Nov 2023 16:35:39 +0800 Subject: [PATCH 099/182] =?UTF-8?q?fix=5Fcount=E6=94=B9=E4=B8=BAfit=5Fcoun?= =?UTF-8?q?t=EF=BC=9Bclick()by=5Fjs=E9=BB=98=E8=AE=A4None?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_units/clicker.py | 6 +++--- DrissionPage/_units/clicker.pyi | 4 ++-- DrissionPage/_units/network_listener.py | 8 ++++---- DrissionPage/_units/network_listener.pyi | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index 643da4c..2f8d425 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -17,7 +17,7 @@ class Clicker(object): """ self._ele = ele - def __call__(self, by_js=False, timeout=2, wait_stop=True): + def __call__(self, by_js=None, timeout=1.5, wait_stop=True): """点击元素 如果遇到遮挡，可选择是否用js点击 :param by_js: 是否用js点击，为None时先用模拟点击，遇到遮挡改用js，为True时直接用js点击，为False时只用模拟点击 @@ -27,7 +27,7 @@ class Clicker(object): """ return self.left(by_js, timeout, wait_stop) - def left(self, by_js=False, timeout=2, wait_stop=True): + def left(self, by_js=None, timeout=1.5, wait_stop=True): """点击元素，可选择是否用js点击 :param by_js: 是否用js点击，为None时先用模拟点击，遇到遮挡改用js，为True时直接用js点击，为False时只用模拟点击 :param timeout: 模拟点击的超时时间，等待元素可见、可用、进入视口 @@ -55,7 +55,7 @@ class Clicker(object): rect = self._ele.states.has_rect sleep(.001) - if wait_stop: + if wait_stop and rect: self._ele.wait.stop_moving(timeout=end_time - perf_counter()) if rect: self._ele.scroll.to_see() diff --git a/DrissionPage/_units/clicker.pyi b/DrissionPage/_units/clicker.pyi index 10a8121..4866ff5 100644 --- a/DrissionPage/_units/clicker.pyi +++ b/DrissionPage/_units/clicker.pyi @@ -12,9 +12,9 @@ class Clicker(object): def __init__(self, ele: ChromiumElement): self._ele: ChromiumElement = ... - def __call__(self, by_js: Union[None, bool] = False, timeout: float = 2, wait_stop: bool = True) -> bool: ... + def __call__(self, by_js: Union[None, bool] = None, timeout: float = 1.5, wait_stop: bool = True) -> bool: ... - def left(self, by_js: Union[None, bool] = False, timeout: float = 2, wait_stop: bool = True) -> bool: ... + def left(self, by_js: Union[None, bool] = None, timeout: float = 1.5, wait_stop: bool = True) -> bool: ... def right(self) -> None: ... diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index a0bf2c0..08c47b5 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -87,12 +87,12 @@ class NetworkListener(object): self._set_callback() - def wait(self, count=1, timeout=None, fix_count=True): + def wait(self, count=1, timeout=None, fit_count=True): """等待符合要求的数据包到达指定数量 :param count: 需要捕捉的数据包数量 :param timeout: 超时时间，为None无限等待 - :param fix_count: 是否必须满足总数要求，发生超时，为True返回False，为False返回已捕捉到的数据包 - :return: count为1时返回数据包对象，大于1时返回列表，超时且fix_count为True时返回False + :param fit_count: 是否必须满足总数要求，发生超时，为True返回False，为False返回已捕捉到的数据包 + :return: count为1时返回数据包对象，大于1时返回列表，超时且fit_count为True时返回False """ if not self.listening: raise RuntimeError('监听未启动或已暂停。') @@ -112,7 +112,7 @@ class NetworkListener(object): break if fail: - if fix_count or not self._caught.qsize(): + if fit_count or not self._caught.qsize(): return False else: return [self._caught.get_nowait() for _ in range(self._caught.qsize())] diff --git a/DrissionPage/_units/network_listener.pyi b/DrissionPage/_units/network_listener.pyi index 081d4e3..8ef48cd 100644 --- a/DrissionPage/_units/network_listener.pyi +++ b/DrissionPage/_units/network_listener.pyi @@ -37,7 +37,7 @@ class NetworkListener(object): def go_on(self) -> None: ... def wait(self, count: int = 1, timeout: float = None, - fix_count: bool = True) -> Union[List[DataPacket], DataPacket, None]: ... + fit_count: bool = True) -> Union[List[DataPacket], DataPacket, None]: ... @property def results(self) -> Union[DataPacket, Dict[str, List[DataPacket]], False]: ... From 11027b01b2f1bfa09d5784daefe3932b5ccbab84 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 21 Nov 2023 17:49:27 +0800 Subject: [PATCH 100/182] =?UTF-8?q?4.0.0b11=E4=BF=AE=E5=A4=8DWebPage()?= =?UTF-8?q?=E7=9A=84get=5Ftab()=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_configs/session_options.py | 3 ++- setup.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 92f5c0d..5e4d74e 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b10' +__version__ = '4.0.0b11' diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index 4e7f422..cd710f6 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -411,10 +411,11 @@ class SessionOptions(object): self._params = session.params self._verify = session.verify self._cert = session.cert - self._adapters = session.adapters self._stream = session.stream self._trust_env = session.trust_env self._max_redirects = session.max_redirects + if session.adapters: + self._adapters = [(k, i) for k, i in session.adapters.items()] return self # --------------即将废弃--------------- diff --git a/setup.py b/setup.py index d2ca9ee..d3d84ce 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b10", + version="4.0.0b11", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From 5fa71a7f0aa14a6aa28941dcf040a6272197c323 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 21 Nov 2023 18:00:15 +0800 Subject: [PATCH 101/182] =?UTF-8?q?click.twice()=E6=94=B9=E4=B8=BAclick.mu?= =?UTF-8?q?ltiple()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_units/clicker.py | 15 ++++++++++++--- DrissionPage/_units/clicker.pyi | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index 2f8d425..c103004 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -124,9 +124,12 @@ class Clicker(object): x, y = offset_scroll(self._ele, offset_x, offset_y) self._click(x, y, button, count) - def twice(self): - """双击元素""" - self.at(count=2) + def multiple(self, times=2): + """多次点击 + :param times: 默认双击 + :return: None + """ + self.at(count=times) def _click(self, client_x, client_y, button='left', count=1): """实施点击 @@ -141,3 +144,9 @@ class Clicker(object): # sleep(.05) self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mouseReleased', x=client_x, y=client_y, button=button) + + # -------------即将废弃-------------- + + def twice(self): + """双击元素""" + self.at(count=2) diff --git a/DrissionPage/_units/clicker.pyi b/DrissionPage/_units/clicker.pyi index 4866ff5..d62b99f 100644 --- a/DrissionPage/_units/clicker.pyi +++ b/DrissionPage/_units/clicker.pyi @@ -22,6 +22,6 @@ class Clicker(object): def at(self, offset_x: float = None, offset_y: float = None, button: str = 'left', count: int = 1) -> None: ... - def twice(self, by_js: bool = False) -> None: ... + def multiple(self, times: int = 2) -> None: ... def _click(self, client_x: float, client_y: float, button: str = 'left', count: int = 1) -> None: ... From 977242ad0ae784c1aef29e014e2375aa1c2a62dc Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 21 Nov 2023 20:21:27 +0800 Subject: [PATCH 102/182] =?UTF-8?q?=E5=88=A0=E9=99=A4easy=5Fset=EF=BC=9B?= =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=A1=B9=E7=9B=AE=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_base/base.py | 2 +- DrissionPage/_commons/browser.py | 96 ++++++- DrissionPage/_commons/browser.pyi | 7 + DrissionPage/_commons/cli.py | 20 +- .../_commons/{constants.py => settings.py} | 2 - DrissionPage/_commons/tools.py | 12 + DrissionPage/_commons/tools.pyi | 3 + DrissionPage/_elements/chromium_element.py | 12 +- DrissionPage/_elements/none_element.py | 2 +- DrissionPage/_pages/chromium_base.py | 10 +- DrissionPage/_units/clicker.py | 2 +- DrissionPage/_units/waiter.py | 2 +- DrissionPage/common.py | 10 +- DrissionPage/common.pyi | 10 - DrissionPage/easy_set.py | 260 ------------------ DrissionPage/easy_set.pyi | 56 ---- setup.py | 2 +- 18 files changed, 158 insertions(+), 352 deletions(-) rename DrissionPage/_commons/{constants.py => settings.py} (82%) delete mode 100644 DrissionPage/common.pyi delete mode 100644 DrissionPage/easy_set.py delete mode 100644 DrissionPage/easy_set.pyi diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 5e4d74e..c4bce69 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b11' +__version__ = '4.0.0b12' diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index 7735145..45a79f4 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -9,7 +9,7 @@ from urllib.parse import quote from DownloadKit import DownloadKit -from .._commons.constants import Settings +from .._commons.settings import Settings from .._commons.locator import get_loc from .._commons.web import format_html from .._elements.none_element import NoneElement diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_commons/browser.py index 8e3d6aa..f06f881 100644 --- a/DrissionPage/_commons/browser.py +++ b/DrissionPage/_commons/browser.py @@ -4,16 +4,19 @@ @Contact : g1879@qq.com """ from json import load, dump, JSONDecodeError +from os import popen from pathlib import Path +from platform import system +from re import search from subprocess import Popen, DEVNULL from tempfile import gettempdir from time import perf_counter, sleep -from platform import system from requests import get as requests_get -from ..errors import BrowserConnectError from .tools import port_is_using +from .._configs.options_manage import OptionsManager +from ..errors import BrowserConnectError def connect_browser(option): @@ -43,7 +46,6 @@ def connect_browser(option): # 传入的路径找不到，主动在ini文件、注册表、系统变量中找 except FileNotFoundError: - from ..easy_set import get_chrome_path chrome_path = get_chrome_path(show_msg=False) if not chrome_path: @@ -282,3 +284,91 @@ def _remove_arg_from_dict(target_dict: dict, arg: str) -> None: exec(src) except: pass + + +def get_chrome_path(ini_path=None, show_msg=True, from_ini=True, + from_regedit=True, from_system_path=True): + """从ini文件或系统变量中获取chrome.exe的路径 + :param ini_path: ini文件路径 + :param show_msg: 是否打印信息 + :param from_ini: 是否从ini文件获取 + :param from_regedit: 是否从注册表获取 + :param from_system_path: 是否从系统路径获取 + :return: chrome.exe路径 + """ + # -----------从ini文件中获取-------------- + if ini_path and from_ini: + try: + path = OptionsManager(ini_path).chrome_options['browser_path'] + except KeyError: + path = None + else: + path = None + + if path and Path(path).is_file(): + if show_msg: + print('ini文件中', end='') + return str(path) + + from platform import system + sys = system().lower() + if sys in ('macos', 'darwin'): + return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' + + elif sys == 'linux': + paths = ('/usr/bin/google-chrome', '/opt/google/chrome/google-chrome', + '/user/lib/chromium-browser/chromium-browser') + for p in paths: + if Path(p).exists(): + return p + return None + + elif sys != 'windows': + return None + + # -----------从注册表中获取-------------- + if from_regedit: + import winreg + try: + key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, + r'SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\chrome.exe', + reserved=0, access=winreg.KEY_READ) + k = winreg.EnumValue(key, 0) + winreg.CloseKey(key) + + if show_msg: + print('注册表中', end='') + + return k[1] + + except FileNotFoundError: + pass + + # -----------从系统变量中获取-------------- + if from_system_path: + try: + paths = popen('set path').read().lower() + except: + return None + r = search(r'[^;]*chrome[^;]*', paths) + + if r: + path = Path(r.group(0)) if 'chrome.exe' in r.group(0) else Path(r.group(0)) / 'chrome.exe' + + if path.exists(): + if show_msg: + print('系统变量中', end='') + return str(path) + + paths = paths.split(';') + + for path in paths: + path = Path(path) / 'chrome.exe' + + try: + if path.exists(): + if show_msg: + print('系统变量中', end='') + return str(path) + except OSError: + pass diff --git a/DrissionPage/_commons/browser.pyi b/DrissionPage/_commons/browser.pyi index 76a2008..b32952b 100644 --- a/DrissionPage/_commons/browser.pyi +++ b/DrissionPage/_commons/browser.pyi @@ -21,3 +21,10 @@ def set_flags(opt: ChromiumOptions) -> None: ... def test_connect(ip: str, port: Union[int, str], timeout: float = 30) -> None: ... + + +def get_chrome_path(ini_path: str = None, + show_msg: bool = True, + from_ini: bool = True, + from_regedit: bool = True, + from_system_path: bool = True, ) -> Union[str, None]: ... diff --git a/DrissionPage/_commons/cli.py b/DrissionPage/_commons/cli.py index 82e991a..ec3bec8 100644 --- a/DrissionPage/_commons/cli.py +++ b/DrissionPage/_commons/cli.py @@ -5,8 +5,9 @@ """ from click import command, option +from .._commons.tools import configs_to_here as ch +from .._configs.chromium_options import ChromiumOptions from .._pages.chromium_page import ChromiumPage -from ..easy_set import set_paths, configs_to_here as ch @command() @@ -29,5 +30,22 @@ def main(set_browser_path, set_user_path, configs_to_here, launch_browser): ChromiumPage(port) +def set_paths(browser_path=None, user_data_path=None): + """快捷的路径设置函数 + :param browser_path: 浏览器可执行文件路径 + :param user_data_path: 用户数据路径 + :return: None + """ + co = ChromiumOptions() + + if browser_path is not None: + co.set_browser_path(browser_path) + + if user_data_path is not None: + co.set_user_data_path(user_data_path) + + co.save() + + if __name__ == '__main__': main() diff --git a/DrissionPage/_commons/constants.py b/DrissionPage/_commons/settings.py similarity index 82% rename from DrissionPage/_commons/constants.py rename to DrissionPage/_commons/settings.py index 4208282..ee5cda9 100644 --- a/DrissionPage/_commons/constants.py +++ b/DrissionPage/_commons/settings.py @@ -3,8 +3,6 @@ @Author : g1879 @Contact : g1879@qq.com """ -FRAME_ELEMENT = ('iframe', 'frame') -ERROR = 'error' class Settings(object): diff --git a/DrissionPage/_commons/tools.py b/DrissionPage/_commons/tools.py index 5e7dd80..813c3d1 100644 --- a/DrissionPage/_commons/tools.py +++ b/DrissionPage/_commons/tools.py @@ -11,6 +11,8 @@ from time import perf_counter, sleep from psutil import process_iter, AccessDenied, NoSuchProcess, ZombieProcess +from .._configs.options_manage import OptionsManager + def get_usable_path(path, is_file=True, parents=True): """检查文件或文件夹是否有重名，并返回可以使用的路径 @@ -238,3 +240,13 @@ def stop_process_on_port(port): pass except Exception as e: print(f"{proc.pid} {port}: {e}") + + +def configs_to_here(save_name=None): + """把默认ini文件复制到当前目录 + :param save_name: 指定文件名，为None则命名为'dp_configs.ini' + :return: None + """ + om = OptionsManager('default') + save_name = f'{save_name}.ini' if save_name is not None else 'dp_configs.ini' + om.save(save_name) diff --git a/DrissionPage/_commons/tools.pyi b/DrissionPage/_commons/tools.pyi index d4294fb..03aae58 100644 --- a/DrissionPage/_commons/tools.pyi +++ b/DrissionPage/_commons/tools.pyi @@ -39,3 +39,6 @@ def wait_until(page, condition: Union[FunctionType, str, tuple], timeout: float, def stop_process_on_port(port: Union[int, str]) -> None: ... + + +def configs_to_here(file_name: Union[Path, str] = None) -> None: ... diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 3e1bc4e..115bdd0 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -10,7 +10,7 @@ from time import perf_counter, sleep from .none_element import NoneElement from .session_element import make_session_ele from .._base.base import DrissionElement, BaseElement -from .._commons.constants import FRAME_ELEMENT, Settings +from .._commons.settings import Settings from .._commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions from .._commons.locator import get_loc from .._commons.tools import get_usable_path @@ -25,6 +25,8 @@ from .._units.waiter import ElementWaiter from ..errors import (ContextLossError, ElementLossError, JavaScriptError, ElementNotFoundError, CDPError, NoResourceError, AlertExistsError) +__FRAME_ELEMENT__ = ('iframe', 'frame') + class ChromiumElement(DrissionElement): """控制浏览器元素的对象""" @@ -402,7 +404,7 @@ class ChromiumElement(DrissionElement): :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 :return: SessionElement对象或属性、文本 """ - if self.tag in FRAME_ELEMENT: + if self.tag in __FRAME_ELEMENT__: r = make_session_ele(self.inner_html, loc_or_str) else: r = make_session_ele(self, loc_or_str) @@ -419,7 +421,7 @@ class ChromiumElement(DrissionElement): :param loc_or_str: 定位符 :return: SessionElement或属性、文本组成的列表 """ - if self.tag in FRAME_ELEMENT: + if self.tag in __FRAME_ELEMENT__: return make_session_ele(self.inner_html, loc_or_str, single=False) return make_session_ele(self, loc_or_str, single=False) @@ -1103,7 +1105,7 @@ def find_by_xpath(ele, xpath, single, timeout, relative=True): :return: ChromiumElement或其组成的列表 """ type_txt = '9' if single else '7' - node_txt = 'this.contentDocument' if ele.tag in FRAME_ELEMENT and not relative else 'this' + node_txt = 'this.contentDocument' if ele.tag in __FRAME_ELEMENT__ and not relative else 'this' js = make_js_for_find_ele_by_xpath(xpath, type_txt, node_txt) r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, returnByValue=False, awaitPromise=True, userGesture=True) @@ -1200,7 +1202,7 @@ def make_chromium_ele(page, node_id=None, obj_id=None): raise ElementLossError ele = ChromiumElement(page, obj_id=obj_id, node_id=node_id, backend_id=backend_id) - if ele.tag in FRAME_ELEMENT: + if ele.tag in __FRAME_ELEMENT__: from .._pages.chromium_frame import ChromiumFrame ele = ChromiumFrame(page, ele) diff --git a/DrissionPage/_elements/none_element.py b/DrissionPage/_elements/none_element.py index 96c1555..0ef09d3 100644 --- a/DrissionPage/_elements/none_element.py +++ b/DrissionPage/_elements/none_element.py @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from .._commons.constants import Settings +from .._commons.settings import Settings from ..errors import ElementNotFoundError diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 6557701..69f5ec7 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -10,7 +10,7 @@ from threading import Thread from time import perf_counter, sleep from .._base.base import BasePage -from .._commons.constants import ERROR, Settings +from .._commons.settings import Settings from .._commons.locator import get_loc, is_loc from .._commons.tools import get_usable_path from .._commons.web import location_in_viewport @@ -28,6 +28,8 @@ from .._units.waiter import BaseWaiter from ..errors import (ContextLossError, ElementLossError, CDPError, PageClosedError, NoRectError, AlertExistsError, GetDocumentError, ElementNotFoundError) +__ERROR__ = 'error' + class ChromiumBase(BasePage): """标签页、frame、页面基类""" @@ -439,10 +441,10 @@ class ChromiumBase(BasePage): :return: 执行的结果 """ r = self.driver.run(cmd, **cmd_args) - if ERROR not in r: + if __ERROR__ not in r: return r - error = r[ERROR] + error = r[__ERROR__] if error in ('Cannot find context with specified id', 'Inspected target navigated or closed'): raise ContextLossError elif error in ('Could not find node with given id', 'Could not find object with given id', @@ -725,7 +727,7 @@ class ChromiumBase(BasePage): if isinstance(loc_ind_ele, str): if not is_loc(loc_ind_ele): xpath = f'xpath://*[(name()="iframe" or name()="frame") and ' \ - f'(@name="{loc_ind_ele}" or @id="{loc_ind_ele}")]' + f'(@name="{loc_ind_ele}" or @id="{loc_ind_ele}")]' else: xpath = loc_ind_ele ele = self._ele(xpath, timeout=timeout) diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index c103004..18359ff 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -5,7 +5,7 @@ """ from time import perf_counter, sleep -from .._commons.constants import Settings +from .._commons.settings import Settings from .._commons.web import offset_scroll from ..errors import CanNotClickError, CDPError, NoRectError diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 9360ba9..3cbca57 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -1,7 +1,7 @@ # -*- coding:utf-8 -*- from time import sleep, perf_counter -from .._commons.constants import Settings +from .._commons.settings import Settings from ..errors import WaitTimeoutError, NoRectError diff --git a/DrissionPage/common.py b/DrissionPage/common.py index 72a0deb..88f9e57 100644 --- a/DrissionPage/common.py +++ b/DrissionPage/common.py @@ -3,11 +3,11 @@ @Author : g1879 @Contact : g1879@qq.com """ +from ._commons.by import By +from ._commons.keys import Keys +from ._commons.settings import Settings +from ._commons.tools import wait_until, configs_to_here from ._elements.session_element import make_session_ele from ._units.action_chains import ActionChains -from ._commons.keys import Keys -from ._commons.by import By -from ._commons.constants import Settings -from ._commons.tools import wait_until -__all__ = ['make_session_ele', 'ActionChains', 'Keys', 'By', 'Settings', 'wait_until'] +__all__ = ['make_session_ele', 'ActionChains', 'Keys', 'By', 'Settings', 'wait_until', 'configs_to_here'] diff --git a/DrissionPage/common.pyi b/DrissionPage/common.pyi deleted file mode 100644 index f1d57c4..0000000 --- a/DrissionPage/common.pyi +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding:utf-8 -*- -""" -@Author : g1879 -@Contact : g1879@qq.com -""" -from ._commons.by import By as By -from ._commons.constants import Settings as Settings -from ._commons.keys import Keys as Keys -from ._elements.session_element import make_session_ele as make_session_ele -from ._units.action_chains import ActionChains as ActionChains diff --git a/DrissionPage/easy_set.py b/DrissionPage/easy_set.py deleted file mode 100644 index 2bf6df5..0000000 --- a/DrissionPage/easy_set.py +++ /dev/null @@ -1,260 +0,0 @@ -# -*- coding:utf-8 -*- -""" -@Author : g1879 -@Contact : g1879@qq.com -""" -from os import popen -from pathlib import Path -from re import search - -from ._configs.chromium_options import ChromiumOptions -from ._configs.options_manage import OptionsManager - - -def configs_to_here(save_name=None): - """把默认ini文件复制到当前目录 - :param save_name: 指定文件名，为None则命名为'dp_configs.ini' - :return: None - """ - om = OptionsManager('default') - save_name = f'{save_name}.ini' if save_name is not None else 'dp_configs.ini' - om.save(save_name) - - -def show_settings(ini_path=None): - """打印ini文件内容 - :param ini_path: ini文件路径 - :return: None - """ - OptionsManager(ini_path).show() - - -def set_paths(browser_path=None, - local_port=None, - debugger_address=None, - download_path=None, - user_data_path=None, - cache_path=None, - ini_path=None): - """快捷的路径设置函数 - :param browser_path: 浏览器可执行文件路径 - :param local_port: 本地端口号 - :param debugger_address: 调试浏览器地址，例：127.0.0.1:9222 - :param download_path: 下载文件路径 - :param user_data_path: 用户数据路径 - :param cache_path: 缓存路径 - :param ini_path: 要修改的ini文件路径 - :return: None - """ - om = OptionsManager(ini_path) - - def format_path(path: str) -> str: - return str(path) if path else '' - - if browser_path is not None: - om.set_item('chrome_options', 'browser_path', format_path(browser_path)) - - if local_port is not None: - om.set_item('chrome_options', 'debugger_address', f'127.0.0.1:{local_port}') - - if debugger_address is not None: - address = debugger_address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') - om.set_item('chrome_options', 'debugger_address', address) - - if download_path is not None: - om.set_item('paths', 'download_path', format_path(download_path)) - - om.save() - - if user_data_path is not None: - set_argument('--user-data-dir', format_path(user_data_path), ini_path) - - if cache_path is not None: - set_argument('--disk-cache-dir', format_path(cache_path), ini_path) - - -def use_auto_port(on_off=True, ini_path=None): - """设置启动浏览器时使用自动分配的端口和临时文件夹 - :param on_off: 是否开启自动端口 - :param ini_path: 要修改的ini文件路径 - :return: None - """ - if not isinstance(on_off, bool): - raise TypeError('on_off参数只能输入bool值。') - om = OptionsManager(ini_path) - om.set_item('chrome_options', 'auto_port', on_off) - om.save() - - -def use_system_user_path(on_off=True, ini_path=None): - """设置是否使用系统安装的浏览器默认用户文件夹 - :param on_off: 开或关 - :param ini_path: 要修改的ini文件路径 - :return: 当前对象 - """ - if not isinstance(on_off, bool): - raise TypeError('on_off参数只能输入bool值。') - om = OptionsManager(ini_path) - om.set_item('chrome_options', 'system_user_path', on_off) - om.save() - - -def set_argument(arg, value=None, ini_path=None): - """设置浏览器配置argument属性 - :param arg: 属性名 - :param value: 属性值，有值的属性传入值，没有的传入None - :param ini_path: 要修改的ini文件路径 - :return: None - """ - co = ChromiumOptions(ini_path=ini_path) - co.set_argument(arg, value) - co.save() - - -def set_headless(on_off=True, ini_path=None): - """设置是否隐藏浏览器界面 - :param on_off: 开或关 - :param ini_path: 要修改的ini文件路径 - :return: None - """ - on_off = 'new' if on_off else False - set_argument('--headless', on_off, ini_path) - - -def set_no_imgs(on_off=True, ini_path=None): - """设置是否禁止加载图片 - :param on_off: 开或关 - :param ini_path: 要修改的ini文件路径 - :return: None - """ - on_off = None if on_off else False - set_argument('--blink-settings=imagesEnabled=false', on_off, ini_path) - - -def set_no_js(on_off=True, ini_path=None): - """设置是否禁用js - :param on_off: 开或关 - :param ini_path: 要修改的ini文件路径 - :return: None - """ - on_off = None if on_off else False - set_argument('--disable-javascript', on_off, ini_path) - - -def set_mute(on_off=True, ini_path=None): - """设置是否静音 - :param on_off: 开或关 - :param ini_path: 要修改的ini文件路径 - :return: None - """ - on_off = None if on_off else False - set_argument('--mute-audio', on_off, ini_path) - - -def set_user_agent(user_agent, ini_path=None): - """设置user agent - :param user_agent: user agent文本 - :param ini_path: 要修改的ini文件路径 - :return: None - """ - set_argument('--user-agent', user_agent, ini_path) - - -def set_proxy(proxy, ini_path=None): - """设置代理 - :param proxy: 代理网址和端口 - :param ini_path: 要修改的ini文件路径 - :return: None - """ - set_argument('--proxy-server', proxy, ini_path) - - -def get_chrome_path(ini_path=None, - show_msg=True, - from_ini=True, - from_regedit=True, - from_system_path=True): - """从ini文件或系统变量中获取chrome.exe的路径 - :param ini_path: ini文件路径 - :param show_msg: 是否打印信息 - :param from_ini: 是否从ini文件获取 - :param from_regedit: 是否从注册表获取 - :param from_system_path: 是否从系统路径获取 - :return: chrome.exe路径 - """ - # -----------从ini文件中获取-------------- - if ini_path and from_ini: - try: - path = OptionsManager(ini_path).chrome_options['browser_path'] - except KeyError: - path = None - else: - path = None - - if path and Path(path).is_file(): - if show_msg: - print('ini文件中', end='') - return str(path) - - from platform import system - sys = system().lower() - if sys in ('macos', 'darwin'): - return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' - - elif sys == 'linux': - paths = ('/usr/bin/google-chrome', '/opt/google/chrome/google-chrome', - '/user/lib/chromium-browser/chromium-browser') - for p in paths: - if Path(p).exists(): - return p - return None - - elif sys != 'windows': - return None - - # -----------从注册表中获取-------------- - if from_regedit: - import winreg - try: - key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, - r'SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\chrome.exe', - reserved=0, access=winreg.KEY_READ) - k = winreg.EnumValue(key, 0) - winreg.CloseKey(key) - - if show_msg: - print('注册表中', end='') - - return k[1] - - except FileNotFoundError: - pass - - # -----------从系统变量中获取-------------- - if from_system_path: - try: - paths = popen('set path').read().lower() - except: - return None - r = search(r'[^;]*chrome[^;]*', paths) - - if r: - path = Path(r.group(0)) if 'chrome.exe' in r.group(0) else Path(r.group(0)) / 'chrome.exe' - - if path.exists(): - if show_msg: - print('系统变量中', end='') - return str(path) - - paths = paths.split(';') - - for path in paths: - path = Path(path) / 'chrome.exe' - - try: - if path.exists(): - if show_msg: - print('系统变量中', end='') - return str(path) - except OSError: - pass diff --git a/DrissionPage/easy_set.pyi b/DrissionPage/easy_set.pyi deleted file mode 100644 index 3e8fc47..0000000 --- a/DrissionPage/easy_set.pyi +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding:utf-8 -*- -""" -@Author : g1879 -@Contact : g1879@qq.com -""" -from pathlib import Path -from typing import Union - - -def configs_to_here(file_name: Union[Path, str] = None) -> None: ... - - -def show_settings(ini_path: Union[str, Path] = None) -> None: ... - - -def set_paths(browser_path: Union[str, Path] = None, - local_port: Union[int, str] = None, - debugger_address: str = None, - download_path: Union[str, Path] = None, - user_data_path: Union[str, Path] = None, - cache_path: Union[str, Path] = None, - ini_path: Union[str, Path] = None) -> None: ... - - -def use_auto_port(on_off: bool = True, ini_path: Union[str, Path] = None) -> None: ... - - -def use_system_user_path(on_off: bool = True, ini_path: Union[str, Path] = None) -> None: ... - - -def set_argument(arg: str, value: Union[bool, str] = None, ini_path: Union[str, Path] = None) -> None: ... - - -def set_headless(on_off: bool = True, ini_path: Union[str, Path] = None) -> None: ... - - -def set_no_imgs(on_off: bool = True, ini_path: Union[str, Path] = None) -> None: ... - - -def set_no_js(on_off: bool = True, ini_path: Union[str, Path] = None) -> None: ... - - -def set_mute(on_off: bool = True, ini_path: Union[str, Path] = None) -> None: ... - - -def set_user_agent(user_agent: str, ini_path: Union[str, Path] = None) -> None: ... - - -def set_proxy(proxy: str, ini_path: Union[str, Path] = None) -> None: ... - - -def get_chrome_path(ini_path: str = None, - show_msg: bool = True, - from_ini: bool = True, - from_regedit: bool = True, - from_system_path: bool = True, ) -> Union[str, None]: ... diff --git a/setup.py b/setup.py index d3d84ce..1bd9af1 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b11", + version="4.0.0b12", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From bd18b8e427cb0b09f5c99f809556c2fc063f573e Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 21 Nov 2023 23:52:43 +0800 Subject: [PATCH 103/182] =?UTF-8?q?=E4=BC=98=E5=8C=96SessionPage=E9=80=9F?= =?UTF-8?q?=E5=BA=A6=EF=BC=9B=E9=A1=B5=E9=9D=A2=E5=AF=B9=E8=B1=A1=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0raw=5Fdata?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_configs/session_options.py | 16 +++++++--------- DrissionPage/_configs/session_options.pyi | 7 ++++--- DrissionPage/_pages/chromium_tab.py | 11 ++++++++++- DrissionPage/_pages/chromium_tab.pyi | 3 +++ DrissionPage/_pages/session_page.py | 12 ++++++++++-- DrissionPage/_pages/session_page.pyi | 6 +++++- DrissionPage/_pages/web_page.py | 8 ++++++++ DrissionPage/_pages/web_page.pyi | 3 +++ DrissionPage/_units/setter.py | 6 +++--- 9 files changed, 53 insertions(+), 19 deletions(-) diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index cd710f6..c702991 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -378,32 +378,30 @@ class SessionOptions(object): return session_options_to_dict(self) def make_session(self): - """根据内在的配置生成Session对象""" + """根据内在的配置生成Session对象，ua从对象中分离""" s = Session() + h = CaseInsensitiveDict(self.headers) if self.headers else CaseInsensitiveDict() - if self.headers: - s.headers = CaseInsensitiveDict(self.headers) if self.cookies: set_session_cookies(s, self.cookies) if self.adapters: for url, adapter in self.adapters: s.mount(url, adapter) - attrs = ['auth', 'proxies', 'hooks', 'params', 'verify', - 'cert', 'stream', 'trust_env', 'max_redirects'] - for i in attrs: + for i in ['auth', 'proxies', 'hooks', 'params', 'verify', 'cert', 'stream', 'trust_env', 'max_redirects']: attr = self.__getattribute__(i) if attr: s.__setattr__(i, attr) - return s + return s, h - def from_session(self, session): + def from_session(self, session, headers=None): """从Session对象中读取配置 :param session: Session对象 + :param headers: headers :return: 当前对象 """ - self._headers = session.headers + self._headers = CaseInsensitiveDict(**session.headers, **headers) if headers else session.headers self._cookies = session.cookies self._auth = session.auth self._proxies = session.proxies diff --git a/DrissionPage/_configs/session_options.pyi b/DrissionPage/_configs/session_options.pyi index ed3acf8..e4fc4cf 100644 --- a/DrissionPage/_configs/session_options.pyi +++ b/DrissionPage/_configs/session_options.pyi @@ -4,12 +4,13 @@ @Contact : g1879@qq.com """ from pathlib import Path -from typing import Any, Union, Tuple +from typing import Any, Union, Tuple, Optional from requests import Session from requests.adapters import HTTPAdapter from requests.auth import HTTPBasicAuth from requests.cookies import RequestsCookieJar +from requests.structures import CaseInsensitiveDict class SessionOptions(object): @@ -113,9 +114,9 @@ class SessionOptions(object): def as_dict(self) -> dict: ... - def make_session(self) -> Session: ... + def make_session(self) -> Tuple[Session, Optional[CaseInsensitiveDict]]: ... - def from_session(self, session: Session) -> SessionOptions: ... + def from_session(self, session: Session, headers: CaseInsensitiveDict = None) -> SessionOptions: ... def session_options_to_dict(options: Union[dict, SessionOptions, None]) -> Union[dict, None]: ... diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 55227b8..fdd9a57 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -68,7 +68,8 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): self._mode = 'd' self._has_driver = True self._has_session = True - super().__init__(session_or_options=SessionOptions(read_file=False).from_session(copy(page.session))) + super().__init__(session_or_options=SessionOptions(read_file=False).from_session(copy(page.session), + page._headers)) super(SessionPage, self).__init__(page=page, tab_id=tab_id) def __call__(self, loc_or_str, timeout=None): @@ -111,6 +112,14 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): elif self._mode == 'd': return super(SessionPage, self).title + @property + def raw_data(self): + """返回页码原始数据数据""" + if self._mode == 's': + return super().raw_data + elif self._mode == 'd': + return super(SessionPage, self).html if self._has_driver else '' + @property def html(self): """返回页面html文本""" diff --git a/DrissionPage/_pages/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi index 687a88c..d00e710 100644 --- a/DrissionPage/_pages/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -65,6 +65,9 @@ class WebPageTab(SessionPage, ChromiumTab): @property def title(self) -> str: ... + @property + def raw_data(self) -> Union[str, bytes]: ... + @property def html(self) -> str: ... diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index b39637c..811b44a 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -28,6 +28,7 @@ class SessionPage(BasePage): :param timeout: 连接超时时间，为None时从ini文件读取 """ super(SessionPage, SessionPage).__init__(self) + self._headers = None self._response = None self._session = None self._set = None @@ -58,7 +59,7 @@ class SessionPage(BasePage): def _create_session(self): """创建内建Session对象""" if not self._session: - self._session = self._session_options.make_session() + self._session, self._headers = self._session_options.make_session() def __call__(self, loc_or_str, timeout=None): """在内部查找元素 @@ -86,6 +87,11 @@ class SessionPage(BasePage): """返回当前访问url""" return self._url + @property + def raw_data(self): + """返回页面原始数据""" + return self.response.content if self.response else b'' + @property def html(self): """返回页面的html文本""" @@ -102,7 +108,7 @@ class SessionPage(BasePage): @property def user_agent(self): """返回user agent""" - return self.session.headers.get('user-agent', '') + return self._headers.get('user-agent', '') @property def session(self): @@ -269,6 +275,8 @@ class SessionPage(BasePage): if not check_headers(kwargs, self.session.headers, 'timeout'): kwargs['timeout'] = self.timeout + kwargs['headers'] = {**self._headers, **kwargs['headers']} + r = err = None retry = retry if retry is not None else self.retry_times interval = interval if interval is not None else self.retry_interval diff --git a/DrissionPage/_pages/session_page.pyi b/DrissionPage/_pages/session_page.pyi index 4ae4164..7a214f1 100644 --- a/DrissionPage/_pages/session_page.pyi +++ b/DrissionPage/_pages/session_page.pyi @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Any, Union, Tuple, List +from typing import Any, Union, Tuple, List, Optional from requests import Session, Response from requests.structures import CaseInsensitiveDict @@ -19,6 +19,7 @@ class SessionPage(BasePage): def __init__(self, session_or_options: Union[Session, SessionOptions] = None, timeout: float = None): + self._headers: Optional[CaseInsensitiveDict] = ... self._session: Session = ... self._session_options: SessionOptions = ... self._url: str = ... @@ -49,6 +50,9 @@ class SessionPage(BasePage): @property def _session_url(self) -> str: ... + @property + def raw_data(self) -> Union[str, bytes]: ... + @property def html(self) -> str: ... diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index a873b3a..98b1da2 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -77,6 +77,14 @@ class WebPage(SessionPage, ChromiumPage, BasePage): elif self._mode == 'd': return super(SessionPage, self).title + @property + def raw_data(self): + """返回页码原始数据数据""" + if self._mode == 's': + return super().raw_data + elif self._mode == 'd': + return super(SessionPage, self).html if self._has_driver else '' + @property def html(self): """返回页面html文本""" diff --git a/DrissionPage/_pages/web_page.pyi b/DrissionPage/_pages/web_page.pyi index 60159d4..8f461d0 100644 --- a/DrissionPage/_pages/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -47,6 +47,9 @@ class WebPage(SessionPage, ChromiumPage, BasePage): @property def title(self) -> str: ... + @property + def raw_data(self) -> Union[str, bytes]: ... + @property def html(self) -> str: ... diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index 644492c..da1d97b 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -244,7 +244,7 @@ class SessionPageSetter(object): :param headers: dict形式的headers :return: None """ - self._page.session.headers = CaseInsensitiveDict(headers) + self._page._headers = CaseInsensitiveDict(headers) def header(self, attr, value): """设置headers中一个项 @@ -252,14 +252,14 @@ class SessionPageSetter(object): :param value: 设置值 :return: None """ - self._page.session.headers[attr.lower()] = value + self._page._headers[attr] = value def user_agent(self, ua): """设置user agent :param ua: user agent :return: None """ - self._page.session.headers['user-agent'] = ua + self._page._headers['user-agent'] = ua def proxies(self, http=None, https=None): """设置proxies参数 From 8699bc82d30a251c99aa7d40ff5760f34aabd9d3 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 22 Nov 2023 15:48:06 +0800 Subject: [PATCH 104/182] =?UTF-8?q?ElementLossError=E6=94=B9=E4=B8=BAEleme?= =?UTF-8?q?ntLostError=EF=BC=9B=E4=BF=AE=E5=A4=8D=5Freload()=E5=B0=8F?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=9BSessionPage=E5=92=8CWebPage=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0close()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 12 ++++++------ DrissionPage/_pages/chromium_base.py | 16 ++++++++-------- DrissionPage/_pages/chromium_frame.py | 22 +++++++++++----------- DrissionPage/_pages/chromium_tab.py | 7 +++++++ DrissionPage/_pages/chromium_tab.pyi | 2 ++ DrissionPage/_pages/session_page.py | 6 ++++++ DrissionPage/_pages/web_page.py | 11 +++++++++++ DrissionPage/_units/states.py | 4 ++-- DrissionPage/errors.py | 4 ++-- 9 files changed, 55 insertions(+), 29 deletions(-) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 115bdd0..6be8334 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -22,7 +22,7 @@ from .._units.select_element import SelectElement from .._units.setter import ChromiumElementSetter from .._units.states import ElementStates, ShadowRootStates from .._units.waiter import ElementWaiter -from ..errors import (ContextLossError, ElementLossError, JavaScriptError, ElementNotFoundError, +from ..errors import (ContextLostError, ElementLostError, JavaScriptError, ElementNotFoundError, CDPError, NoResourceError, AlertExistsError) __FRAME_ELEMENT__ = ('iframe', 'frame') @@ -66,7 +66,7 @@ class ChromiumElement(DrissionElement): self._node_id = self._get_node_id(obj_id=self._obj_id) self._backend_id = backend_id else: - raise ElementLossError + raise ElementLostError doc = self.run_js('return this.ownerDocument;') self._doc_id = doc['objectId'] if doc else None @@ -1199,7 +1199,7 @@ def make_chromium_ele(page, node_id=None, obj_id=None): node_id = node['node']['nodeId'] else: - raise ElementLossError + raise ElementLostError ele = ChromiumElement(page, obj_id=obj_id, node_id=node_id, backend_id=backend_id) if ele.tag in __FRAME_ELEMENT__: @@ -1285,11 +1285,11 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): arguments=[convert_argument(arg) for arg in args], returnByValue=False, awaitPromise=True, userGesture=True, _timeout=timeout) - except ContextLossError: + except ContextLostError: if is_page: - raise ContextLossError('页面已被刷新，请尝试等待页面加载完成再执行操作。') + raise ContextLostError('页面已被刷新，请尝试等待页面加载完成再执行操作。') else: - raise ElementLossError('原来获取到的元素对象已不在页面内。') + raise ElementLostError('原来获取到的元素对象已不在页面内。') if res is None and page.driver.has_alert: # 存在alert的情况 return None diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 69f5ec7..d1e9cc7 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -25,7 +25,7 @@ from .._units.scroller import PageScroller from .._units.setter import ChromiumBaseSetter from .._units.states import PageStates from .._units.waiter import BaseWaiter -from ..errors import (ContextLossError, ElementLossError, CDPError, PageClosedError, NoRectError, AlertExistsError, +from ..errors import (ContextLostError, ElementLostError, CDPError, PageClosedError, NoRectError, AlertExistsError, GetDocumentError, ElementNotFoundError) __ERROR__ = 'error' @@ -429,7 +429,7 @@ class ChromiumBase(BasePage): try: return self.run_cdp('Runtime.evaluate', expression='document.readyState;', _timeout=3)['result']['value'] - except ContextLossError: + except ContextLostError: return None except TimeoutError: return 'timeout' @@ -446,11 +446,11 @@ class ChromiumBase(BasePage): error = r[__ERROR__] if error in ('Cannot find context with specified id', 'Inspected target navigated or closed'): - raise ContextLossError + raise ContextLostError elif error in ('Could not find node with given id', 'Could not find object with given id', 'No node with given id found', 'Node with given id does not belong to the document', 'No node found for given backend id'): - raise ElementLossError + raise ElementLostError elif error == 'tab closed': raise PageClosedError elif error == 'timeout': @@ -600,7 +600,7 @@ class ChromiumBase(BasePage): try: search_result = self.run_cdp_loaded('DOM.performSearch', query=loc, includeUserAgentShadowDOM=True) count = search_result['resultCount'] - except ContextLossError: + except ContextLostError: search_result = None count = 0 @@ -626,13 +626,13 @@ class ChromiumBase(BasePage): r = [make_chromium_ele(self, node_id=i) for i in nodeIds['nodeIds']] break - except ElementLossError: + except ElementLostError: ok = False try: search_result = self.run_cdp_loaded('DOM.performSearch', query=loc, includeUserAgentShadowDOM=True) count = search_result['resultCount'] - except ContextLossError: + except ContextLostError: pass if perf_counter() >= end_time: @@ -1099,7 +1099,7 @@ def close_privacy_dialog(page, tid): """关闭隐私声明弹窗 :param page: ChromiumBase对象 :param tid: tab id - :return: ChromiumDriver对象 + :return: None """ try: driver = page.browser._get_driver(tid) diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 681d143..16460b6 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -14,7 +14,7 @@ from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter from .._units.states import FrameStates from .._units.waiter import FrameWaiter -from ..errors import ContextLossError, ElementLossError, GetDocumentError, PageClosedError +from ..errors import ContextLostError, ElementLostError, GetDocumentError, PageClosedError class ChromiumFrame(ChromiumBase): @@ -106,17 +106,17 @@ class ChromiumFrame(ChromiumBase): self._driver.stop() try: self._frame_ele = ChromiumElement(self._target_page, backend_id=self._backend_id) - except (ElementLossError, PageClosedError): - return + end_time = perf_counter() + 2 + while perf_counter() < end_time: + node = self._target_page.run_cdp('DOM.describeNode', + backendNodeId=self._frame_ele._backend_id)['node'] + if 'frameId' in node: + break - end_time = perf_counter() + 2 - while perf_counter() < end_time: - node = self._target_page.run_cdp('DOM.describeNode', - backendNodeId=self._frame_ele._backend_id)['node'] - if 'frameId' in node: - break + else: + return - else: + except (ElementLostError, PageClosedError): return if self._is_inner_frame(): @@ -347,7 +347,7 @@ class ChromiumFrame(ChromiumBase): else: try: return self.doc_ele.run_js('return this.readyState;') - except ContextLossError: + except ContextLostError: try: node = self.run_cdp('DOM.describeNode', backendNodeId=self.frame_ele._backend_id)['node'] doc = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index fdd9a57..f7a5514 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -332,6 +332,13 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): elif self._mode == 'd': return super(SessionPage, self).get_cookies(as_dict, all_domains, all_info) + def close(self): + """关闭当前标签页""" + self.page.close_tabs(self.tab_id) + self._session.close() + if self._response is not None: + self._response.close() + def _find_elements(self, loc_or_ele, timeout=None, single=True, relative=False, raise_err=None): """返回页面中符合条件的元素、属性或节点文本，默认返回第一个 :param loc_or_ele: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 diff --git a/DrissionPage/_pages/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi index d00e710..29c5c7b 100644 --- a/DrissionPage/_pages/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -140,6 +140,8 @@ class WebPageTab(SessionPage, ChromiumTab): def get_cookies(self, as_dict: bool = False, all_domains: bool = False, all_info: bool = False) -> Union[dict, list]: ... + def close(self) -> None: ... + # ----------------重写SessionPage的函数----------------------- def post(self, url: str, diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index 811b44a..eed198f 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -220,6 +220,12 @@ class SessionPage(BasePage): """ return self._s_connect(url, 'post', data, show_errmsg, retry, interval, **kwargs) + def close(self): + """关闭Session对象""" + self._session.close() + if self._response is not None: + self._response.close() + def _s_connect(self, url, mode, data=None, show_errmsg=False, retry=None, interval=None, **kwargs): """执行get或post连接 :param url: 目标url diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 98b1da2..e17d4ae 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -334,10 +334,21 @@ class WebPage(SessionPage, ChromiumPage, BasePage): if self._has_session: self.change_mode('d') self._session.close() + if self._response is not None: + self._response.close() self._session = None self._response = None self._has_session = None + def close(self): + """关闭标签页""" + if self._has_driver: + self.close_tabs(self.tab_id) + if self._session: + self._session.close() + if self._response is not None: + self._response.close() + def _find_elements(self, loc_or_ele, timeout=None, single=True, relative=False, raise_err=None): """返回页面中符合条件的元素、属性或节点文本，默认返回第一个 :param loc_or_ele: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index 2e3214f..c178d5e 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from .._commons.web import location_in_viewport -from ..errors import CDPError, NoRectError, PageClosedError, ElementLossError +from ..errors import CDPError, NoRectError, PageClosedError, ElementLostError class ElementStates(object): @@ -146,7 +146,7 @@ class FrameStates(object): try: node = self._frame._target_page.run_cdp('DOM.describeNode', backendNodeId=self._frame._frame_ele._backend_id)['node'] - except (ElementLossError, PageClosedError): + except (ElementLostError, PageClosedError): return False return 'frameId' in node diff --git a/DrissionPage/errors.py b/DrissionPage/errors.py index 88bb001..6bd9abd 100644 --- a/DrissionPage/errors.py +++ b/DrissionPage/errors.py @@ -33,11 +33,11 @@ class AlertExistsError(BaseError): _info = '存在未处理的提示框。' -class ContextLossError(BaseError): +class ContextLostError(BaseError): _info = '页面被刷新，请操作前尝试等待页面刷新或加载完成。' -class ElementLossError(BaseError): +class ElementLostError(BaseError): _info = '元素对象因刷新已失效。' From 75f05062fb2ee8d6fb74271cc81be1b381906f74 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 23 Nov 2023 18:05:57 +0800 Subject: [PATCH 105/182] =?UTF-8?q?=E6=94=B9=E8=BF=9Bcss=5Fpath=EF=BC=9B?= =?UTF-8?q?=E4=BF=AE=E5=A4=8Dsr=E4=B8=AD=E5=8F=AF=E8=83=BD=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E9=94=99=E5=85=83=E7=B4=A0=E9=97=AE=E9=A2=98=EF=BC=8C?= =?UTF-8?q?=E6=9C=AA=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 7 +- DrissionPage/_commons/locator.py | 154 ++++++++++++++++++++- DrissionPage/_commons/locator.pyi | 10 +- DrissionPage/_commons/tools.py | 29 +++- DrissionPage/_commons/tools.pyi | 3 + DrissionPage/_elements/chromium_element.py | 51 ++++--- DrissionPage/_elements/session_element.py | 4 +- DrissionPage/_pages/chromium_base.py | 31 +---- 8 files changed, 233 insertions(+), 56 deletions(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 9b9ebdf..addd7e2 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -6,9 +6,11 @@ from time import sleep, perf_counter from .chromium_driver import BrowserDriver, ChromiumDriver -from .._commons.tools import stop_process_on_port +from .._commons.tools import stop_process_on_port, raise_error from .._units.download_manager import DownloadManager +__ERROR__ = 'error' + class Browser(object): BROWSERS = {} @@ -88,7 +90,8 @@ class Browser(object): :param cmd_args: 参数 :return: 执行的结果 """ - return self._driver.run(cmd, **cmd_args) + r = self._driver.run(cmd, **cmd_args) + return r if __ERROR__ not in r else raise_error(r) @property def driver(self): diff --git a/DrissionPage/_commons/locator.py b/DrissionPage/_commons/locator.py index 393559b..f4d9976 100644 --- a/DrissionPage/_commons/locator.py +++ b/DrissionPage/_commons/locator.py @@ -13,17 +13,18 @@ def is_loc(text): 'text^', 'text$', 'xpath:', 'xpath=', 'x:', 'x=', 'css:', 'css=', 'c:', 'c=')) -def get_loc(loc, translate_css=False): +def get_loc(loc, translate_css=False, css_mode=False): """接收本库定位语法或selenium定位元组，转换为标准定位元组，可翻译css selector为xpath :param loc: 本库定位语法或selenium定位元组 :param translate_css: 是否翻译css selector为xpath + :param css_mode: 是否尽量用css selector方式 :return: DrissionPage定位元组 """ if isinstance(loc, tuple): - loc = translate_loc(loc) + loc = translate_css_loc(loc) if css_mode else translate_loc(loc) elif isinstance(loc, str): - loc = str_to_loc(loc) + loc = str_to_css_loc(loc) if css_mode else str_to_loc(loc) else: raise TypeError('loc参数只能是tuple或str。') @@ -127,6 +128,100 @@ def str_to_loc(loc): return loc_by, loc_str +def str_to_css_loc(loc): + """处理元素查找语句 + :param loc: 查找语法字符串 + :return: 匹配符元组 + """ + return str_to_loc(loc) + loc_by = 'css selector' + + if loc.startswith('.'): + if loc.startswith(('.=', '.:', '.^', '.$')): + loc = loc.replace('.', '@class', 1) + else: + loc = loc.replace('.', '@class=', 1) + + elif loc.startswith('#'): + if loc.startswith(('#=', '#:', '#^', '#$')): + loc = loc.replace('#', '@id', 1) + else: + loc = loc.replace('#', '@id=', 1) + + elif loc.startswith(('t:', 't=')): + loc = f'tag:{loc[2:]}' + + elif loc.startswith(('tx:', 'tx=', 'tx^', 'tx$')): + loc = f'text{loc[2:]}' + + # ------------------------------------------------------------------ + # 多属性查找 + if loc.startswith('@@') and loc != '@@': + loc_str = _make_multi_xpath_str('*', loc) + + elif loc.startswith('@|') and loc != '@|': + loc_str = _make_multi_xpath_str('*', loc, False) + + # 单属性查找 + elif loc.startswith('@') and loc != '@': + loc_str = _make_single_xpath_str('*', loc) + + # 根据tag name查找 + elif loc.startswith(('tag:', 'tag=')) and loc not in ('tag:', 'tag='): + at_ind = loc.find('@') + if at_ind == -1: + loc_str = loc[4:] + else: + if loc[at_ind:].startswith('@@'): + loc_str = _make_multi_xpath_str(loc[4:at_ind], loc[at_ind:]) + elif loc[at_ind:].startswith('@|'): + loc_str = _make_multi_xpath_str(loc[4:at_ind], loc[at_ind:], False) + else: + loc_str = _make_single_xpath_str(loc[4:at_ind], loc[at_ind:]) + + # 根据文本查找 + elif loc.startswith('text='): + loc_by = 'xpath' + loc_str = f'//*[text()={_make_search_str(loc[5:])}]' + + elif loc.startswith('text:') and loc != 'text:': + loc_by = 'xpath' + loc_str = f'//*/text()[contains(., {_make_search_str(loc[5:])})]/..' + + elif loc.startswith('text^') and loc != 'text^': + loc_by = 'xpath' + loc_str = f'//*/text()[starts-with(., {_make_search_str(loc[5:])})]/..' + + elif loc.startswith('text$') and loc != 'text$': + loc_by = 'xpath' + loc_str = f'//*/text()[substring(., string-length(.) - string-length({_make_search_str(loc[5:])}) +1) = ' \ + f'{_make_search_str(loc[5:])}]/..' + + # 用xpath查找 + elif loc.startswith(('xpath:', 'xpath=')) and loc not in ('xpath:', 'xpath='): + loc_by = 'xpath' + loc_str = loc[6:] + elif loc.startswith(('x:', 'x=')) and loc not in ('x:', 'x='): + loc_by = 'xpath' + loc_str = loc[2:] + + # 用css selector查找 + elif loc.startswith(('css:', 'css=')) and loc not in ('css:', 'css='): + loc_str = loc[4:] + elif loc.startswith(('c:', 'c=')) and loc not in ('c:', 'c='): + loc_str = loc[2:] + + # 根据文本模糊查找 + elif loc: + loc_by = 'xpath' + loc_str = f'//*/text()[contains(., {_make_search_str(loc)})]/..' + + else: + loc_str = '*' + + return loc_by, loc_str + + def _make_single_xpath_str(tag: str, text: str) -> str: """生成xpath语句 :param tag: 标签名 @@ -298,3 +393,56 @@ def translate_loc(loc): raise ValueError('无法识别的定位符。') return loc_by, loc_str + + +def translate_css_loc(loc): + """把By类型的loc元组转换为css selector或xpath类型的 + :param loc: By类型的loc元组 + :return: css selector或xpath类型的loc元组 + """ + if len(loc) != 2: + raise ValueError('定位符长度必须为2。') + + loc_by = By.CSS_SELECTOR + loc_0 = loc[0].lower() + if loc_0 == By.XPATH: + loc_by = By.XPATH + loc_str = loc[1] + + elif loc_0 == By.CSS_SELECTOR: + loc_by = loc_0 + loc_str = loc[1] + + elif loc_0 == By.ID: + loc_str = f'#{css_trans(loc[1])}' + + elif loc_0 == By.CLASS_NAME: + loc_str = f'.{css_trans(loc[1])}' + + elif loc_0 == By.PARTIAL_LINK_TEXT: + loc_by = By.XPATH + loc_str = f'//a[text()="{css_trans(loc[1])}"]' + + elif loc_0 == By.NAME: + loc_str = f'*[@name={css_trans(loc[1])}]' + + elif loc_0 == By.TAG_NAME: + loc_str = loc[1] + + elif loc_0 == By.PARTIAL_LINK_TEXT: + loc_by = By.XPATH + loc_str = f'//a[contains(text(),"{loc[1]}")]' + + else: + raise ValueError('无法识别的定位符。') + + if loc_by == By.CSS_SELECTOR: + pass + + return loc_by, loc_str + + +def css_trans(txt): + c = ('!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', + '[', '\\', ']', '^', '`', ',', '{', '|', '}', '~', ' ') + return ''.join([fr'\{i}' if i in c else i for i in txt]) diff --git a/DrissionPage/_commons/locator.pyi b/DrissionPage/_commons/locator.pyi index ec6a426..32d7e57 100644 --- a/DrissionPage/_commons/locator.pyi +++ b/DrissionPage/_commons/locator.pyi @@ -6,13 +6,19 @@ from typing import Union -def is_loc(text:str) -> bool: ... +def is_loc(text: str) -> bool: ... -def get_loc(loc: Union[tuple, str], translate_css: bool = False) -> tuple: ... +def get_loc(loc: Union[tuple, str], translate_css: bool = False, css_mode: bool = False) -> tuple: ... def str_to_loc(loc: str) -> tuple: ... def translate_loc(loc: tuple) -> tuple: ... + + +def translate_css_loc(loc: tuple) -> tuple: ... + + +def css_trans(txt: str) -> str: ... diff --git a/DrissionPage/_commons/tools.py b/DrissionPage/_commons/tools.py index 813c3d1..4811a40 100644 --- a/DrissionPage/_commons/tools.py +++ b/DrissionPage/_commons/tools.py @@ -3,8 +3,8 @@ @Author : g1879 @Contact : g1879@qq.com """ -from platform import system from pathlib import Path +from platform import system from re import search, sub from shutil import rmtree from time import perf_counter, sleep @@ -12,6 +12,7 @@ from time import perf_counter, sleep from psutil import process_iter, AccessDenied, NoSuchProcess, ZombieProcess from .._configs.options_manage import OptionsManager +from ..errors import ContextLostError, ElementLostError, CDPError, PageClosedError, NoRectError, AlertExistsError def get_usable_path(path, is_file=True, parents=True): @@ -250,3 +251,29 @@ def configs_to_here(save_name=None): om = OptionsManager('default') save_name = f'{save_name}.ini' if save_name is not None else 'dp_configs.ini' om.save(save_name) + + +def raise_error(r): + """抛出error对应报错 + :param r: 包含error的dict + :return: None + """ + error = r['error'] + if error in ('Cannot find context with specified id', 'Inspected target navigated or closed'): + raise ContextLostError + elif error in ('Could not find node with given id', 'Could not find object with given id', + 'No node with given id found', 'Node with given id does not belong to the document', + 'No node found for given backend id'): + raise ElementLostError + elif error == 'tab closed': + raise PageClosedError + elif error == 'timeout': + raise TimeoutError + elif error == 'alert exists.': + raise AlertExistsError + elif error in ('Node does not have a layout object', 'Could not compute box model.'): + raise NoRectError + elif r['type'] == 'call_method_error': + raise CDPError(f'\n错误：{r["error"]}\nmethod：{r["method"]}\nargs：{r["args"]}') + else: + raise RuntimeError(r) diff --git a/DrissionPage/_commons/tools.pyi b/DrissionPage/_commons/tools.pyi index 03aae58..784265b 100644 --- a/DrissionPage/_commons/tools.pyi +++ b/DrissionPage/_commons/tools.pyi @@ -42,3 +42,6 @@ def stop_process_on_port(port: Union[int, str]) -> None: ... def configs_to_here(file_name: Union[Path, str] = None) -> None: ... + + +def raise_error(r: dict) -> None: ... diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 6be8334..099b1f4 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -10,9 +10,9 @@ from time import perf_counter, sleep from .none_element import NoneElement from .session_element import make_session_ele from .._base.base import DrissionElement, BaseElement -from .._commons.settings import Settings from .._commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions from .._commons.locator import get_loc +from .._commons.settings import Settings from .._commons.tools import get_usable_path from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, offset_scroll from .._units.clicker import Clicker @@ -711,7 +711,7 @@ class ChromiumElement(DrissionElement): elif mode == 'css': txt1 = '' txt3 = '' - txt4 = '''path = '>' + ":nth-child(" + nth + ")" + path;''' + txt4 = '''path = '>' + el.tagName + ":nth-child(" + nth + ")" + path;''' txt5 = '''return path.substr(1);''' else: @@ -736,7 +736,7 @@ class ChromiumElement(DrissionElement): return e(this);} ''' t = self.run_js(js) - return f':root{t}' if mode == 'css' else t + return f'{t}' if mode == 'css' else t def _set_file_input(self, files): """对上传控件写入路径 @@ -1022,31 +1022,42 @@ class ChromiumShadowRoot(BaseElement): :param raise_err: 找不到元素是是否抛出异常，为None时根据全局设置 :return: ChromiumElement对象或其组成的列表 """ - loc = get_loc(loc_or_str) + loc = get_loc(loc_or_str, css_mode=False) if loc[0] == 'css selector' and str(loc[1]).startswith(':root'): loc = loc[0], loc[1][5:] + result = None timeout = timeout if timeout is not None else self.page.timeout end_time = perf_counter() + timeout - eles = make_session_ele(self.html).eles(loc) - while not eles and perf_counter() <= end_time: - eles = make_session_ele(self.html).eles(loc) + while not result and perf_counter() <= end_time: + if loc[0] == 'css selector': + if single: + nod_id = self.page.run_cdp('DOM.querySelector', nodeId=self._node_id, selector=loc[1])['nodeId'] + result = make_chromium_ele(self.page, node_id=nod_id) if nod_id else NoneElement() - if not eles: - return NoneElement() if single else eles + else: + nod_ids = self.page.run_cdp('DOM.querySelectorAll', nodeId=self._node_id, selector=loc[1])['nodeId'] + result = [make_chromium_ele(self.page, node_id=n) for n in nod_ids] - css_paths = [i.css_path[47:] for i in eles] - if single: - node_id = self.page.run_cdp('DOM.querySelector', nodeId=self._node_id, selector=css_paths[0])['nodeId'] - return make_chromium_ele(self.page, node_id=node_id) if node_id else NoneElement() + else: + eles = make_session_ele(self.html).eles(loc) + if not eles: + result = NoneElement() if single else eles + continue - else: - results = [] - for i in css_paths: - node_id = self.page.run_cdp('DOM.querySelector', nodeId=self._node_id, selector=i)['nodeId'] - if node_id: - results.append(make_chromium_ele(self.page, node_id=node_id)) - return results + css = [i.css_path[61:] for i in eles] + if single: + node_id = self.page.run_cdp('DOM.querySelector', nodeId=self._node_id, selector=css[0])['nodeId'] + result = make_chromium_ele(self.page, node_id=node_id) if node_id else NoneElement() + + else: + result = [] + for i in css: + node_id = self.page.run_cdp('DOM.querySelector', nodeId=self._node_id, selector=i)['nodeId'] + if node_id: + result.append(make_chromium_ele(self.page, node_id=node_id)) + + return result def _get_node_id(self, obj_id): """返回元素node id""" diff --git a/DrissionPage/_elements/session_element.py b/DrissionPage/_elements/session_element.py index 177ae96..f646b1c 100644 --- a/DrissionPage/_elements/session_element.py +++ b/DrissionPage/_elements/session_element.py @@ -267,14 +267,14 @@ class SessionElement(DrissionElement): while ele: if mode == 'css': brothers = len(ele.eles(f'xpath:./preceding-sibling::*')) - path_str = f'>:nth-child({brothers + 1}){path_str}' + path_str = f'>{ele.tag}:nth-child({brothers + 1}){path_str}' else: brothers = len(ele.eles(f'xpath:./preceding-sibling::{ele.tag}')) path_str = f'/{ele.tag}[{brothers + 1}]{path_str}' if brothers > 0 else f'/{ele.tag}{path_str}' ele = ele.parent() - return f':root{path_str[1:]}' if mode == 'css' else path_str + return f'{path_str[1:]}' if mode == 'css' else path_str def make_session_ele(html_or_ele, loc=None, single=True): diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index d1e9cc7..3783ee9 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -10,9 +10,9 @@ from threading import Thread from time import perf_counter, sleep from .._base.base import BasePage -from .._commons.settings import Settings from .._commons.locator import get_loc, is_loc -from .._commons.tools import get_usable_path +from .._commons.settings import Settings +from .._commons.tools import get_usable_path, raise_error from .._commons.web import location_in_viewport from .._elements.chromium_element import ChromiumElement, run_js, make_chromium_ele from .._elements.none_element import NoneElement @@ -25,8 +25,8 @@ from .._units.scroller import PageScroller from .._units.setter import ChromiumBaseSetter from .._units.states import PageStates from .._units.waiter import BaseWaiter -from ..errors import (ContextLostError, ElementLostError, CDPError, PageClosedError, NoRectError, AlertExistsError, - GetDocumentError, ElementNotFoundError) +from ..errors import (ContextLostError, ElementLostError, CDPError, PageClosedError, GetDocumentError, + ElementNotFoundError) __ERROR__ = 'error' @@ -441,28 +441,7 @@ class ChromiumBase(BasePage): :return: 执行的结果 """ r = self.driver.run(cmd, **cmd_args) - if __ERROR__ not in r: - return r - - error = r[__ERROR__] - if error in ('Cannot find context with specified id', 'Inspected target navigated or closed'): - raise ContextLostError - elif error in ('Could not find node with given id', 'Could not find object with given id', - 'No node with given id found', 'Node with given id does not belong to the document', - 'No node found for given backend id'): - raise ElementLostError - elif error == 'tab closed': - raise PageClosedError - elif error == 'timeout': - raise TimeoutError - elif error == 'alert exists.': - raise AlertExistsError - elif error in ('Node does not have a layout object', 'Could not compute box model.'): - raise NoRectError - elif r['type'] == 'call_method_error': - raise CDPError(f'\n错误：{r["error"]}\nmethod：{r["method"]}\nargs：{r["args"]}') - else: - raise RuntimeError(r) + return r if __ERROR__ not in r else raise_error(r) def run_cdp_loaded(self, cmd, **cmd_args): """执行Chrome DevTools Protocol语句，执行前等待页面加载完毕 From e3f654ce1234db8e8f00feb9150c0dccd4f47731 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 24 Nov 2023 01:40:22 +0800 Subject: [PATCH 106/182] =?UTF-8?q?@@=E6=94=B9=E4=B8=BA@&=EF=BC=8C@@-?= =?UTF-8?q?=E6=94=B9=E4=B8=BA@!=EF=BC=9B=E5=A2=9E=E5=8A=A0css=E6=96=B9?= =?UTF-8?q?=E5=BC=8F=E7=9A=84=E5=AE=9A=E4=BD=8D=E7=AC=A6=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 4 +- DrissionPage/_commons/locator.py | 147 ++++++++++++++++----- DrissionPage/_commons/locator.pyi | 2 +- DrissionPage/_elements/chromium_element.py | 6 +- 4 files changed, 120 insertions(+), 39 deletions(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index addd7e2..3fdabcd 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -173,12 +173,12 @@ class Browser(object): end_time = perf_counter() + timeout while perf_counter() < end_time: p = popen(txt) + sleep(.1) try: if f' {self.process_id} ' not in p.read(): return except TypeError: - return - sleep(.2) + pass if force: ip, port = self.address.split(':') diff --git a/DrissionPage/_commons/locator.py b/DrissionPage/_commons/locator.py index f4d9976..608b796 100644 --- a/DrissionPage/_commons/locator.py +++ b/DrissionPage/_commons/locator.py @@ -16,7 +16,7 @@ def is_loc(text): def get_loc(loc, translate_css=False, css_mode=False): """接收本库定位语法或selenium定位元组，转换为标准定位元组，可翻译css selector为xpath :param loc: 本库定位语法或selenium定位元组 - :param translate_css: 是否翻译css selector为xpath + :param translate_css: 是否翻译css selector为xpath，用于相对定位 :param css_mode: 是否尽量用css selector方式 :return: DrissionPage定位元组 """ @@ -24,7 +24,7 @@ def get_loc(loc, translate_css=False, css_mode=False): loc = translate_css_loc(loc) if css_mode else translate_loc(loc) elif isinstance(loc, str): - loc = str_to_css_loc(loc) if css_mode else str_to_loc(loc) + loc = str_to_css_loc(loc) if css_mode else str_to_xpath_loc(loc) else: raise TypeError('loc参数只能是tuple或str。') @@ -41,7 +41,7 @@ def get_loc(loc, translate_css=False, css_mode=False): return loc -def str_to_loc(loc): +def str_to_xpath_loc(loc): """处理元素查找语句 :param loc: 查找语法字符串 :return: 匹配符元组 @@ -68,15 +68,24 @@ def str_to_loc(loc): # ------------------------------------------------------------------ # 多属性查找 - if loc.startswith('@@') and loc != '@@': - loc_str = _make_multi_xpath_str('*', loc) + if loc.startswith('@!') and loc != '@!': + r = split(r'(@!|@&|@\|)', loc) + if '@&' in r and '@|' in r: + raise ValueError('@&和@|不能同时出现在一个定位语句中。') + elif '@&' in r: + loc_str = _make_multi_xpath_str('*', loc)[1] + else: # @| + loc_str = _make_multi_xpath_str('*', loc, False)[1] + + elif loc.startswith('@&') and loc != '@&': + loc_str = _make_multi_xpath_str('*', loc)[1] elif loc.startswith('@|') and loc != '@|': - loc_str = _make_multi_xpath_str('*', loc, False) + loc_str = _make_multi_xpath_str('*', loc, False)[1] # 单属性查找 elif loc.startswith('@') and loc != '@': - loc_str = _make_single_xpath_str('*', loc) + loc_str = _make_single_xpath_str('*', loc)[1] # 根据tag name查找 elif loc.startswith(('tag:', 'tag=')) and loc not in ('tag:', 'tag='): @@ -84,12 +93,12 @@ def str_to_loc(loc): if at_ind == -1: loc_str = f'//*[name()="{loc[4:]}"]' else: - if loc[at_ind:].startswith('@@'): - loc_str = _make_multi_xpath_str(loc[4:at_ind], loc[at_ind:]) + if loc[at_ind:].startswith(('@&', '@!')): + loc_str = _make_multi_xpath_str(loc[4:at_ind], loc[at_ind:])[1] elif loc[at_ind:].startswith('@|'): - loc_str = _make_multi_xpath_str(loc[4:at_ind], loc[at_ind:], False) + loc_str = _make_multi_xpath_str(loc[4:at_ind], loc[at_ind:], False)[1] else: - loc_str = _make_single_xpath_str(loc[4:at_ind], loc[at_ind:]) + loc_str = _make_single_xpath_str(loc[4:at_ind], loc[at_ind:])[1] # 根据文本查找 elif loc.startswith('text='): @@ -133,7 +142,6 @@ def str_to_css_loc(loc): :param loc: 查找语法字符串 :return: 匹配符元组 """ - return str_to_loc(loc) loc_by = 'css selector' if loc.startswith('.'): @@ -156,15 +164,24 @@ def str_to_css_loc(loc): # ------------------------------------------------------------------ # 多属性查找 - if loc.startswith('@@') and loc != '@@': - loc_str = _make_multi_xpath_str('*', loc) + if loc.startswith('@!') and loc != '@!': + r = split(r'(@!|@&|@\|)', loc) + if '@&' in r and '@|' in r: + raise ValueError('@&和@|不能同时出现在一个定位语句中。') + elif '@&' in r: + loc_by, loc_str = _make_multi_css_str('*', loc) + else: # @| + loc_by, loc_str = _make_multi_css_str('*', loc, False) + + elif loc.startswith('@&') and loc != '@&': + loc_by, loc_str = _make_multi_css_str('*', loc) elif loc.startswith('@|') and loc != '@|': - loc_str = _make_multi_xpath_str('*', loc, False) + loc_by, loc_str = _make_multi_css_str('*', loc, False) # 单属性查找 elif loc.startswith('@') and loc != '@': - loc_str = _make_single_xpath_str('*', loc) + loc_by, loc_str = _make_single_css_str('*', loc) # 根据tag name查找 elif loc.startswith(('tag:', 'tag=')) and loc not in ('tag:', 'tag='): @@ -172,12 +189,12 @@ def str_to_css_loc(loc): if at_ind == -1: loc_str = loc[4:] else: - if loc[at_ind:].startswith('@@'): - loc_str = _make_multi_xpath_str(loc[4:at_ind], loc[at_ind:]) + if loc[at_ind:].startswith(('@&', '@!')): + loc_by, loc_str = _make_multi_css_str(loc[4:at_ind], loc[at_ind:]) elif loc[at_ind:].startswith('@|'): - loc_str = _make_multi_xpath_str(loc[4:at_ind], loc[at_ind:], False) + loc_by, loc_str = _make_multi_css_str(loc[4:at_ind], loc[at_ind:], False) else: - loc_str = _make_single_xpath_str(loc[4:at_ind], loc[at_ind:]) + loc_by, loc_str = _make_single_css_str(loc[4:at_ind], loc[at_ind:]) # 根据文本查找 elif loc.startswith('text='): @@ -222,8 +239,8 @@ def str_to_css_loc(loc): return loc_by, loc_str -def _make_single_xpath_str(tag: str, text: str) -> str: - """生成xpath语句 +def _make_single_xpath_str(tag: str, text: str) -> tuple: + """生成单属性xpath语句 :param tag: 标签名 :param text: 待处理的字符串 :return: xpath字符串 @@ -238,16 +255,13 @@ def _make_single_xpath_str(tag: str, text: str) -> str: r = split(r'([:=$^])', text, maxsplit=1) len_r = len(r) len_r0 = len(r[0]) - if len_r != 3 and len_r0 > 1: - arg_str = 'normalize-space(text())' if r[0] in ('@text()', '@tx()') else f'{r[0]}' - - elif len_r == 3 and len_r0 > 1: + if len_r == 3 and len_r0 > 1: symbol = r[1] if symbol == '=': # 精确查找 arg = '.' if r[0] in ('@text()', '@tx()') else r[0] arg_str = f'{arg}={_make_search_str(r[2])}' - elif symbol == '^': # 开头开头 + elif symbol == '^': # 匹配开头 if r[0] in ('@text()', '@tx()'): txt_str = f'/text()[starts-with(., {_make_search_str(r[2])})]/..' arg_str = '' @@ -273,13 +287,16 @@ def _make_single_xpath_str(tag: str, text: str) -> str: else: raise ValueError(f'符号不正确：{symbol}') + elif len_r != 3 and len_r0 > 1: + arg_str = 'normalize-space(text())' if r[0] in ('@text()', '@tx()') else f'{r[0]}' + if arg_str: arg_list.append(arg_str) arg_str = ' and '.join(arg_list) - return f'//*[{arg_str}]{txt_str}' if arg_str else f'//*{txt_str}' + return 'xpath', f'//*[{arg_str}]{txt_str}' if arg_str else f'//*{txt_str}' -def _make_multi_xpath_str(tag: str, text: str, _and: bool = True) -> str: +def _make_multi_xpath_str(tag: str, text: str, _and: bool = True) -> tuple: """生成多属性查找的xpath语句 :param tag: 标签名 :param text: 待处理的字符串 @@ -287,10 +304,12 @@ def _make_multi_xpath_str(tag: str, text: str, _and: bool = True) -> str: :return: xpath字符串 """ arg_list = [] - args = text.split('@@') if _and else text.split('@|') + args = split(r'(@!|@&)', text)[1:] if _and else split(r'(@!|@\|)', text)[1:] + if (_and and '@|' in args) or (not _and and '@&' in args): + raise ValueError('@&和@|不能同时出现在一个定位语句中。') - for arg in args[1:]: - r = split(r'([:=$^])', arg, maxsplit=1) + for k in range(0, len(args) - 1, 2): + r = split(r'([:=$^])', args[k + 1], maxsplit=1) arg_str = '' len_r = len(r) @@ -298,8 +317,7 @@ def _make_multi_xpath_str(tag: str, text: str, _and: bool = True) -> str: arg_str = 'not(@*)' else: - r[0], ignore = (r[0][1:], True) if r[0][0] == '-' else (r[0], None) # 是否去除某个属性 - + ignore = True if args[k] == '@!' else False # 是否去除某个属性 if len_r != 3: # 只有属性名没有属性内容，查询是否存在该属性 arg_str = 'normalize-space(text())' if r[0] in ('text()', 'tx()') else f'@{r[0]}' @@ -333,7 +351,7 @@ def _make_multi_xpath_str(tag: str, text: str, _and: bool = True) -> str: condition = f' and ({arg_str})' if arg_str else '' arg_str = f'name()="{tag}"{condition}' - return f'//*[{arg_str}]' if arg_str else f'//*' + return 'xpath', f'//*[{arg_str}]' if arg_str else f'//*' def _make_search_str(search_str: str) -> str: @@ -353,6 +371,65 @@ def _make_search_str(search_str: str) -> str: return search_str +def _make_multi_css_str(tag: str, text: str, _and: bool = True) -> tuple: + """生成多属性查找的css selector语句 + :param tag: 标签名 + :param text: 待处理的字符串 + :param _and: 是否与方式 + :return: css selector字符串 + """ + arg_list = [] + args = split(r'(@!|@&)', text)[1:] if _and else split(r'(@!|@\|)', text)[1:] + if (_and and '@|' in args) or (not _and and '@&' in args): + raise ValueError('@&和@|不能同时出现在一个定位语句中。') + + for k in range(0, len(args)-1, 2): + r = split(r'([:=$^])', args[k+1], maxsplit=1) + if not r[0] or r[0].startswith(('text()', 'tx()')): + return _make_multi_xpath_str(tag, text, _and) + + arg_str = '' + len_r = len(r) + ignore = True if args[k] == '@!' else False # 是否去除某个属性 + if len_r != 3: # 只有属性名没有属性内容，查询是否存在该属性 + arg_str = f'[{r[0]}]' + + elif len_r == 3: # 属性名和内容都有 + d = {'=': '', '^': '^', '$': '$', ':': '*'} + arg_str = f'[{r[0]}{d[r[1]]}={css_trans(r[2])}]' + + if arg_str and ignore: + arg_str = f':not({arg_str})' + + if arg_str: + arg_list.append(arg_str) + + if _and: + return 'css selector', f'{tag}{"".join(arg_list)}' + + return 'css selector', f'{tag}{("," + tag).join(arg_list)}' + + +def _make_single_css_str(tag: str, text: str) -> tuple: + """生成单属性css selector语句 + :param tag: 标签名 + :param text: 待处理的字符串 + :return: css selector字符串 + """ + if text == '@' or text.startswith(('@text()', '@tx()')): + return _make_single_xpath_str(tag, text) + + r = split(r'([:=$^])', text, maxsplit=1) + if len(r) == 3: + d = {'=': '', '^': '^', '$': '$', ':': '*'} + arg_str = f'[{r[0][1:]}{d[r[1]]}={css_trans(r[2])}]' + + else: + arg_str = f'[{css_trans(r[0][1:])}]' + + return 'css selector', f'{tag}{arg_str}' + + def translate_loc(loc): """把By类型的loc元组转换为css selector或xpath类型的 :param loc: By类型的loc元组 diff --git a/DrissionPage/_commons/locator.pyi b/DrissionPage/_commons/locator.pyi index 32d7e57..b890c03 100644 --- a/DrissionPage/_commons/locator.pyi +++ b/DrissionPage/_commons/locator.pyi @@ -12,7 +12,7 @@ def is_loc(text: str) -> bool: ... def get_loc(loc: Union[tuple, str], translate_css: bool = False, css_mode: bool = False) -> tuple: ... -def str_to_loc(loc: str) -> tuple: ... +def str_to_xpath_loc(loc: str) -> tuple: ... def translate_loc(loc: tuple) -> tuple: ... diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 099b1f4..d89c887 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -1029,7 +1029,7 @@ class ChromiumShadowRoot(BaseElement): result = None timeout = timeout if timeout is not None else self.page.timeout end_time = perf_counter() + timeout - while not result and perf_counter() <= end_time: + while perf_counter() <= end_time: if loc[0] == 'css selector': if single: nod_id = self.page.run_cdp('DOM.querySelector', nodeId=self._node_id, selector=loc[1])['nodeId'] @@ -1057,6 +1057,10 @@ class ChromiumShadowRoot(BaseElement): if node_id: result.append(make_chromium_ele(self.page, node_id=node_id)) + if result: + break + sleep(.1) + return result def _get_node_id(self, obj_id): From 6d0f8a27f417b1427ff15c5302c58c25a83f5d93 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 24 Nov 2023 19:47:30 +0800 Subject: [PATCH 107/182] =?UTF-8?q?4.0.0b12=E8=A7=A3=E5=86=B3=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E6=B5=8F=E8=A7=88=E5=99=A8403=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/chromium_driver.py | 7 ++++--- DrissionPage/_commons/browser.py | 8 +------- DrissionPage/_configs/configs.ini | 2 +- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index a7392a4..6ba3127 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -9,8 +9,8 @@ from threading import Thread, Event from time import perf_counter from requests import get -from websocket import WebSocketTimeoutException, WebSocketException, WebSocketConnectionClosedException, \ - create_connection +from websocket import (WebSocketTimeoutException, WebSocketException, WebSocketConnectionClosedException, \ + create_connection) class ChromiumDriver(object): @@ -172,7 +172,8 @@ class ChromiumDriver(object): def start(self): """启动连接""" self._stopped.clear() - self._ws = create_connection(self._websocket_url, enable_multithread=True) + self._ws = create_connection(self._websocket_url, enable_multithread=True, + suppress_origin=True) self._recv_th.start() self._handle_event_th.start() return True diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_commons/browser.py index f06f881..b8c3440 100644 --- a/DrissionPage/_commons/browser.py +++ b/DrissionPage/_commons/browser.py @@ -65,7 +65,6 @@ def get_launch_args(opt): # ----------处理arguments----------- result = set() has_user_path = False - remote_allow = False headless = None for i in opt.arguments: if i.startswith(('--load-extension=', '--remote-debugging-port=')): @@ -74,8 +73,6 @@ def get_launch_args(opt): result.add(f'--user-data-dir={Path(i[16:]).absolute()}') has_user_path = True continue - elif i.startswith('--remote-allow-origins='): - remote_allow = True elif i.startswith('--headless'): if i == '--headless=false': headless = False @@ -95,9 +92,6 @@ def get_launch_args(opt): opt.set_user_data_path(path) result.add(f'--user-data-dir={path}') - if not remote_allow: - result.add('--remote-allow-origins=*') - if headless is None and system().lower() == 'linux': from os import popen r = popen('systemctl list-units | grep graphical.target') @@ -215,7 +209,7 @@ def test_connect(ip, port, timeout=30): sleep(.2) raise BrowserConnectError(f'\n{ip}:{port}浏览器无法链接。\n请确认：\n1、该端口为浏览器\n' - f'2、已添加--remote-allow-origins=*和--remote-debugging-port={port}启动项\n' + f'2、已添加--remote-debugging-port={port}启动项\n' f'3、用户文件夹没有和已打开的浏览器冲突\n' f'4、如为无界面系统，请添加--headless=new参数\n' f'5、如果是Linux系统，可能还要添加--no-sandbox启动参数\n' diff --git a/DrissionPage/_configs/configs.ini b/DrissionPage/_configs/configs.ini index 62e1746..a977a3b 100644 --- a/DrissionPage/_configs/configs.ini +++ b/DrissionPage/_configs/configs.ini @@ -4,7 +4,7 @@ download_path = [chrome_options] debugger_address = 127.0.0.1:9222 browser_path = chrome -arguments = ['--remote-allow-origins=*', '--no-first-run', '--disable-infobars', '--disable-popup-blocking'] +arguments = ['--no-first-run', '--disable-infobars', '--disable-popup-blocking'] extensions = [] prefs = {'profile.default_content_settings.popups': 0, 'profile.default_content_setting_values': {'notifications': 2}} flags = {} From acfd774d1f712e3a0b1bc2bc32ac32f0b986f910 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 26 Nov 2023 23:15:06 +0800 Subject: [PATCH 108/182] =?UTF-8?q?4.0.0b13=EF=BC=88=E8=AF=A6=EF=BC=89=20?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E7=B1=BB=E5=92=8Cini=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=8A=A0=E4=B8=8A=E8=BF=9E=E6=8E=A5=E9=87=8D=E8=AF=95=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=EF=BC=9B=20ini=E6=96=87=E4=BB=B6chrome=5Foptions?= =?UTF-8?q?=E6=94=B9=E4=B8=BAchromium=5Foptions=20@&=E6=94=B9=E5=9B=9E@@?= =?UTF-8?q?=EF=BC=8C=E4=BC=98=E5=8C=96=E5=92=8C=E4=BF=AE=E5=A4=8D=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E9=97=AE=E9=A2=98=EF=BC=9B=20WebPage=E7=9A=84driver?= =?UTF-8?q?=5Foptions=E5=8F=82=E6=95=B0=E6=94=B9=E4=B8=BAchromium=5Foption?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_commons/browser.py | 2 +- DrissionPage/_commons/locator.py | 112 ++++++--------------- DrissionPage/_configs/chromium_options.py | 42 +++++++- DrissionPage/_configs/chromium_options.pyi | 10 ++ DrissionPage/_configs/configs.ini | 5 +- DrissionPage/_configs/session_options.py | 70 +++++++++---- DrissionPage/_configs/session_options.pyi | 10 ++ DrissionPage/_pages/chromium_page.py | 47 ++++----- DrissionPage/_pages/chromium_page.pyi | 2 +- DrissionPage/_pages/chromium_tab.py | 2 +- DrissionPage/_pages/session_page.py | 2 + DrissionPage/_pages/web_page.py | 18 ++-- DrissionPage/_pages/web_page.pyi | 4 +- setup.py | 2 +- 15 files changed, 184 insertions(+), 146 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index c4bce69..6be91ef 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b12' +__version__ = '4.0.0b13' diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_commons/browser.py index b8c3440..b263674 100644 --- a/DrissionPage/_commons/browser.py +++ b/DrissionPage/_commons/browser.py @@ -293,7 +293,7 @@ def get_chrome_path(ini_path=None, show_msg=True, from_ini=True, # -----------从ini文件中获取-------------- if ini_path and from_ini: try: - path = OptionsManager(ini_path).chrome_options['browser_path'] + path = OptionsManager(ini_path).chromium_options['browser_path'] except KeyError: path = None else: diff --git a/DrissionPage/_commons/locator.py b/DrissionPage/_commons/locator.py index 608b796..58d6222 100644 --- a/DrissionPage/_commons/locator.py +++ b/DrissionPage/_commons/locator.py @@ -68,21 +68,9 @@ def str_to_xpath_loc(loc): # ------------------------------------------------------------------ # 多属性查找 - if loc.startswith('@!') and loc != '@!': - r = split(r'(@!|@&|@\|)', loc) - if '@&' in r and '@|' in r: - raise ValueError('@&和@|不能同时出现在一个定位语句中。') - elif '@&' in r: - loc_str = _make_multi_xpath_str('*', loc)[1] - else: # @| - loc_str = _make_multi_xpath_str('*', loc, False)[1] - - elif loc.startswith('@&') and loc != '@&': + if loc.startswith(('@@', '@|', '@!')) and loc not in ('@@', '@|', '@!'): loc_str = _make_multi_xpath_str('*', loc)[1] - elif loc.startswith('@|') and loc != '@|': - loc_str = _make_multi_xpath_str('*', loc, False)[1] - # 单属性查找 elif loc.startswith('@') and loc != '@': loc_str = _make_single_xpath_str('*', loc)[1] @@ -92,24 +80,18 @@ def str_to_xpath_loc(loc): at_ind = loc.find('@') if at_ind == -1: loc_str = f'//*[name()="{loc[4:]}"]' + elif loc[at_ind:].startswith(('@@', '@|', '@!')): + loc_str = _make_multi_xpath_str(loc[4:at_ind], loc[at_ind:])[1] else: - if loc[at_ind:].startswith(('@&', '@!')): - loc_str = _make_multi_xpath_str(loc[4:at_ind], loc[at_ind:])[1] - elif loc[at_ind:].startswith('@|'): - loc_str = _make_multi_xpath_str(loc[4:at_ind], loc[at_ind:], False)[1] - else: - loc_str = _make_single_xpath_str(loc[4:at_ind], loc[at_ind:])[1] + loc_str = _make_single_xpath_str(loc[4:at_ind], loc[at_ind:])[1] # 根据文本查找 elif loc.startswith('text='): loc_str = f'//*[text()={_make_search_str(loc[5:])}]' - elif loc.startswith('text:') and loc != 'text:': loc_str = f'//*/text()[contains(., {_make_search_str(loc[5:])})]/..' - elif loc.startswith('text^') and loc != 'text^': loc_str = f'//*/text()[starts-with(., {_make_search_str(loc[5:])})]/..' - elif loc.startswith('text$') and loc != 'text$': loc_str = f'//*/text()[substring(., string-length(.) - string-length({_make_search_str(loc[5:])}) +1) = ' \ f'{_make_search_str(loc[5:])}]/..' @@ -164,20 +146,8 @@ def str_to_css_loc(loc): # ------------------------------------------------------------------ # 多属性查找 - if loc.startswith('@!') and loc != '@!': - r = split(r'(@!|@&|@\|)', loc) - if '@&' in r and '@|' in r: - raise ValueError('@&和@|不能同时出现在一个定位语句中。') - elif '@&' in r: - loc_by, loc_str = _make_multi_css_str('*', loc) - else: # @| - loc_by, loc_str = _make_multi_css_str('*', loc, False) - - elif loc.startswith('@&') and loc != '@&': - loc_by, loc_str = _make_multi_css_str('*', loc) - - elif loc.startswith('@|') and loc != '@|': - loc_by, loc_str = _make_multi_css_str('*', loc, False) + if loc.startswith(('@@', '@|', '@!')) and loc not in ('@@', '@|', '@!'): + loc_str = _make_multi_css_str('*', loc)[1] # 单属性查找 elif loc.startswith('@') and loc != '@': @@ -188,39 +158,14 @@ def str_to_css_loc(loc): at_ind = loc.find('@') if at_ind == -1: loc_str = loc[4:] + elif loc[at_ind:].startswith(('@@', '@|', '@!')): + loc_by, loc_str = _make_multi_css_str(loc[4:at_ind], loc[at_ind:]) else: - if loc[at_ind:].startswith(('@&', '@!')): - loc_by, loc_str = _make_multi_css_str(loc[4:at_ind], loc[at_ind:]) - elif loc[at_ind:].startswith('@|'): - loc_by, loc_str = _make_multi_css_str(loc[4:at_ind], loc[at_ind:], False) - else: - loc_by, loc_str = _make_single_css_str(loc[4:at_ind], loc[at_ind:]) + loc_by, loc_str = _make_single_css_str(loc[4:at_ind], loc[at_ind:]) # 根据文本查找 - elif loc.startswith('text='): - loc_by = 'xpath' - loc_str = f'//*[text()={_make_search_str(loc[5:])}]' - - elif loc.startswith('text:') and loc != 'text:': - loc_by = 'xpath' - loc_str = f'//*/text()[contains(., {_make_search_str(loc[5:])})]/..' - - elif loc.startswith('text^') and loc != 'text^': - loc_by = 'xpath' - loc_str = f'//*/text()[starts-with(., {_make_search_str(loc[5:])})]/..' - - elif loc.startswith('text$') and loc != 'text$': - loc_by = 'xpath' - loc_str = f'//*/text()[substring(., string-length(.) - string-length({_make_search_str(loc[5:])}) +1) = ' \ - f'{_make_search_str(loc[5:])}]/..' - - # 用xpath查找 - elif loc.startswith(('xpath:', 'xpath=')) and loc not in ('xpath:', 'xpath='): - loc_by = 'xpath' - loc_str = loc[6:] - elif loc.startswith(('x:', 'x=')) and loc not in ('x:', 'x='): - loc_by = 'xpath' - loc_str = loc[2:] + elif loc.startswith(('text=', 'text:', 'text^', 'text$', 'xpath=', 'xpath:', 'x:', 'x=')): + loc_by, loc_str = str_to_xpath_loc(loc) # 用css selector查找 elif loc.startswith(('css:', 'css=')) and loc not in ('css:', 'css='): @@ -230,8 +175,7 @@ def str_to_css_loc(loc): # 根据文本模糊查找 elif loc: - loc_by = 'xpath' - loc_str = f'//*/text()[contains(., {_make_search_str(loc)})]/..' + loc_by, loc_str = str_to_xpath_loc(loc) else: loc_str = '*' @@ -296,17 +240,20 @@ def _make_single_xpath_str(tag: str, text: str) -> tuple: return 'xpath', f'//*[{arg_str}]{txt_str}' if arg_str else f'//*{txt_str}' -def _make_multi_xpath_str(tag: str, text: str, _and: bool = True) -> tuple: +def _make_multi_xpath_str(tag: str, text: str) -> tuple: """生成多属性查找的xpath语句 :param tag: 标签名 :param text: 待处理的字符串 - :param _and: 是否与方式 :return: xpath字符串 """ arg_list = [] - args = split(r'(@!|@&)', text)[1:] if _and else split(r'(@!|@\|)', text)[1:] - if (_and and '@|' in args) or (not _and and '@&' in args): - raise ValueError('@&和@|不能同时出现在一个定位语句中。') + args = split(r'(@!|@@|@\|)', text)[1:] + if '@@' in args and '@|' in args: + raise ValueError('@@和@|不能同时出现在一个定位语句中。') + elif '@@' in args: + _and = True + else: # @| + _and = False for k in range(0, len(args) - 1, 2): r = split(r'([:=$^])', args[k + 1], maxsplit=1) @@ -371,22 +318,25 @@ def _make_search_str(search_str: str) -> str: return search_str -def _make_multi_css_str(tag: str, text: str, _and: bool = True) -> tuple: +def _make_multi_css_str(tag: str, text: str) -> tuple: """生成多属性查找的css selector语句 :param tag: 标签名 :param text: 待处理的字符串 - :param _and: 是否与方式 :return: css selector字符串 """ arg_list = [] - args = split(r'(@!|@&)', text)[1:] if _and else split(r'(@!|@\|)', text)[1:] - if (_and and '@|' in args) or (not _and and '@&' in args): - raise ValueError('@&和@|不能同时出现在一个定位语句中。') + args = split(r'(@!|@@|@\|)', text)[1:] + if '@@' in args and '@|' in args: + raise ValueError('@@和@|不能同时出现在一个定位语句中。') + elif '@@' in args: + _and = True + else: # @| + _and = False - for k in range(0, len(args)-1, 2): - r = split(r'([:=$^])', args[k+1], maxsplit=1) + for k in range(0, len(args) - 1, 2): + r = split(r'([:=$^])', args[k + 1], maxsplit=1) if not r[0] or r[0].startswith(('text()', 'tx()')): - return _make_multi_xpath_str(tag, text, _and) + return _make_multi_xpath_str(tag, text) arg_str = '' len_r = len(r) diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index 3f1e962..9ec74b0 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -28,8 +28,8 @@ class ChromiumOptions(object): ini_path = str(ini_path) if ini_path else None om = OptionsManager(ini_path) self.ini_path = om.ini_path - options = om.chrome_options + options = om.chromium_options self._download_path = om.paths.get('download_path', None) or None self._arguments = options.get('arguments', []) self._browser_path = options.get('browser_path', '') @@ -63,6 +63,11 @@ class ChromiumOptions(object): port, path = PortFinder().get_port() self._debugger_address = f'127.0.0.1:{port}' self.set_argument('--user-data-dir', path) + + others = om.others + self._retry_times = others.get('retry_times', 3) + self._retry_interval = others.get('retry_interval', 2) + return self.ini_path = None @@ -79,6 +84,8 @@ class ChromiumOptions(object): self._auto_port = False self._system_user_path = False self._existing_only = False + self._retry_times = 3 + self._retry_interval = 2 @property def download_path(self): @@ -155,6 +162,28 @@ class ChromiumOptions(object): """返回是否只接管现有浏览器方式""" return self._existing_only + @property + def retry_times(self): + """返回连接失败时的重试次数""" + return self._retry_times + + @property + def retry_interval(self): + """返回连接失败时的重试间隔（秒）""" + return self._retry_interval + + def set_retry(self, times=None, interval=None): + """设置连接失败时的重试操作 + :param times: 重试次数 + :param interval: 重试间隔 + :return: 当前对象 + """ + if times is not None: + self._retry_times = times + if interval is not None: + self._retry_interval = interval + return self + def set_argument(self, arg, value=None): """设置浏览器配置的argument属性 :param arg: 属性名 @@ -477,22 +506,25 @@ class ChromiumOptions(object): else: om = OptionsManager(self.ini_path or str(Path(__file__).parent / 'configs.ini')) - # 设置chrome_options + # 设置chromium_options attrs = ('debugger_address', 'browser_path', 'arguments', 'extensions', 'user', 'load_mode', 'auto_port', 'system_user_path', 'existing_only', 'flags') for i in attrs: - om.set_item('chrome_options', i, self.__getattribute__(f'_{i}')) + om.set_item('chromium_options', i, self.__getattribute__(f'_{i}')) # 设置代理 om.set_item('proxies', 'http', self._proxy) om.set_item('proxies', 'https', self._proxy) # 设置路径 - om.set_item('paths', 'download_path', self._download_path) + om.set_item('paths', 'download_path', self._download_path or '') # 设置timeout om.set_item('timeouts', 'implicit', self._timeouts['implicit']) om.set_item('timeouts', 'page_load', self._timeouts['pageLoad']) om.set_item('timeouts', 'script', self._timeouts['script']) + # 设置重试 + om.set_item('others', 'retry_times', self.retry_times) + om.set_item('others', 'retry_interval', self.retry_interval) # 设置prefs - om.set_item('chrome_options', 'prefs', self._prefs) + om.set_item('chromium_options', 'prefs', self._prefs) path = str(path) om.save(path) diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index 76f8624..9a1b0d4 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -29,6 +29,8 @@ class ChromiumOptions(object): self._system_user_path: bool = ... self._existing_only: bool = ... self._headless: bool = ... + self._retry_times: int = ... + self._retry_interval: float = ... @property def download_path(self) -> str: ... @@ -72,6 +74,14 @@ class ChromiumOptions(object): @property def is_existing_only(self) -> bool: ... + @property + def retry_times(self) -> int: ... + + @property + def retry_interval(self) -> float: ... + + def set_retry(self, times: int = None, interval: float = None) -> ChromiumOptions: ... + def set_argument(self, arg: str, value: Union[str, None, bool] = None) -> ChromiumOptions: ... def remove_argument(self, value: str) -> ChromiumOptions: ... diff --git a/DrissionPage/_configs/configs.ini b/DrissionPage/_configs/configs.ini index a977a3b..72722bd 100644 --- a/DrissionPage/_configs/configs.ini +++ b/DrissionPage/_configs/configs.ini @@ -1,7 +1,7 @@ [paths] download_path = -[chrome_options] +[chromium_options] debugger_address = 127.0.0.1:9222 browser_path = chrome arguments = ['--no-first-run', '--disable-infobars', '--disable-popup-blocking'] @@ -27,3 +27,6 @@ script = 30 http = https = +[others] +retry_times = 3 +retry_interval = 2 diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index c702991..728c6a1 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -37,6 +37,8 @@ class SessionOptions(object): self._stream = None self._trust_env = None self._max_redirects = None + self._retry_times = 3 + self._retry_interval = 2 if read_file is False: return @@ -44,39 +46,43 @@ class SessionOptions(object): ini_path = str(ini_path) if ini_path else None om = OptionsManager(ini_path) self.ini_path = om.ini_path - options_dict = om.session_options - if options_dict.get('headers', None) is not None: - self.set_headers(options_dict['headers']) + options = om.session_options + if options.get('headers', None) is not None: + self.set_headers(options['headers']) - if options_dict.get('cookies', None) is not None: - self.set_cookies(options_dict['cookies']) + if options.get('cookies', None) is not None: + self.set_cookies(options['cookies']) - if options_dict.get('auth', None) is not None: - self._auth = options_dict['auth'] + if options.get('auth', None) is not None: + self._auth = options['auth'] - if options_dict.get('params', None) is not None: - self._params = options_dict['params'] + if options.get('params', None) is not None: + self._params = options['params'] - if options_dict.get('verify', None) is not None: - self._verify = options_dict['verify'] + if options.get('verify', None) is not None: + self._verify = options['verify'] - if options_dict.get('cert', None) is not None: - self._cert = options_dict['cert'] + if options.get('cert', None) is not None: + self._cert = options['cert'] - if options_dict.get('stream', None) is not None: - self._stream = options_dict['stream'] + if options.get('stream', None) is not None: + self._stream = options['stream'] - if options_dict.get('trust_env', None) is not None: - self._trust_env = options_dict['trust_env'] + if options.get('trust_env', None) is not None: + self._trust_env = options['trust_env'] - if options_dict.get('max_redirects', None) is not None: - self._max_redirects = options_dict['max_redirects'] + if options.get('max_redirects', None) is not None: + self._max_redirects = options['max_redirects'] self.set_proxies(om.proxies.get('http', None), om.proxies.get('https', None)) self._timeout = om.timeouts.get('implicit', 10) self._download_path = om.paths.get('download_path', None) or None + others = om.others + self._retry_times = others.get('retry_times', 3) + self._retry_interval = others.get('retry_interval', 2) + # ===========须独立处理的项开始============ @property def download_path(self): @@ -120,6 +126,28 @@ class SessionOptions(object): self._sets('proxies', {'http': http, 'https': https}) return self + @property + def retry_times(self): + """返回连接失败时的重试次数""" + return self._retry_times + + @property + def retry_interval(self): + """返回连接失败时的重试间隔（秒）""" + return self._retry_interval + + def set_retry(self, times=None, interval=None): + """设置连接失败时的重试操作 + :param times: 重试次数 + :param interval: 重试间隔 + :return: 当前对象 + """ + if times is not None: + self._retry_times = times + if interval is not None: + self._retry_interval = interval + return self + # ===========须独立处理的项结束============ @property @@ -350,10 +378,12 @@ class SessionOptions(object): if i not in ('download_path', 'timeout', 'proxies'): om.set_item('session_options', i, options[i]) - om.set_item('paths', 'download_path', self.download_path) + om.set_item('paths', 'download_path', self.download_path or '') om.set_item('timeouts', 'implicit', self.timeout) om.set_item('proxies', 'http', self.proxies.get('http', None)) om.set_item('proxies', 'https', self.proxies.get('https', None)) + om.set_item('others', 'retry_times', self.retry_times) + om.set_item('others', 'retry_interval', self.retry_interval) for i in self._del_set: if i == 'download_path': diff --git a/DrissionPage/_configs/session_options.pyi b/DrissionPage/_configs/session_options.pyi index e4fc4cf..cbd2931 100644 --- a/DrissionPage/_configs/session_options.pyi +++ b/DrissionPage/_configs/session_options.pyi @@ -31,6 +31,8 @@ class SessionOptions(object): self._max_redirects: int = ... self._timeout: float = ... self._del_set: set = ... + self._retry_times: int = ... + self._retry_interval: float = ... @property def download_path(self) -> str: ... @@ -66,6 +68,14 @@ class SessionOptions(object): def set_proxies(self, http: Union[str, None], https: Union[str, None] = None) -> SessionOptions: ... + @property + def retry_times(self) -> int: ... + + @property + def retry_interval(self) -> float: ... + + def set_retry(self, times: int = None, interval: float = None) -> SessionOptions: ... + @property def hooks(self) -> dict: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 54e2be6..677ce88 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -42,53 +42,54 @@ class ChromiumPage(ChromiumBase): :return: 返回浏览器地址 """ if not addr_or_opts: - self._driver_options = ChromiumOptions(addr_or_opts) + self._chromium_options = ChromiumOptions(addr_or_opts) elif isinstance(addr_or_opts, ChromiumOptions): - self._driver_options = addr_or_opts + self._chromium_options = addr_or_opts elif isinstance(addr_or_opts, str): - self._driver_options = ChromiumOptions() - self._driver_options.set_debugger_address(addr_or_opts) + self._chromium_options = ChromiumOptions() + self._chromium_options.set_debugger_address(addr_or_opts) elif isinstance(addr_or_opts, int): - self._driver_options = ChromiumOptions() - self._driver_options.set_local_port(addr_or_opts) + self._chromium_options = ChromiumOptions() + self._chromium_options.set_local_port(addr_or_opts) else: raise TypeError('只能接收ip:port格式或ChromiumOptions类型参数。') - return self._driver_options.debugger_address + return self._chromium_options.debugger_address def _run_browser(self): """连接浏览器""" - is_exist = connect_browser(self._driver_options) - ws = get(f'http://{self._driver_options.debugger_address}/json/version', + is_exist = connect_browser(self._chromium_options) + ws = get(f'http://{self._chromium_options.debugger_address}/json/version', headers={'Connection': 'close'}) if not ws: raise BrowserConnectError('\n浏览器连接失败，请检查是否启用全局代理。如有，须开放127.0.0.1地址。') ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] - self._browser = Browser(self._driver_options.debugger_address, ws, self) + self._browser = Browser(self._chromium_options.debugger_address, ws, self) - if (is_exist and self._driver_options._headless is False and + if (is_exist and self._chromium_options._headless is False and 'headless' in self._browser.run_cdp('Browser.getVersion')['userAgent'].lower()): self._browser.quit(3) - connect_browser(self._driver_options) - ws = get(f'http://{self._driver_options.debugger_address}/json/version', headers={'Connection': 'close'}) + connect_browser(self._chromium_options) + ws = get(f'http://{self._chromium_options.debugger_address}/json/version', headers={'Connection': 'close'}) ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] - self._browser = Browser(self._driver_options.debugger_address, ws, self) + self._browser = Browser(self._chromium_options.debugger_address, ws, self) def _d_set_runtime_settings(self): """设置运行时用到的属性""" - self._timeouts = Timeout(self, - page_load=self._driver_options.timeouts['pageLoad'], - script=self._driver_options.timeouts['script'], - implicit=self._driver_options.timeouts['implicit']) - if self._driver_options.timeouts['implicit'] is not None: - self._timeout = self._driver_options.timeouts['implicit'] - self._load_mode = self._driver_options.load_mode - self._download_path = None if self._driver_options.download_path is None \ - else str(Path(self._driver_options.download_path).absolute()) + self._timeouts = Timeout(self, page_load=self._chromium_options.timeouts['pageLoad'], + script=self._chromium_options.timeouts['script'], + implicit=self._chromium_options.timeouts['implicit']) + if self._chromium_options.timeouts['implicit'] is not None: + self._timeout = self._chromium_options.timeouts['implicit'] + self._load_mode = self._chromium_options.load_mode + self._download_path = None if self._chromium_options.download_path is None \ + else str(Path(self._chromium_options.download_path).absolute()) + self.retry_times = self._chromium_options.retry_times + self.retry_interval = self._chromium_options.retry_interval def _page_init(self): """浏览器相关设置""" diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index c834405..b4bc381 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -20,7 +20,7 @@ class ChromiumPage(ChromiumBase): addr_or_opts: Union[str, int, ChromiumOptions] = None, tab_id: str = None, timeout: float = None): - self._driver_options: ChromiumOptions = ... + self._chromium_options: ChromiumOptions = ... self._browser: Browser = ... self._rect: Optional[TabRect] = ... diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index f7a5514..64eee41 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -274,7 +274,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): # s模式转d模式 if self._mode == 'd': if self._driver is None: - self._connect_browser(self.page._driver_options) + self._connect_browser(self.page._chromium_options) self._url = None if not self._has_driver else super(SessionPage, self).url self._has_driver = True diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index eed198f..7b5fe44 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -55,6 +55,8 @@ class SessionPage(BasePage): self._timeout = self._session_options.timeout self._download_path = None if self._session_options.download_path is None \ else str(Path(self._session_options.download_path).absolute()) + self.retry_times = self._session_options.retry_times + self.retry_interval = self._session_options.retry_interval def _create_session(self): """创建内建Session对象""" diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index e17d4ae..5e7db55 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -15,15 +15,15 @@ from .._units.setter import WebPageSetter class WebPage(SessionPage, ChromiumPage, BasePage): """整合浏览器和request的页面类""" - def __init__(self, mode='d', timeout=None, driver_options=None, session_or_options=None, driver_or_options=None): + def __init__(self, mode='d', timeout=None, chromium_options=None, session_or_options=None, driver_or_options=None): """初始化函数 :param mode: 'd' 或 's'，即driver模式和session模式 :param timeout: 超时时间，d模式时为寻找元素时间，s模式时为连接时间，默认10秒 - :param driver_options: ChromiumDriver对象，只使用s模式时应传入False + :param chromium_options: ChromiumDriver对象，只使用s模式时应传入False :param session_or_options: Session对象或SessionOptions对象，只使用d模式时应传入False """ - if not driver_options and driver_or_options: - driver_options = driver_or_options + if not chromium_options and driver_or_options: + chromium_options = driver_or_options self._mode = mode.lower() if self._mode not in ('s', 'd'): raise ValueError('mode参数只能是s或d。') @@ -31,10 +31,10 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self._has_session = True super().__init__(session_or_options=session_or_options) - if not driver_options: - driver_options = ChromiumOptions(read_file=driver_options) - driver_options.set_timeouts(implicit=self._timeout).set_paths(download_path=self.download_path) - super(SessionPage, self).__init__(addr_or_opts=driver_options, timeout=timeout) + if not chromium_options: + chromium_options = ChromiumOptions(read_file=chromium_options) + chromium_options.set_timeouts(implicit=self._timeout).set_paths(download_path=self.download_path) + super(SessionPage, self).__init__(addr_or_opts=chromium_options, timeout=timeout) self.change_mode(self._mode, go=False, copy_cookies=False) def __call__(self, loc_or_str, timeout=None): @@ -239,7 +239,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): # s模式转d模式 if self._mode == 'd': if self._driver is None: - self._connect_browser(self._driver_options) + self._connect_browser(self._chromium_options) self._url = None if not self._has_driver else super(SessionPage, self).url self._has_driver = True diff --git a/DrissionPage/_pages/web_page.pyi b/DrissionPage/_pages/web_page.pyi index 8f461d0..bdbd81f 100644 --- a/DrissionPage/_pages/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -25,13 +25,13 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def __init__(self, mode: str = 'd', timeout: float = None, - driver_options: Union[ChromiumOptions, bool] = None, + chromium_options: Union[ChromiumOptions, bool] = None, session_or_options: Union[Session, SessionOptions, bool] = None) -> None: self._mode: str = ... self._has_driver: bool = ... self._has_session: bool = ... self._session_options: Union[SessionOptions, None] = ... - self._driver_options: Union[ChromiumOptions, None] = ... + self._chromium_options: Union[ChromiumOptions, None] = ... def __call__(self, loc_or_str: Union[Tuple[str, str], str, ChromiumElement, SessionElement], diff --git a/setup.py b/setup.py index 1bd9af1..33e5c2b 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b12", + version="4.0.0b13", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From 893a8e495789d5eddbc91aa343d26265213307b7 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 27 Nov 2023 18:06:14 +0800 Subject: [PATCH 109/182] =?UTF-8?q?4.0.0b14=E5=B0=9D=E8=AF=95=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E8=8E=B7=E5=8F=96=E6=96=87=E6=A1=A3=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/chromium_driver.py | 6 +-- DrissionPage/_configs/options_manage.py | 4 +- DrissionPage/_pages/chromium_base.py | 69 ++++++++++++++----------- DrissionPage/_pages/chromium_base.pyi | 3 +- DrissionPage/_pages/chromium_frame.py | 65 +++++++++++------------ DrissionPage/_pages/chromium_frame.pyi | 2 +- DrissionPage/_units/network_listener.py | 3 +- 7 files changed, 79 insertions(+), 73 deletions(-) diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index 6ba3127..f1963f5 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -162,10 +162,8 @@ class ChromiumDriver(object): if result is None: return {'error': 'tab closed', 'type': 'tab_closed'} if 'result' not in result and 'error' in result: - return {'error': result['error']['message'], - 'type': result.get('type', 'call_method_error'), - 'method': _method, - 'args': kwargs} + return {'error': result['error']['message'], 'type': result.get('type', 'call_method_error'), + 'method': _method, 'args': kwargs} return result['result'] diff --git a/DrissionPage/_configs/options_manage.py b/DrissionPage/_configs/options_manage.py index 0be5b8f..a524c08 100644 --- a/DrissionPage/_configs/options_manage.py +++ b/DrissionPage/_configs/options_manage.py @@ -26,8 +26,8 @@ class OptionsManager(object): self.ini_path = str(path) if not Path(self.ini_path).exists(): - raise FileNotFoundError('\nini文件不存在。\n如果是打包使用，请查看打包注意事项\nhttps://g1879.gitee.io/drission' - 'pagedocs/advance/packaging/') + input('\nini文件不存在。\n如果是打包使用，请查看打包注意事项\nhttps://g1879.gitee.io/drission' + 'pagedocs/advance/packaging/') self._conf = RawConfigParser() self._conf.read(self.ini_path, encoding='utf-8') diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 3783ee9..3e67165 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -25,8 +25,7 @@ from .._units.scroller import PageScroller from .._units.setter import ChromiumBaseSetter from .._units.states import PageStates from .._units.waiter import BaseWaiter -from ..errors import (ContextLostError, ElementLostError, CDPError, PageClosedError, GetDocumentError, - ElementNotFoundError) +from ..errors import ContextLostError, ElementLostError, CDPError, PageClosedError, ElementNotFoundError __ERROR__ = 'error' @@ -57,6 +56,7 @@ class ChromiumBase(BasePage): self._upload_list = None self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc self._download_path = None + self._load_end_time = 0 if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): address = f'127.0.0.1:{address}' @@ -142,31 +142,37 @@ class ChromiumBase(BasePage): self._driver.set_callback('Page.frameAttached', self._onFrameAttached) self._driver.set_callback('Page.frameDetached', self._onFrameDetached) - def _get_document(self): + def _get_document(self, timeout=10): + """获取页面文档 + :param timeout: 超时时间 + :return: 是否获取成功 + """ if self._debug: print('获取文档开始') if self._is_reading: return + timeout = timeout if timeout >= .5 else .5 self._is_reading = True - end_time = perf_counter() + 10 - while perf_counter() < end_time: - try: - b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] - self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id)['object']['objectId'] - break - except: - continue - else: - raise GetDocumentError + try: + b_id = self.run_cdp('DOM.getDocument', _timeout=timeout)['root']['backendNodeId'] + self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id, _timeout=1)['object']['objectId'] - r = self.run_cdp('Page.getFrameTree') - for i in findall(r"'id': '(.*?)'", str(r)): - self.browser._frames[i] = self.tab_id + r = self.run_cdp('Page.getFrameTree') + for i in findall(r"'id': '(.*?)'", str(r)): + self.browser._frames[i] = self.tab_id + if self._debug: + print('获取文档结束') + return True - self._is_loading = False - self._is_reading = False - if self._debug: - print('获取文档结束') + except: + print('获取文档失败') + if self._debug: + print('获取文档失败') + return False + + finally: + self._is_loading = False + self._is_reading = False def _onFrameDetached(self, **kwargs): self.browser._frames.pop(kwargs['frameId'], None) @@ -185,6 +191,7 @@ class ChromiumBase(BasePage): self._doc_got = False self._ready_state = 'loading' self._is_loading = True + self._load_end_time = perf_counter() + self.timeouts.page_load if self._load_mode == 'eager': t = Thread(target=self._wait_to_stop) t.daemon = True @@ -215,7 +222,7 @@ class ChromiumBase(BasePage): if self._load_mode == 'eager': self.run_cdp('Page.stopLoading') - self._get_document() + self._get_document(self._load_end_time - perf_counter() - .1) self._doc_got = True self._ready_state = 'interactive' @@ -229,7 +236,7 @@ class ChromiumBase(BasePage): print('在LoadEventFired变成complete') if self._doc_got is False: - self._get_document() + self._get_document(self._load_end_time - perf_counter() - .1) self._doc_got = True self._ready_state = 'complete' @@ -245,7 +252,7 @@ class ChromiumBase(BasePage): print('在FrameStoppedLoading变成complete') if self._doc_got is False: - self._get_document() + self._get_document(self._load_end_time - perf_counter() - .1) self._ready_state = 'complete' if self._debug: @@ -680,7 +687,7 @@ class ChromiumBase(BasePage): print('停止页面加载') try: self.run_cdp('Page.stopLoading') - except PageClosedError: + except (PageClosedError, CDPError): pass end_time = perf_counter() + self.timeouts.page_load while self._ready_state != 'complete' and perf_counter() < end_time: @@ -910,9 +917,10 @@ class ChromiumBase(BasePage): err = TimeoutError('页面连接超时。') if err: - sleep(interval) - if self._debug or show_errmsg: - print(f'重试{t + 1} {to_url}') + if t < times: + sleep(interval) + if self._debug or show_errmsg: + print(f'重试{t + 1} {to_url}') self.stop_loading() continue @@ -923,9 +931,10 @@ class ChromiumBase(BasePage): ok = self._wait_loaded(1 if yu <= 0 else yu) if not ok: err = TimeoutError('页面连接超时。') - sleep(interval) - if self._debug or show_errmsg: - print(f'重试{t + 1} {to_url}') + if t < times: + sleep(interval) + if self._debug or show_errmsg: + print(f'重试{t + 1} {to_url}') continue if not err: diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index df43bbd..eb5b437 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -54,6 +54,7 @@ class ChromiumBase(BasePage): self._alert: Alert = ... self._has_alert: bool = ... self._doc_got: bool = ... + self._load_end_time: float = ... self._ready_state: Optional[str] = ... self._rect: TabRect = ... @@ -61,7 +62,7 @@ class ChromiumBase(BasePage): def _driver_init(self, tab_id: str) -> None: ... - def _get_document(self) -> None: ... + def _get_document(self, timeout: float = 10) -> bool: ... def _wait_loaded(self, timeout: float = None) -> bool: ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 16460b6..d5c2824 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -108,8 +108,7 @@ class ChromiumFrame(ChromiumBase): self._frame_ele = ChromiumElement(self._target_page, backend_id=self._backend_id) end_time = perf_counter() + 2 while perf_counter() < end_time: - node = self._target_page.run_cdp('DOM.describeNode', - backendNodeId=self._frame_ele._backend_id)['node'] + node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self._frame_ele._backend_id)['node'] if 'frameId' in node: break @@ -121,8 +120,7 @@ class ChromiumFrame(ChromiumBase): if self._is_inner_frame(): self._is_diff_domain = False - self.doc_ele = ChromiumElement(self._target_page, - backend_id=node['contentDocument']['backendNodeId']) + self.doc_ele = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) self._frame_id = node['frameId'] super().__init__(self.address, self._target_page.tab_id, self._target_page.timeout) self._debug = debug @@ -152,47 +150,46 @@ class ChromiumFrame(ChromiumBase): if self._debug: print(f'{self._frame_id} reload 完毕') - def _get_document(self): - """刷新cdp使用的document数据""" + def _get_document(self, timeout=10): + """刷新cdp使用的document数据 + :param timeout: 超时时间 + :return: 是否获取成功 + """ if self._is_reading: return if self._debug: - print('>>> get new doc') + print('获取文档开始') self._is_reading = True - end_time = perf_counter() + 10 - while perf_counter() < end_time: - try: - if self._is_diff_domain is False: - node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self._backend_id)['node'] - self.doc_ele = ChromiumElement(self._target_page, - backend_id=node['contentDocument']['backendNodeId']) + try: + if self._is_diff_domain is False: + node = self._target_page.run_cdp('DOM.describeNode', backendNodeId=self._backend_id)['node'] + self.doc_ele = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) - else: - b_id = self.run_cdp('DOM.getDocument')['root']['backendNodeId'] - self.doc_ele = ChromiumElement(self, backend_id=b_id) + else: + timeout = timeout if timeout >= .5 else .5 + b_id = self.run_cdp('DOM.getDocument', _timeout=timeout)['root']['backendNodeId'] + self.doc_ele = ChromiumElement(self, backend_id=b_id) - self._root_id = self.doc_ele._obj_id + self._root_id = self.doc_ele._obj_id - break + r = self.run_cdp('Page.getFrameTree') + for i in findall(r"'id': '(.*?)'", str(r)): + self.browser._frames[i] = self.tab_id + if self._debug: + print('获取文档结束') + return True - except: - continue + except: + if self._debug: + print('获取文档失败') + return False - else: - raise GetDocumentError - - r = self.run_cdp('Page.getFrameTree') - for i in findall(r"'id': '(.*?)'", str(r)): - self.browser._frames[i] = self.tab_id - - if not self._reloading: # 阻止reload时标识 - self._is_loading = False - self._is_reading = False - - if self._debug: - print('>>> new doc got') + finally: + if not self._reloading: # 阻止reload时标识 + self._is_loading = False + self._is_reading = False def _onInspectorDetached(self, **kwargs): """异域转同域或退出""" diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 9c627b2..f203667 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -47,7 +47,7 @@ class ChromiumFrame(ChromiumBase): def _reload(self) -> None: ... - def _get_document(self) -> None: ... + def _get_document(self, timeout: float = 10) -> bool: ... def _onFrameStoppedLoading(self, **kwargs): ... diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/network_listener.py index 08c47b5..ea57ae8 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/network_listener.py @@ -199,7 +199,8 @@ class NetworkListener(object): p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, None)) p._raw_request = kwargs if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): - p._raw_post_data = self._driver.run('Network.getRequestPostData', requestId=rid)['postData'] + p._raw_post_data = self._driver.run('Network.getRequestPostData', + requestId=rid).get('postData', None) else: rid = kwargs['requestId'] From ddd7aba9ae62d961fe73b8d240cd39cd984228f7 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 27 Nov 2023 22:08:38 +0800 Subject: [PATCH 110/182] =?UTF-8?q?get=5Ftab()=E5=8F=82=E6=95=B0=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=EF=BC=8C=E5=8F=AF=E6=8E=A5=E6=94=B6=E5=BA=8F=E5=8F=B7?= =?UTF-8?q?=EF=BC=9B=E4=BF=AE=E5=A4=8D8x=E7=89=88=E6=B5=8F=E8=A7=88?= =?UTF-8?q?=E5=99=A8select=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_page.py | 15 ++++++++++++--- DrissionPage/_pages/chromium_page.pyi | 2 +- DrissionPage/_pages/web_page.py | 15 ++++++++++++--- DrissionPage/_pages/web_page.pyi | 2 +- DrissionPage/_units/select_element.py | 2 +- 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 677ce88..5c825be 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -138,12 +138,21 @@ class ChromiumPage(ChromiumBase): """返回浏览器进程id""" return self.browser.process_id - def get_tab(self, tab_id=None): + def get_tab(self, id_or_num=None): """获取一个标签页对象 - :param tab_id: 要获取的标签页id，为None时获取当前tab + :param id_or_num: 要获取的标签页id或序号，为None时获取当前tab，序号不是视觉排列顺序，而是激活顺序 :return: 标签页对象 """ - return tab_id if isinstance(tab_id, ChromiumTab) else ChromiumTab(self, tab_id or self.tab_id) + if isinstance(id_or_num, str): + return ChromiumTab(self, id_or_num) + elif isinstance(id_or_num, int): + return ChromiumTab(self, self.tabs[id_or_num]) + elif id_or_num is None: + return ChromiumTab(self, self.tab_id) + elif isinstance(id_or_num, ChromiumTab): + return id_or_num + else: + raise TypeError(f'id_or_num需传入tab id或序号，非{id_or_num}。') def find_tabs(self, title=None, url=None, tab_type=None, single=True): """查找符合条件的tab，返回它们的id组成的列表 diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index b4bc381..49e519b 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -54,7 +54,7 @@ class ChromiumPage(ChromiumBase): @property def set(self) -> ChromiumPageSetter: ... - def get_tab(self, tab_id: Union[str, ChromiumTab] = None) -> ChromiumTab: ... + def get_tab(self, tab_id: Union[str, ChromiumTab, int] = None) -> ChromiumTab: ... def find_tabs(self, title: str = None, url: str = None, tab_type: Union[str, list, tuple] = None, single: bool = True) -> Union[str, List[str]]: ... diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 5e7db55..339436d 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -297,12 +297,21 @@ class WebPage(SessionPage, ChromiumPage, BasePage): elif self._mode == 'd': return super(SessionPage, self).get_cookies(as_dict, all_domains, all_info) - def get_tab(self, tab_id=None): + def get_tab(self, id_or_num=None): """获取一个标签页对象 - :param tab_id: 要获取的标签页id，为None时获取当前tab + :param id_or_num: 要获取的标签页id或序号，为None时获取当前tab，序号不是视觉排列顺序，而是激活顺序 :return: 标签页对象 """ - return tab_id if isinstance(tab_id, WebPageTab) else WebPageTab(self, tab_id or self.tab_id) + if isinstance(id_or_num, str): + return WebPageTab(self, id_or_num) + elif isinstance(id_or_num, int): + return WebPageTab(self, self.tabs[id_or_num]) + elif id_or_num is None: + return WebPageTab(self, self.tab_id) + elif isinstance(id_or_num, WebPageTab): + return id_or_num + else: + raise TypeError(f'id_or_num需传入tab id或序号，非{id_or_num}。') def new_tab(self, url=None, new_window=False, background=False, new_context=False): """新建一个标签页 diff --git a/DrissionPage/_pages/web_page.pyi b/DrissionPage/_pages/web_page.pyi index bdbd81f..7931c75 100644 --- a/DrissionPage/_pages/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -122,7 +122,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def get_cookies(self, as_dict: bool = False, all_domains: bool = False, all_info: bool = False) -> Union[dict, list]: ... - def get_tab(self, tab_id: Union[str, WebPageTab] = None) -> WebPageTab: ... + def get_tab(self, id_or_num: Union[str, WebPageTab, int] = None) -> WebPageTab: ... def new_tab(self, url: str = None, new_window: bool = False, background: bool = False, new_context: bool = False) -> WebPageTab: ... diff --git a/DrissionPage/_units/select_element.py b/DrissionPage/_units/select_element.py index 9399bb4..9494d16 100644 --- a/DrissionPage/_units/select_element.py +++ b/DrissionPage/_units/select_element.py @@ -36,7 +36,7 @@ class SelectElement(object): @property def options(self): """返回所有选项元素组成的列表""" - return self._ele.eles('xpath://option') + return [i for i in self._ele.eles('xpath://option') if not isinstance(i, int)] @property def selected_option(self): From af6a759b4ad5acf748af613a177c59a687d36357 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 28 Nov 2023 17:44:03 +0800 Subject: [PATCH 111/182] =?UTF-8?q?quit()=E7=9A=84force=E5=8F=82=E6=95=B0?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E7=AB=8B=E5=8D=B3=E6=89=A7=E8=A1=8C=EF=BC=9B?= =?UTF-8?q?=E5=85=B6=E5=AE=83=E5=BE=AE=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_base/browser.py | 17 +++++++++-------- DrissionPage/_base/browser.pyi | 2 +- DrissionPage/_configs/chromium_options.py | 2 +- DrissionPage/_configs/configs.ini | 1 - DrissionPage/_pages/chromium_base.py | 1 - DrissionPage/_pages/chromium_page.py | 3 +-- DrissionPage/_pages/web_page.py | 3 +-- setup.py | 2 +- 9 files changed, 15 insertions(+), 18 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 6be91ef..0b0732b 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b13' +__version__ = '4.0.0b15' diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 3fdabcd..7f882c0 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -156,15 +156,22 @@ class Browser(object): """ return self.run_cdp('Browser.getWindowForTarget', targetId=tab_id or self.id)['bounds'] - def quit(self, timeout=5, force=True): + def quit(self, timeout=5, force=False): """关闭浏览器 :param timeout: 等待浏览器关闭超时时间 - :param force: 关闭超时是否强制终止进程 + :param force: 是否立刻强制终止进程 :return: None """ self.run_cdp('Browser.close') self.driver.stop() + if force: + ip, port = self.address.split(':') + if ip not in ('127.0.0.1', 'localhost'): + return + stop_process_on_port(port) + return + if self.process_id: from os import popen from platform import system @@ -180,11 +187,5 @@ class Browser(object): except TypeError: pass - if force: - ip, port = self.address.split(':') - if ip not in ('127.0.0.1', 'localhost'): - return - stop_process_on_port(port) - def _on_quit(self): Browser.BROWSERS.pop(self.id, None) diff --git a/DrissionPage/_base/browser.pyi b/DrissionPage/_base/browser.pyi index 190662f..963e3f7 100644 --- a/DrissionPage/_base/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -57,6 +57,6 @@ class Browser(object): def _onTargetDestroyed(self, **kwargs) -> None: ... - def quit(self, timeout: float = 5, force: bool = True) -> None: ... + def quit(self, timeout: float = 5, force: bool = False) -> None: ... def _on_quit(self) -> None: ... diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index 9ec74b0..fdd2edf 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -40,7 +40,7 @@ class ChromiumOptions(object): self._load_mode = options.get('load_mode', 'normal') self._proxy = om.proxies.get('http', None) self._system_user_path = options.get('system_user_path', False) - self._existing_only = options.get('is_existing_only', False) + self._existing_only = options.get('existing_only', False) user_path = user = False for arg in self._arguments: diff --git a/DrissionPage/_configs/configs.ini b/DrissionPage/_configs/configs.ini index 72722bd..3f0645f 100644 --- a/DrissionPage/_configs/configs.ini +++ b/DrissionPage/_configs/configs.ini @@ -12,7 +12,6 @@ load_mode = normal user = Default auto_port = False system_user_path = False -is_existing_only = False existing_only = False [session_options] diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 3e67165..74320fc 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -165,7 +165,6 @@ class ChromiumBase(BasePage): return True except: - print('获取文档失败') if self._debug: print('获取文档失败') return False diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 5c825be..5eb8158 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -27,8 +27,7 @@ class ChromiumPage(ChromiumBase): :param tab_id: 要控制的标签页id，不指定默认为激活的 :param timeout: 超时时间 """ - if not addr_or_opts and addr_driver_opts: - addr_or_opts = addr_driver_opts + addr_or_opts = addr_or_opts or addr_driver_opts self._page = self address = self._handle_options(addr_or_opts) self._run_browser() diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 339436d..eaca23b 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -22,8 +22,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): :param chromium_options: ChromiumDriver对象，只使用s模式时应传入False :param session_or_options: Session对象或SessionOptions对象，只使用d模式时应传入False """ - if not chromium_options and driver_or_options: - chromium_options = driver_or_options + chromium_options = chromium_options or driver_or_options self._mode = mode.lower() if self._mode not in ('s', 'd'): raise ValueError('mode参数只能是s或d。') diff --git a/setup.py b/setup.py index 33e5c2b..7b3053d 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b13", + version="4.0.0b15", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From 072aad6eaf6e9b374f65dd2dc2b67186b09f0939 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 29 Nov 2023 14:42:42 +0800 Subject: [PATCH 112/182] =?UTF-8?q?=E5=BE=AE=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_commons/browser.py | 6 +++--- DrissionPage/_pages/chromium_page.py | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_commons/browser.py index b263674..e5431b2 100644 --- a/DrissionPage/_commons/browser.py +++ b/DrissionPage/_commons/browser.py @@ -209,10 +209,10 @@ def test_connect(ip, port, timeout=30): sleep(.2) raise BrowserConnectError(f'\n{ip}:{port}浏览器无法链接。\n请确认：\n1、该端口为浏览器\n' - f'2、已添加--remote-debugging-port={port}启动项\n' + f'2、已添加\'--remote-debugging-port={port}\'启动项\n' f'3、用户文件夹没有和已打开的浏览器冲突\n' - f'4、如为无界面系统，请添加--headless=new参数\n' - f'5、如果是Linux系统，可能还要添加--no-sandbox启动参数\n' + f'4、如为无界面系统，请添加\'--headless=new\'参数\n' + f'5、如果是Linux系统，可能还要添加\'--no-sandbox\'启动参数\n' f'可使用ChromiumOptions设置端口和用户文件夹路径。') diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 5eb8158..0814dce 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -62,10 +62,13 @@ class ChromiumPage(ChromiumBase): def _run_browser(self): """连接浏览器""" is_exist = connect_browser(self._chromium_options) - ws = get(f'http://{self._chromium_options.debugger_address}/json/version', - headers={'Connection': 'close'}) - if not ws: - raise BrowserConnectError('\n浏览器连接失败，请检查是否启用全局代理。如有，须开放127.0.0.1地址。') + try: + ws = get(f'http://{self._chromium_options.debugger_address}/json/version', headers={'Connection': 'close'}) + if not ws: + raise BrowserConnectError('\n浏览器连接失败，请检查是否启用全局代理。如是，须设置不代理127.0.0.1地址。') + except : + raise BrowserConnectError('\n浏览器连接失败，请检查是否启用全局代理。如是，须设置不代理127.0.0.1地址。') + ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] self._browser = Browser(self._chromium_options.debugger_address, ws, self) From 1e311e778c9500a1d51b871c21fbb65cd1bc99a0 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 29 Nov 2023 17:16:31 +0800 Subject: [PATCH 113/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=80=E4=B8=AA?= =?UTF-8?q?=E8=AE=BE=E7=BD=AEcookies=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_commons/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DrissionPage/_commons/web.py b/DrissionPage/_commons/web.py index c2598c6..5c1d182 100644 --- a/DrissionPage/_commons/web.py +++ b/DrissionPage/_commons/web.py @@ -207,7 +207,7 @@ def cookies_to_tuple(cookies): cookies = tuple(cookie_to_dict(cookie) for cookie in cookies) elif isinstance(cookies, str): - cookies = tuple(cookie_to_dict(cookie.lstrip()) for cookie in cookies.split(";")) + cookies = tuple(cookie_to_dict(c.lstrip()) for c in cookies.rstrip(';,').split(',' if ',' in cookies else ';')) elif isinstance(cookies, dict): cookies = tuple({'name': cookie, 'value': cookies[cookie]} for cookie in cookies) From 5090fd5c0bcda95c1e24d37bc4a181d74df8ee36 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 30 Nov 2023 00:11:46 +0800 Subject: [PATCH 114/182] =?UTF-8?q?=E5=BE=AE=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_commons/tools.py | 3 ++- DrissionPage/errors.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/DrissionPage/_commons/tools.py b/DrissionPage/_commons/tools.py index 4811a40..afcb270 100644 --- a/DrissionPage/_commons/tools.py +++ b/DrissionPage/_commons/tools.py @@ -274,6 +274,7 @@ def raise_error(r): elif error in ('Node does not have a layout object', 'Could not compute box model.'): raise NoRectError elif r['type'] == 'call_method_error': - raise CDPError(f'\n错误：{r["error"]}\nmethod：{r["method"]}\nargs：{r["args"]}') + raise CDPError(f'\n错误：{r["error"]}\nmethod：{r["method"]}\nargs：{r["args"]}\n出现这个错误可能意味着程序有bug，' + '请把错误信息和重现方法告知作者，谢谢。\n报告网站：https://gitee.com/g1879/DrissionPage/issues') else: raise RuntimeError(r) diff --git a/DrissionPage/errors.py b/DrissionPage/errors.py index 6bd9abd..7423445 100644 --- a/DrissionPage/errors.py +++ b/DrissionPage/errors.py @@ -38,7 +38,7 @@ class ContextLostError(BaseError): class ElementLostError(BaseError): - _info = '元素对象因刷新已失效。' + _info = '元素对象已失效。可能是页面整体刷新，或js局部刷新把元素替换或去除了。' class CDPError(BaseError): From 018c944405339c9967600d802730fef1f51ca492 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 1 Dec 2023 17:22:35 +0800 Subject: [PATCH 115/182] =?UTF-8?q?4.0.0b15=EF=BC=88+=EF=BC=89=20get=5Fsrc?= =?UTF-8?q?()=E5=8F=AF=E8=8E=B7=E5=8F=96src=E5=B1=9E=E6=80=A7=E5=86=85?= =?UTF-8?q?=E7=9A=84base64=E6=95=B0=E6=8D=AE=20NoneElement=5Fvalue?= =?UTF-8?q?=E6=94=B9=E7=94=A8=E9=A1=B5=E9=9D=A2=E5=AF=B9=E8=B1=A1=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/base.py | 40 ++++++++++++---------- DrissionPage/_base/base.pyi | 4 ++- DrissionPage/_commons/settings.py | 1 - DrissionPage/_elements/chromium_element.py | 37 +++++++++++++------- DrissionPage/_elements/none_element.py | 15 +++++--- DrissionPage/_elements/session_element.py | 2 +- DrissionPage/_pages/chromium_base.py | 2 +- DrissionPage/_units/setter.py | 20 +++++++++-- DrissionPage/_units/setter.pyi | 14 ++++++-- 9 files changed, 90 insertions(+), 45 deletions(-) diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index 45a79f4..2f017f2 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -156,19 +156,21 @@ class DrissionElement(BaseElement): nodes = self.children(filter_loc=filter_loc, timeout=timeout, ele_only=ele_only) if not nodes: if Settings.raise_when_ele_not_found: - raise ElementNotFoundError(None, 'child()', - {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) + raise ElementNotFoundError(None, 'child()', {'filter_loc': filter_loc, + 'index': index, 'ele_only': ele_only}) else: - return NoneElement('child()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) + return NoneElement(self.page, 'child()', {'filter_loc': filter_loc, + 'index': index, 'ele_only': ele_only}) try: return nodes[index - 1] except IndexError: if Settings.raise_when_ele_not_found: - raise ElementNotFoundError(None, 'child()', - {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) + raise ElementNotFoundError(None, 'child()', {'filter_loc': filter_loc, + 'index': index, 'ele_only': ele_only}) else: - return NoneElement('child()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) + return NoneElement(self.page, 'child()', {'filter_loc': filter_loc, + 'index': index, 'ele_only': ele_only}) def prev(self, filter_loc='', index=1, timeout=0, ele_only=True): """返回前面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -185,10 +187,10 @@ class DrissionElement(BaseElement): if nodes: return nodes[-1] if Settings.raise_when_ele_not_found: - raise ElementNotFoundError(None, 'prev()', - {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) + raise ElementNotFoundError(None, 'prev()', {'filter_loc': filter_loc, + 'index': index, 'ele_only': ele_only}) else: - return NoneElement('prev()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) + return NoneElement(self.page, 'prev()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) def next(self, filter_loc='', index=1, timeout=0, ele_only=True): """返回后面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -205,10 +207,10 @@ class DrissionElement(BaseElement): if nodes: return nodes[0] if Settings.raise_when_ele_not_found: - raise ElementNotFoundError(None, 'next()', - {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) + raise ElementNotFoundError(None, 'next()', {'filter_loc': filter_loc, + 'index': index, 'ele_only': ele_only}) else: - return NoneElement('next()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) + return NoneElement(self.page, 'next()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) def before(self, filter_loc='', index=1, timeout=None, ele_only=True): """返回前面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -225,10 +227,10 @@ class DrissionElement(BaseElement): if nodes: return nodes[-1] if Settings.raise_when_ele_not_found: - raise ElementNotFoundError(None, 'before()', - {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) + raise ElementNotFoundError(None, 'before()', {'filter_loc': filter_loc, + 'index': index, 'ele_only': ele_only}) else: - return NoneElement('before()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) + return NoneElement(self.page, 'before()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) def after(self, filter_loc='', index=1, timeout=None, ele_only=True): """返回后面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -245,10 +247,10 @@ class DrissionElement(BaseElement): if nodes: return nodes[0] if Settings.raise_when_ele_not_found: - raise ElementNotFoundError(None, 'after()', - {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) + raise ElementNotFoundError(None, 'after()', {'filter_loc': filter_loc, + 'index': index, 'ele_only': ele_only}) else: - return NoneElement('after()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) + return NoneElement(self.page, 'after()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) def children(self, filter_loc='', timeout=None, ele_only=True): """返回直接子元素元素或节点组成的列表，可用查询语法筛选 @@ -378,6 +380,8 @@ class BasePage(BaseParser): self.retry_interval = 2 self._DownloadKit = None self._download_path = None + self._none_ele_return_value = False + self._none_ele_value = None @property def title(self): diff --git a/DrissionPage/_base/base.pyi b/DrissionPage/_base/base.pyi index 20b793e..fd18289 100644 --- a/DrissionPage/_base/base.pyi +++ b/DrissionPage/_base/base.pyi @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from abc import abstractmethod -from typing import Union, Tuple, List +from typing import Union, Tuple, List, Any from DownloadKit import DownloadKit @@ -165,6 +165,8 @@ class BasePage(BaseParser): self._timeout: float = ... self._download_path: str = ... self._DownloadKit: DownloadKit = ... + self._none_ele_return_value: bool = ... + self._none_ele_value: Any = ... @property def title(self) -> Union[str, None]: ... diff --git a/DrissionPage/_commons/settings.py b/DrissionPage/_commons/settings.py index ee5cda9..0107271 100644 --- a/DrissionPage/_commons/settings.py +++ b/DrissionPage/_commons/settings.py @@ -9,4 +9,3 @@ class Settings(object): raise_when_ele_not_found = False raise_when_click_failed = False raise_when_wait_failed = False - NoneElement_value = None diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index d89c887..3c7dab8 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -5,6 +5,7 @@ """ from os.path import basename, sep from pathlib import Path +from re import search from time import perf_counter, sleep from .none_element import NoneElement @@ -462,6 +463,14 @@ class ChromiumElement(DrissionElement): sleep(.1) src = self.attr('src') + if src.lower().startswith('data:image'): + if base64_to_bytes: + from base64 import b64decode + return b64decode(src.split(',', 1)[-1]) + + else: + return src.split(',', 1)[-1] + is_blob = src.startswith('blob') result = None end_time = perf_counter() + timeout @@ -494,8 +503,7 @@ class ChromiumElement(DrissionElement): continue node = self.page.run_cdp('DOM.describeNode', backendNodeId=self._backend_id)['node'] - frame = node.get('frameId', None) - frame = frame or self.page._target_id + frame = node.get('frameId', None) or self.page._frame_id try: result = self.page.run_cdp('Page.getResourceContent', frameId=frame, url=src) @@ -532,6 +540,11 @@ class ChromiumElement(DrissionElement): raise NoResourceError path = path or '.' + if not name and self.tag == 'img': + src = self.attr('src') + if src.lower().startswith('data:image'): + r = search(r'data:image/(.*?);base64,', src) + name = f'img.{r.group(1)}' if r else None name = name or basename(self.prop('currentSrc')) path = get_usable_path(f'{path}{sep}{name}').absolute() write_type = 'wb' if isinstance(data, bytes) else 'w' @@ -871,7 +884,7 @@ class ChromiumShadowRoot(BaseElement): if Settings.raise_when_ele_not_found: raise ElementNotFoundError(None, 'child()', {'filter_loc': filter_loc, 'index': index}) else: - return NoneElement('child()', {'filter_loc': filter_loc, 'index': index}) + return NoneElement(self.page, 'child()', {'filter_loc': filter_loc, 'index': index}) try: return nodes[index - 1] @@ -879,7 +892,7 @@ class ChromiumShadowRoot(BaseElement): if Settings.raise_when_ele_not_found: raise ElementNotFoundError(None, 'child()', {'filter_loc': filter_loc, 'index': index}) else: - return NoneElement('child()', {'filter_loc': filter_loc, 'index': index}) + return NoneElement(self.page, 'child()', {'filter_loc': filter_loc, 'index': index}) def next(self, filter_loc='', index=1): """返回当前元素后面一个符合条件的同级元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -893,7 +906,7 @@ class ChromiumShadowRoot(BaseElement): if Settings.raise_when_ele_not_found: raise ElementNotFoundError(None, 'next()', {'filter_loc': filter_loc, 'index': index}) else: - return NoneElement('next()', {'filter_loc': filter_loc, 'index': index}) + return NoneElement(self.page, 'next()', {'filter_loc': filter_loc, 'index': index}) def before(self, filter_loc='', index=1): """返回文档中当前元素前面符合条件的第一个元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -908,7 +921,7 @@ class ChromiumShadowRoot(BaseElement): if Settings.raise_when_ele_not_found: raise ElementNotFoundError(None, 'before()', {'filter_loc': filter_loc, 'index': index}) else: - return NoneElement('before()', {'filter_loc': filter_loc, 'index': index}) + return NoneElement(self.page, 'before()', {'filter_loc': filter_loc, 'index': index}) def after(self, filter_loc='', index=1): """返回文档中此当前元素后面符合条件的第一个元素，可用查询语法筛选，可指定返回筛选结果的第几个 @@ -923,7 +936,7 @@ class ChromiumShadowRoot(BaseElement): if Settings.raise_when_ele_not_found: raise ElementNotFoundError(None, 'after()', {'filter_loc': filter_loc, 'index': index}) else: - return NoneElement('after()', {'filter_loc': filter_loc, 'index': index}) + return NoneElement(self.page, 'after()', {'filter_loc': filter_loc, 'index': index}) def children(self, filter_loc=''): """返回当前元素符合条件的直接子元素或节点组成的列表，可用查询语法筛选 @@ -1033,7 +1046,7 @@ class ChromiumShadowRoot(BaseElement): if loc[0] == 'css selector': if single: nod_id = self.page.run_cdp('DOM.querySelector', nodeId=self._node_id, selector=loc[1])['nodeId'] - result = make_chromium_ele(self.page, node_id=nod_id) if nod_id else NoneElement() + result = make_chromium_ele(self.page, node_id=nod_id) if nod_id else NoneElement(self.page) else: nod_ids = self.page.run_cdp('DOM.querySelectorAll', nodeId=self._node_id, selector=loc[1])['nodeId'] @@ -1042,13 +1055,13 @@ class ChromiumShadowRoot(BaseElement): else: eles = make_session_ele(self.html).eles(loc) if not eles: - result = NoneElement() if single else eles + result = NoneElement(self.page) if single else eles continue css = [i.css_path[61:] for i in eles] if single: node_id = self.page.run_cdp('DOM.querySelector', nodeId=self._node_id, selector=css[0])['nodeId'] - result = make_chromium_ele(self.page, node_id=node_id) if node_id else NoneElement() + result = make_chromium_ele(self.page, node_id=node_id) if node_id else NoneElement(self.page) else: result = [] @@ -1143,7 +1156,7 @@ def find_by_xpath(ele, xpath, single, timeout, relative=True): returnByValue=False, awaitPromise=True, userGesture=True) if single: - return NoneElement() if r['result']['subtype'] == 'null' \ + return NoneElement(ele.page) if r['result']['subtype'] == 'null' \ else make_chromium_ele(ele.page, obj_id=r['result']['objectId']) if r['result']['description'] == 'NodeList(0)': @@ -1181,7 +1194,7 @@ def find_by_css(ele, selector, single, timeout): raise SyntaxError(f'查询语句错误：\n{r}') if single: - return NoneElement() if r['result']['subtype'] == 'null' \ + return NoneElement(ele.page) if r['result']['subtype'] == 'null' \ else make_chromium_ele(ele.page, obj_id=r['result']['objectId']) if r['result']['description'] == 'NodeList(0)': diff --git a/DrissionPage/_elements/none_element.py b/DrissionPage/_elements/none_element.py index 0ef09d3..25732e5 100644 --- a/DrissionPage/_elements/none_element.py +++ b/DrissionPage/_elements/none_element.py @@ -3,23 +3,28 @@ @Author : g1879 @Contact : g1879@qq.com """ -from .._commons.settings import Settings from ..errors import ElementNotFoundError class NoneElement(object): - def __init__(self, method=None, args=None): + def __init__(self, page=None, method=None, args=None): + if page: + self._none_ele_value = page._none_ele_value + self._none_ele_return_value = page._none_ele_return_value + else: + self._none_ele_value = None + self._none_ele_return_value = False self.method = method self.args = args def __call__(self, *args, **kwargs): - if Settings.NoneElement_value is None: + if not self._none_ele_return_value: raise ElementNotFoundError(None, self.method, self.args) else: return self def __getattr__(self, item): - if Settings.NoneElement_value is None: + if not self._none_ele_return_value: raise ElementNotFoundError(None, self.method, self.args) elif item in ('ele', 's_ele', 'parent', 'child', 'next', 'prev', 'before', 'after', 'get_frame', 'shadow_root', 'sr'): @@ -27,7 +32,7 @@ class NoneElement(object): else: if item in ('size', 'link', 'css_path', 'xpath', 'comments', 'texts', 'tag', 'html', 'inner_html', 'attrs', 'text', 'raw_text'): - return Settings.NoneElement_value + return self._none_ele_value else: raise ElementNotFoundError(None, self.method, self.args) diff --git a/DrissionPage/_elements/session_element.py b/DrissionPage/_elements/session_element.py index f646b1c..5c0109e 100644 --- a/DrissionPage/_elements/session_element.py +++ b/DrissionPage/_elements/session_element.py @@ -375,7 +375,7 @@ def make_session_ele(html_or_ele, loc=None, single=True): elif isinstance(ele, str): return ele else: - return NoneElement() + return NoneElement(page) else: # 返回全部 return [SessionElement(e, page) if isinstance(e, HtmlElement) else e for e in ele if e != '\n'] diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 74320fc..f338d5b 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -621,7 +621,7 @@ class ChromiumBase(BasePage): pass if perf_counter() >= end_time: - return NoneElement() if single else [] + return NoneElement(self) if single else [] sleep(.1) diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index da1d97b..d51d2eb 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -11,10 +11,24 @@ from .._commons.tools import show_or_hide_browser from .._commons.web import set_browser_cookies, set_session_cookies -class ChromiumBaseSetter(object): +class BasePageSetter(object): def __init__(self, page): self._page = page + def NoneElement_value(self, value=None, on_off=True): + """设置空元素是否返回设定值 + :param value: 返回的设定值 + :param on_off: 是否启用 + :return: None + """ + self._page._none_ele_return_value = on_off + self._page._none_ele_value = value + + +class ChromiumBaseSetter(BasePageSetter): + def __init__(self, page): + super().__init__(page) + @property def load_mode(self): """返回用于设置页面加载策略的对象""" @@ -190,12 +204,12 @@ class ChromiumPageSetter(TabSetter): return PageWindowSetter(self._page) -class SessionPageSetter(object): +class SessionPageSetter(BasePageSetter): def __init__(self, page): """ :param page: SessionPage对象 """ - self._page = page + super().__init__(page) def retry_times(self, times): """设置连接失败时重连次数""" diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index 15db2b7..b936565 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -5,13 +5,14 @@ """ from http.cookiejar import Cookie from pathlib import Path -from typing import Union, Tuple, Literal +from typing import Union, Tuple, Literal, Any from requests.adapters import HTTPAdapter from requests.auth import HTTPBasicAuth from requests.cookies import RequestsCookieJar from .scroller import PageScroller +from .._base.base import BasePage from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame @@ -23,7 +24,14 @@ from .._pages.web_page import WebPage FILE_EXISTS = Literal['skip', 'rename', 'overwrite', 's', 'r', 'o'] -class ChromiumBaseSetter(object): +class BasePageSetter(object): + def __init__(self, page: BasePage): + self._page: BasePage = ... + + def NoneElement_value(self, value: Any = None, on_off: bool = True) -> None: ... + + +class ChromiumBaseSetter(BasePageSetter): def __init__(self, page): self._page: ChromiumBase = ... @@ -80,7 +88,7 @@ class ChromiumPageSetter(TabSetter): def tab_to_front(self, tab_or_id: Union[str, ChromiumTab] = None) -> None: ... -class SessionPageSetter(object): +class SessionPageSetter(BasePageSetter): def __init__(self, page: SessionPage): self._page: SessionPage = ... From 364700df2ce7d3e7d52f1e98fb3cc64fcf86d45d Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 3 Dec 2023 13:40:13 +0800 Subject: [PATCH 116/182] =?UTF-8?q?4.0.0b16(+)=20timeouts=E7=9A=84implicit?= =?UTF-8?q?=E6=94=B9=E6=88=90base=EF=BC=9B=20debugger=5Faddress=E6=94=B9?= =?UTF-8?q?=E6=88=90address=20ActionChains=E6=94=B9=E6=88=90Actions?= =?UTF-8?q?=EF=BC=9B=20=E4=B8=80=E4=BA=9B=E6=96=87=E4=BB=B6=E5=92=8C?= =?UTF-8?q?=E5=86=85=E9=83=A8=E7=B1=BB=E6=94=B9=E5=90=8D=EF=BC=9B=20wait.d?= =?UTF-8?q?ata=5Fpackets()=E5=8D=B3=E5=B0=86=E5=BA=9F=E5=BC=83=EF=BC=9B=20?= =?UTF-8?q?iframe=E5=88=87=E6=8D=A2=E4=BA=86id=E4=B9=9F=E5=8F=AF=E7=BB=A7?= =?UTF-8?q?=E7=BB=AD=E7=9B=91=E5=90=AC=EF=BC=9B=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E7=9B=91=E5=90=AC=E5=99=A8=E6=9C=89=E6=97=B6=E4=B8=8D=E8=83=BD?= =?UTF-8?q?=E8=8E=B7=E5=8F=96postData=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=9B?= =?UTF-8?q?=20=E4=BF=AE=E5=A4=8D=E7=9B=91=E5=90=AC=E5=99=A8=E4=B8=8D?= =?UTF-8?q?=E8=83=BD=E8=8E=B7=E5=8F=96=E5=90=8C=E5=9F=9Fiframe=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=8C=85=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=9B=20?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=AD=89=E5=BE=85=E6=95=B0=E6=8D=AE=E5=8C=85?= =?UTF-8?q?timeout=E6=97=A0=E6=95=88=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_base/browser.py | 2 +- DrissionPage/_base/browser.pyi | 2 +- DrissionPage/_base/chromium_driver.py | 12 +-- DrissionPage/_commons/browser.py | 6 +- DrissionPage/_configs/chromium_options.py | 57 ++++++++------ DrissionPage/_configs/chromium_options.pyi | 12 +-- DrissionPage/_configs/configs.ini | 6 +- DrissionPage/_configs/session_options.py | 4 +- DrissionPage/_elements/chromium_element.py | 2 +- DrissionPage/_elements/chromium_element.pyi | 2 +- DrissionPage/_pages/chromium_base.py | 20 ++--- DrissionPage/_pages/chromium_base.pyi | 16 ++-- DrissionPage/_pages/chromium_frame.py | 12 +++ DrissionPage/_pages/chromium_frame.pyi | 7 +- DrissionPage/_pages/chromium_page.py | 20 ++--- DrissionPage/_pages/chromium_tab.py | 4 +- DrissionPage/_pages/web_page.py | 6 +- .../_units/{action_chains.py => actions.py} | 2 +- .../_units/{action_chains.pyi => actions.pyi} | 48 ++++++------ .../{download_manager.py => downloader.py} | 0 .../{download_manager.pyi => downloader.pyi} | 0 .../{network_listener.py => listener.py} | 78 +++++++++++++------ .../{network_listener.pyi => listener.pyi} | 13 +++- .../_units/{select_element.py => selector.py} | 0 .../{select_element.pyi => selector.pyi} | 0 DrissionPage/_units/setter.py | 11 +-- DrissionPage/_units/setter.pyi | 2 +- DrissionPage/_units/waiter.py | 18 +++-- DrissionPage/_units/waiter.pyi | 7 +- DrissionPage/common.py | 4 +- setup.py | 2 +- 32 files changed, 227 insertions(+), 150 deletions(-) rename DrissionPage/_units/{action_chains.py => actions.py} (99%) rename DrissionPage/_units/{action_chains.pyi => actions.pyi} (65%) rename DrissionPage/_units/{download_manager.py => downloader.py} (100%) rename DrissionPage/_units/{download_manager.pyi => downloader.pyi} (100%) rename DrissionPage/_units/{network_listener.py => listener.py} (88%) rename DrissionPage/_units/{network_listener.pyi => listener.pyi} (93%) rename DrissionPage/_units/{select_element.py => selector.py} (100%) rename DrissionPage/_units/{select_element.pyi => selector.pyi} (100%) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 0b0732b..cd5855e 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b15' +__version__ = '4.0.0b16' diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 7f882c0..5d2ed0c 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -7,7 +7,7 @@ from time import sleep, perf_counter from .chromium_driver import BrowserDriver, ChromiumDriver from .._commons.tools import stop_process_on_port, raise_error -from .._units.download_manager import DownloadManager +from .._units.downloader import DownloadManager __ERROR__ = 'error' diff --git a/DrissionPage/_base/browser.pyi b/DrissionPage/_base/browser.pyi index 963e3f7..3bf21be 100644 --- a/DrissionPage/_base/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -7,7 +7,7 @@ from typing import List, Optional, Union from .chromium_driver import BrowserDriver, ChromiumDriver from .._pages.chromium_page import ChromiumPage -from .._units.download_manager import DownloadManager +from .._units.downloader import DownloadManager class Browser(object): diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index f1963f5..f8f4b4f 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -9,7 +9,7 @@ from threading import Thread, Event from time import perf_counter from requests import get -from websocket import (WebSocketTimeoutException, WebSocketException, WebSocketConnectionClosedException, \ +from websocket import (WebSocketTimeoutException, WebSocketException, WebSocketConnectionClosedException, create_connection) @@ -70,7 +70,7 @@ class ChromiumDriver(object): self.method_results[ws_id] = Queue() try: self._ws.send(message_json) - except OSError: + except (OSError, WebSocketConnectionClosedException): self.method_results.pop(ws_id, None) return None @@ -190,14 +190,14 @@ class ChromiumDriver(object): event = self.event_queue.get_nowait() function = self.event_handlers.get(event['method']) if function: - if self._debug: - print(f'开始执行 {function.__name__}') + # if self._debug: + # print(f'开始执行 {function.__name__}') try: function(**event['params']) except: pass - if self._debug: - print(f'执行 {function.__name__}完毕') + # if self._debug: + # print(f'执行 {function.__name__}完毕') self.event_handlers.clear() self.method_results.clear() diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_commons/browser.py index e5431b2..4e8237c 100644 --- a/DrissionPage/_commons/browser.py +++ b/DrissionPage/_commons/browser.py @@ -24,10 +24,10 @@ def connect_browser(option): :param option: ChromiumOptions对象 :return: 返回是否接管的浏览器 """ - debugger_address = option.debugger_address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') + address = option.address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') chrome_path = option.browser_path - ip, port = debugger_address.split(':') + ip, port = address.split(':') if ip != '127.0.0.1' or port_is_using(ip, port) or option.is_existing_only: test_connect(ip, port) option._headless = False @@ -86,7 +86,7 @@ def get_launch_args(opt): result.add(i) if not has_user_path and not opt.system_user_path: - port = opt.debugger_address.split(':')[-1] if opt.debugger_address else '0' + port = opt.address.split(':')[-1] if opt.address else '0' path = Path(gettempdir()) / 'DrissionPage' / f'userData_{port}' path.mkdir(parents=True, exist_ok=True) opt.set_user_data_path(path) diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index fdd2edf..b8e2eec 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -4,6 +4,7 @@ @Contact : g1879@qq.com """ from pathlib import Path +from re import search from shutil import rmtree from tempfile import gettempdir, TemporaryDirectory from threading import Lock @@ -36,12 +37,13 @@ class ChromiumOptions(object): self._extensions = options.get('extensions', []) self._prefs = options.get('prefs', {}) self._flags = options.get('flags', {}) - self._debugger_address = options.get('debugger_address', None) + self._address = options.get('address', None) self._load_mode = options.get('load_mode', 'normal') - self._proxy = om.proxies.get('http', None) self._system_user_path = options.get('system_user_path', False) self._existing_only = options.get('existing_only', False) + self._proxy = om.proxies.get('http', None) or om.proxies.get('https', None) + user_path = user = False for arg in self._arguments: if arg.startswith('--user-data-dir='): @@ -54,14 +56,14 @@ class ChromiumOptions(object): break timeouts = om.timeouts - self._timeouts = {'implicit': timeouts['implicit'], + self._timeouts = {'base': timeouts['base'], 'pageLoad': timeouts['page_load'], 'script': timeouts['script']} self._auto_port = options.get('auto_port', False) if self._auto_port: port, path = PortFinder().get_port() - self._debugger_address = f'127.0.0.1:{port}' + self._address = f'127.0.0.1:{port}' self.set_argument('--user-data-dir', path) others = om.others @@ -77,8 +79,8 @@ class ChromiumOptions(object): self._extensions = [] self._prefs = {} self._flags = {} - self._timeouts = {'implicit': 10, 'pageLoad': 30, 'script': 30} - self._debugger_address = '127.0.0.1:9222' + self._timeouts = {'base': 10, 'pageLoad': 30, 'script': 30} + self._address = '127.0.0.1:9222' self._load_mode = 'normal' self._proxy = None self._auto_port = False @@ -125,12 +127,17 @@ class ChromiumOptions(object): @property def debugger_address(self): """返回浏览器地址，ip:port""" - return self._debugger_address + return self._address @debugger_address.setter def debugger_address(self, address): """设置浏览器地址，格式ip:port""" - self.set_debugger_address(address) + self.set_address(address) + + @property + def address(self): + """返回浏览器地址，ip:port""" + return self._address @property def arguments(self): @@ -275,15 +282,16 @@ class ChromiumOptions(object): self.clear_file_flags = True return self - def set_timeouts(self, implicit=None, pageLoad=None, script=None): + def set_timeouts(self, base=None, pageLoad=None, script=None, implicit=None): """设置超时时间，单位为秒 - :param implicit: 默认超时时间 + :param base: 默认超时时间 :param pageLoad: 页面加载超时时间 :param script: 脚本运行超时时间 :return: 当前对象 """ - if implicit is not None: - self._timeouts['implicit'] = implicit + base = base if base is not None else implicit + if base is not None: + self._timeouts['base'] = base if pageLoad is not None: self._timeouts['pageLoad'] = pageLoad if script is not None: @@ -352,6 +360,10 @@ class ChromiumOptions(object): :param proxy: 代理url和端口 :return: 当前对象 """ + if search(r'.*?:.*?@.*?\..*', proxy): + print('你似乎在设置使用账号密码的代理，暂时不支持这种代理，可自行用插件实现需求。') + if not proxy.lower().startswith('socks'): + print('你似乎在设置使用socks代理，暂时不支持这种代理，可自行用插件实现需求。') self._proxy = proxy return self.set_argument('--proxy-server', proxy) @@ -368,25 +380,26 @@ class ChromiumOptions(object): self._load_mode = value.lower() return self - def set_paths(self, browser_path=None, local_port=None, debugger_address=None, download_path=None, - user_data_path=None, cache_path=None): + def set_paths(self, browser_path=None, local_port=None, address=None, download_path=None, + user_data_path=None, cache_path=None, debugger_address=None): """快捷的路径设置函数 :param browser_path: 浏览器可执行文件路径 :param local_port: 本地端口号 - :param debugger_address: 调试浏览器地址，例：127.0.0.1:9222 + :param address: 调试浏览器地址，例：127.0.0.1:9222 :param download_path: 下载文件路径 :param user_data_path: 用户数据路径 :param cache_path: 缓存路径 :return: 当前对象 """ + address = address or debugger_address if browser_path is not None: self.set_browser_path(browser_path) if local_port is not None: self.set_local_port(local_port) - if debugger_address is not None: - self.set_debugger_address(debugger_address) + if address is not None: + self.set_address(address) if download_path is not None: self.set_download_path(download_path) @@ -404,17 +417,17 @@ class ChromiumOptions(object): :param port: 端口号 :return: 当前对象 """ - self._debugger_address = f'127.0.0.1:{port}' + self._address = f'127.0.0.1:{port}' self._auto_port = False return self - def set_debugger_address(self, address): + def set_address(self, address): """设置浏览器地址，格式'ip:port' :param address: 浏览器地址 :return: 当前对象 """ address = address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') - self._debugger_address = address + self._address = address return self def set_browser_path(self, path): @@ -507,7 +520,7 @@ class ChromiumOptions(object): om = OptionsManager(self.ini_path or str(Path(__file__).parent / 'configs.ini')) # 设置chromium_options - attrs = ('debugger_address', 'browser_path', 'arguments', 'extensions', 'user', 'load_mode', + attrs = ('address', 'browser_path', 'arguments', 'extensions', 'user', 'load_mode', 'auto_port', 'system_user_path', 'existing_only', 'flags') for i in attrs: om.set_item('chromium_options', i, self.__getattribute__(f'_{i}')) @@ -517,7 +530,7 @@ class ChromiumOptions(object): # 设置路径 om.set_item('paths', 'download_path', self._download_path or '') # 设置timeout - om.set_item('timeouts', 'implicit', self._timeouts['implicit']) + om.set_item('timeouts', 'base', self._timeouts['base']) om.set_item('timeouts', 'page_load', self._timeouts['pageLoad']) om.set_item('timeouts', 'script', self._timeouts['script']) # 设置重试 diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index 9a1b0d4..ec843ca 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -19,7 +19,7 @@ class ChromiumOptions(object): self._load_mode: str = ... self._timeouts: dict = ... self._proxy: str = ... - self._debugger_address: str = ... + self._address: str = ... self._extensions: list = ... self._prefs: dict = ... self._flags: dict = ... @@ -54,7 +54,7 @@ class ChromiumOptions(object): def proxy(self) -> str: ... @property - def debugger_address(self) -> str: ... + def address(self) -> str: ... @property def arguments(self) -> list: ... @@ -100,7 +100,7 @@ class ChromiumOptions(object): def clear_flags_in_file(self) -> ChromiumOptions: ... - def set_timeouts(self, implicit: float = None, pageLoad: float = None, + def set_timeouts(self, base: float = None, pageLoad: float = None, script: float = None) -> ChromiumOptions: ... def set_user(self, user: str = 'Default') -> ChromiumOptions: ... @@ -125,7 +125,7 @@ class ChromiumOptions(object): def set_local_port(self, port: Union[str, int]) -> ChromiumOptions: ... - def set_debugger_address(self, address: str) -> ChromiumOptions: ... + def set_address(self, address: str) -> ChromiumOptions: ... def set_download_path(self, path: Union[str, Path]) -> ChromiumOptions: ... @@ -134,8 +134,8 @@ class ChromiumOptions(object): def set_cache_path(self, path: Union[str, Path]) -> ChromiumOptions: ... def set_paths(self, browser_path: Union[str, Path] = None, local_port: Union[int, str] = None, - debugger_address: str = None, download_path: Union[str, Path] = None, - user_data_path: Union[str, Path] = None, cache_path: Union[str, Path] = None) -> ChromiumOptions: ... + address: str = None, download_path: Union[str, Path] = None, user_data_path: Union[str, Path] = None, + cache_path: Union[str, Path] = None) -> ChromiumOptions: ... def use_system_user_path(self, on_off: bool = True) -> ChromiumOptions: ... diff --git a/DrissionPage/_configs/configs.ini b/DrissionPage/_configs/configs.ini index 3f0645f..7a22f7a 100644 --- a/DrissionPage/_configs/configs.ini +++ b/DrissionPage/_configs/configs.ini @@ -2,7 +2,7 @@ download_path = [chromium_options] -debugger_address = 127.0.0.1:9222 +address = 127.0.0.1:9222 browser_path = chrome arguments = ['--no-first-run', '--disable-infobars', '--disable-popup-blocking'] extensions = [] @@ -18,12 +18,12 @@ existing_only = False headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8', 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'connection': 'keep-alive', 'accept-charset': 'GB2312,utf-8;q=0.7,*;q=0.7'} [timeouts] -implicit = 10 +base = 10 page_load = 30 script = 30 [proxies] -http = +http = https = [others] diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index 728c6a1..88b396f 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -76,7 +76,7 @@ class SessionOptions(object): self._max_redirects = options['max_redirects'] self.set_proxies(om.proxies.get('http', None), om.proxies.get('https', None)) - self._timeout = om.timeouts.get('implicit', 10) + self._timeout = om.timeouts.get('base', 10) self._download_path = om.paths.get('download_path', None) or None others = om.others @@ -379,7 +379,7 @@ class SessionOptions(object): om.set_item('session_options', i, options[i]) om.set_item('paths', 'download_path', self.download_path or '') - om.set_item('timeouts', 'implicit', self.timeout) + om.set_item('timeouts', 'base', self.timeout) om.set_item('proxies', 'http', self.proxies.get('http', None)) om.set_item('proxies', 'https', self.proxies.get('https', None)) om.set_item('others', 'retry_times', self.retry_times) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 3c7dab8..25d9952 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -19,7 +19,7 @@ from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_f from .._units.clicker import Clicker from .._units.rect import ElementRect from .._units.scroller import ElementScroller -from .._units.select_element import SelectElement +from .._units.selector import SelectElement from .._units.setter import ChromiumElementSetter from .._units.states import ElementStates, ShadowRootStates from .._units.waiter import ElementWaiter diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 9bb0cb4..7725ccd 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -16,7 +16,7 @@ from .._pages.web_page import WebPage from .._units.clicker import Clicker from .._units.rect import ElementRect from .._units.scroller import ElementScroller -from .._units.select_element import SelectElement +from .._units.selector import SelectElement from .._units.setter import ChromiumElementSetter from .._units.states import ShadowRootStates, ElementStates from .._units.waiter import ElementWaiter diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index f338d5b..832c9de 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -17,8 +17,8 @@ from .._commons.web import location_in_viewport from .._elements.chromium_element import ChromiumElement, run_js, make_chromium_ele from .._elements.none_element import NoneElement from .._elements.session_element import make_session_ele -from .._units.action_chains import ActionChains -from .._units.network_listener import NetworkListener +from .._units.actions import Actions +from .._units.listener import Listener from .._units.rect import TabRect from .._units.screencast import Screencast from .._units.scroller import PageScroller @@ -46,7 +46,6 @@ class ChromiumBase(BasePage): self._set = None self._screencast = None self._actions = None - self._listener = None self._states = None self._has_alert = False self._ready_state = None @@ -57,6 +56,8 @@ class ChromiumBase(BasePage): self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc self._download_path = None self._load_end_time = 0 + if not hasattr(self, '_listener'): + self._listener = None if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): address = f'127.0.0.1:{address}' @@ -313,7 +314,7 @@ class ChromiumBase(BasePage): def actions(self): """返回用于执行动作链的对象""" if self._actions is None: - self._actions = ActionChains(self) + self._actions = Actions(self) self.wait.load_complete() return self._actions @@ -321,7 +322,7 @@ class ChromiumBase(BasePage): def listen(self): """返回用于聆听数据包的对象""" if self._listener is None: - self._listener = NetworkListener(self) + self._listener = Listener(self) return self._listener @property @@ -1054,20 +1055,21 @@ class ChromiumBase(BasePage): class Timeout(object): """用于保存d模式timeout信息的类""" - def __init__(self, page, implicit=None, page_load=None, script=None): + def __init__(self, page, base=None, page_load=None, script=None, implicit=None): """ :param page: ChromiumBase页面 - :param implicit: 默认超时时间 + :param base: 默认超时时间 :param page_load: 页面加载超时时间 :param script: js超时时间 """ self._page = page - self.implicit = 10 if implicit is None else implicit + base = base if base is not None else implicit + self.base = 10 if base is None else base self.page_load = 30 if page_load is None else page_load self.script = 30 if script is None else script def __repr__(self): - return str({'implicit': self.implicit, 'page_load': self.page_load, 'script': self.script}) + return str({'base': self.base, 'page_load': self.page_load, 'script': self.script}) class Alert(object): diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index eb5b437..3ba30f9 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -14,8 +14,8 @@ from .._elements.none_element import NoneElement from .._elements.session_element import SessionElement from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage -from .._units.action_chains import ActionChains -from .._units.network_listener import NetworkListener +from .._units.actions import Actions +from .._units.listener import Listener from .._units.rect import TabRect from .._units.screencast import Screencast from .._units.scroller import Scroller, PageScroller @@ -48,8 +48,8 @@ class ChromiumBase(BasePage): self._wait: BaseWaiter = ... self._set: ChromiumBaseSetter = ... self._screencast: Screencast = ... - self._actions: ActionChains = ... - self._listener: NetworkListener = ... + self._actions: Actions = ... + self._listener: Listener = ... self._states: PageStates = ... self._alert: Alert = ... self._has_alert: bool = ... @@ -152,10 +152,10 @@ class ChromiumBase(BasePage): def screencast(self) -> Screencast: ... @property - def actions(self) -> ActionChains: ... + def actions(self) -> Actions: ... @property - def listen(self) -> NetworkListener: ... + def listen(self) -> Listener: ... @property def states(self) -> PageStates: ... @@ -237,9 +237,9 @@ class ChromiumBase(BasePage): class Timeout(object): - def __init__(self, page: ChromiumBase, implicit=None, page_load=None, script=None): + def __init__(self, page: ChromiumBase, base=None, page_load=None, script=None): self._page: ChromiumBase = ... - self.implicit: float = ... + self.base: float = ... self.page_load: float = ... self.script: float = ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index d5c2824..5716eb6 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -9,6 +9,7 @@ from time import sleep, perf_counter from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase +from .._units.listener import FrameListener from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter @@ -122,12 +123,16 @@ class ChromiumFrame(ChromiumBase): self._is_diff_domain = False self.doc_ele = ChromiumElement(self._target_page, backend_id=node['contentDocument']['backendNodeId']) self._frame_id = node['frameId'] + if self._listener: + self._listener._to_target(self._target_page.tab_id, self.address, self) super().__init__(self.address, self._target_page.tab_id, self._target_page.timeout) self._debug = debug self.driver._debug = d_debug else: self._is_diff_domain = True + if self._listener: + self._listener._to_target(node['frameId'], self.address, self) super().__init__(self.address, node['frameId'], self._target_page.timeout) end_time = perf_counter() + self.timeouts.page_load while perf_counter() < end_time: @@ -251,6 +256,13 @@ class ChromiumFrame(ChromiumBase): self._rect = FrameRect(self) return self._rect + @property + def listen(self): + """返回用于聆听数据包的对象""" + if self._listener is None: + self._listener = FrameListener(self) + return self._listener + # ----------挂件---------- @property diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index f203667..b7d6892 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -11,10 +11,11 @@ from .chromium_page import ChromiumPage from .chromium_tab import ChromiumTab from .web_page import WebPage from .._elements.chromium_element import ChromiumElement -from .._units.states import FrameStates +from .._units.listener import FrameListener from .._units.rect import FrameRect from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter +from .._units.states import FrameStates from .._units.waiter import FrameWaiter @@ -33,6 +34,7 @@ class ChromiumFrame(ChromiumBase): self._states: FrameStates = ... self._reloading: bool = ... self._rect: FrameRect = ... + self._listener: FrameListener = ... def __call__(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> Union[ChromiumElement, str]: ... @@ -83,6 +85,9 @@ class ChromiumFrame(ChromiumBase): @property def rect(self) -> FrameRect: ... + @property + def listen(self) -> FrameListener: ... + @property def _obj_id(self) -> str: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 0814dce..0fd998b 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -32,7 +32,7 @@ class ChromiumPage(ChromiumBase): address = self._handle_options(addr_or_opts) self._run_browser() super().__init__(address, tab_id) - self.set.timeouts(implicit=timeout) + self.set.timeouts(base=timeout) self._page_init() def _handle_options(self, addr_or_opts): @@ -48,7 +48,7 @@ class ChromiumPage(ChromiumBase): elif isinstance(addr_or_opts, str): self._chromium_options = ChromiumOptions() - self._chromium_options.set_debugger_address(addr_or_opts) + self._chromium_options.set_address(addr_or_opts) elif isinstance(addr_or_opts, int): self._chromium_options = ChromiumOptions() @@ -57,36 +57,36 @@ class ChromiumPage(ChromiumBase): else: raise TypeError('只能接收ip:port格式或ChromiumOptions类型参数。') - return self._chromium_options.debugger_address + return self._chromium_options.address def _run_browser(self): """连接浏览器""" is_exist = connect_browser(self._chromium_options) try: - ws = get(f'http://{self._chromium_options.debugger_address}/json/version', headers={'Connection': 'close'}) + ws = get(f'http://{self._chromium_options.address}/json/version', headers={'Connection': 'close'}) if not ws: raise BrowserConnectError('\n浏览器连接失败，请检查是否启用全局代理。如是，须设置不代理127.0.0.1地址。') except : raise BrowserConnectError('\n浏览器连接失败，请检查是否启用全局代理。如是，须设置不代理127.0.0.1地址。') ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] - self._browser = Browser(self._chromium_options.debugger_address, ws, self) + self._browser = Browser(self._chromium_options.address, ws, self) if (is_exist and self._chromium_options._headless is False and 'headless' in self._browser.run_cdp('Browser.getVersion')['userAgent'].lower()): self._browser.quit(3) connect_browser(self._chromium_options) - ws = get(f'http://{self._chromium_options.debugger_address}/json/version', headers={'Connection': 'close'}) + ws = get(f'http://{self._chromium_options.address}/json/version', headers={'Connection': 'close'}) ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] - self._browser = Browser(self._chromium_options.debugger_address, ws, self) + self._browser = Browser(self._chromium_options.address, ws, self) def _d_set_runtime_settings(self): """设置运行时用到的属性""" self._timeouts = Timeout(self, page_load=self._chromium_options.timeouts['pageLoad'], script=self._chromium_options.timeouts['script'], - implicit=self._chromium_options.timeouts['implicit']) - if self._chromium_options.timeouts['implicit'] is not None: - self._timeout = self._chromium_options.timeouts['implicit'] + base=self._chromium_options.timeouts['base']) + if self._chromium_options.timeouts['base'] is not None: + self._timeout = self._chromium_options.timeouts['base'] self._load_mode = self._chromium_options.load_mode self._download_path = None if self._chromium_options.download_path is None \ else str(Path(self._chromium_options.download_path).absolute()) diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 64eee41..126153f 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -174,7 +174,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): @property def timeout(self): """返回通用timeout设置""" - return self.timeouts.implicit + return self.timeouts.base @timeout.setter def timeout(self, second): @@ -182,7 +182,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): :param second: 秒数 :return: None """ - self.set.timeouts(implicit=second) + self.set.timeouts(base=second) def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None, **kwargs): """跳转到一个url diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index eaca23b..2f75d8e 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -32,7 +32,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): super().__init__(session_or_options=session_or_options) if not chromium_options: chromium_options = ChromiumOptions(read_file=chromium_options) - chromium_options.set_timeouts(implicit=self._timeout).set_paths(download_path=self.download_path) + chromium_options.set_timeouts(base=self._timeout).set_paths(download_path=self.download_path) super(SessionPage, self).__init__(addr_or_opts=chromium_options, timeout=timeout) self.change_mode(self._mode, go=False, copy_cookies=False) @@ -138,7 +138,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): @property def timeout(self): """返回通用timeout设置""" - return self.timeouts.implicit + return self.timeouts.base @timeout.setter def timeout(self, second): @@ -146,7 +146,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): :param second: 秒数 :return: None """ - self.set.timeouts(implicit=second) + self.set.timeouts(base=second) def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None, **kwargs): """跳转到一个url diff --git a/DrissionPage/_units/action_chains.py b/DrissionPage/_units/actions.py similarity index 99% rename from DrissionPage/_units/action_chains.py rename to DrissionPage/_units/actions.py index df74134..42cee3c 100644 --- a/DrissionPage/_units/action_chains.py +++ b/DrissionPage/_units/actions.py @@ -9,7 +9,7 @@ from .._commons.keys import modifierBit, keyDescriptionForString from .._commons.web import location_in_viewport -class ActionChains: +class Actions: """用于实现动作链的类""" def __init__(self, page): diff --git a/DrissionPage/_units/action_chains.pyi b/DrissionPage/_units/actions.pyi similarity index 65% rename from DrissionPage/_units/action_chains.pyi rename to DrissionPage/_units/actions.pyi index 1c66af6..2f5df4e 100644 --- a/DrissionPage/_units/action_chains.pyi +++ b/DrissionPage/_units/actions.pyi @@ -10,7 +10,7 @@ from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase -class ActionChains: +class Actions: def __init__(self, page: ChromiumBase): self.page: ChromiumBase = ... @@ -20,53 +20,53 @@ class ActionChains: self.curr_y: int = ... def move_to(self, ele_or_loc: Union[ChromiumElement, Tuple[int, int], str], - offset_x: int = 0, offset_y: int = 0, duration: float = .5) -> ActionChains: ... + offset_x: int = 0, offset_y: int = 0, duration: float = .5) -> Actions: ... - def move(self, offset_x: int = 0, offset_y: int = 0, duration: float = .5) -> ActionChains: ... + def move(self, offset_x: int = 0, offset_y: int = 0, duration: float = .5) -> Actions: ... - def click(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def click(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def r_click(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def r_click(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def m_click(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def m_click(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def db_click(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def db_click(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def hold(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def hold(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def release(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def release(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def r_hold(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def r_hold(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def r_release(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def r_release(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def m_hold(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def m_hold(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def m_release(self, on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + def m_release(self, on_ele: Union[ChromiumElement, str] = None) -> Actions: ... def _hold(self, on_ele: Union[ChromiumElement, str] = None, button: str = 'left', - count: int = 1) -> ActionChains: ... + count: int = 1) -> Actions: ... - def _release(self, button: str) -> ActionChains: ... + def _release(self, button: str) -> Actions: ... def scroll(self, delta_x: int = 0, delta_y: int = 0, - on_ele: Union[ChromiumElement, str] = None) -> ActionChains: ... + on_ele: Union[ChromiumElement, str] = None) -> Actions: ... - def up(self, pixel: int) -> ActionChains: ... + def up(self, pixel: int) -> Actions: ... - def down(self, pixel: int) -> ActionChains: ... + def down(self, pixel: int) -> Actions: ... - def left(self, pixel: int) -> ActionChains: ... + def left(self, pixel: int) -> Actions: ... - def right(self, pixel: int) -> ActionChains: ... + def right(self, pixel: int) -> Actions: ... - def key_down(self, key: str) -> ActionChains: ... + def key_down(self, key: str) -> Actions: ... - def key_up(self, key: str) -> ActionChains: ... + def key_up(self, key: str) -> Actions: ... - def type(self, text: Union[str, list, tuple]) -> ActionChains: ... + def type(self, text: Union[str, list, tuple]) -> Actions: ... - def wait(self, second: float) -> ActionChains: ... + def wait(self, second: float) -> Actions: ... def _get_key_data(self, key: str, action: str) -> dict: ... diff --git a/DrissionPage/_units/download_manager.py b/DrissionPage/_units/downloader.py similarity index 100% rename from DrissionPage/_units/download_manager.py rename to DrissionPage/_units/downloader.py diff --git a/DrissionPage/_units/download_manager.pyi b/DrissionPage/_units/downloader.pyi similarity index 100% rename from DrissionPage/_units/download_manager.pyi rename to DrissionPage/_units/downloader.pyi diff --git a/DrissionPage/_units/network_listener.py b/DrissionPage/_units/listener.py similarity index 88% rename from DrissionPage/_units/network_listener.py rename to DrissionPage/_units/listener.py index ea57ae8..3323049 100644 --- a/DrissionPage/_units/network_listener.py +++ b/DrissionPage/_units/listener.py @@ -14,7 +14,7 @@ from requests.structures import CaseInsensitiveDict from .._base.chromium_driver import ChromiumDriver -class NetworkListener(object): +class Listener(object): """监听器基类""" def __init__(self, page): @@ -22,6 +22,8 @@ class NetworkListener(object): :param page: ChromiumBase对象 """ self._page = page + self._address = page.address + self._target_id = page._target_id self._driver = None self._caught = None # 临存捕捉到的数据 @@ -77,7 +79,7 @@ class NetworkListener(object): if self.listening: return - self._driver = ChromiumDriver(self._page.tab_id, 'page', self._page.address) + self._driver = ChromiumDriver(self._target_id, 'page', self._address) self._driver.run('Network.enable') self.listening = True @@ -102,7 +104,7 @@ class NetworkListener(object): fail = False else: - end = perf_counter() + count + end = perf_counter() + timeout while True: if perf_counter() > end: fail = True @@ -179,6 +181,26 @@ class NetworkListener(object): self._extra_info_ids = {} self._caught.queue.clear() + def _to_target(self, target_id, address, page): + """切换监听的页面对象 + :param target_id: 新页面对象_target_id + :param address: 新页面对象address + :param page: 新页面对象 + :return: None + """ + self._target_id = target_id + self._address = address + self._page = page + debug = False + if self._driver: + debug = self._driver._debug + self._driver.stop() + if self.listening: + self._driver = ChromiumDriver(self._target_id, 'page', self._address) + self._driver._debug = debug + self._driver.run('Network.enable') + self._set_callback() + def _set_callback(self): """设置监听请求的回调函数""" self._driver.set_callback('Network.requestWillBeSent', self._requestWillBeSent) @@ -190,8 +212,6 @@ class NetworkListener(object): def _requestWillBeSent(self, **kwargs): """接收到请求时的回调函数""" - if kwargs.get('frameId', self._page._frame_id) != self._page._frame_id: - return p = None if not self._targets: if not self._method or kwargs['request']['method'] in self._method: @@ -210,9 +230,6 @@ class NetworkListener(object): not self._method or kwargs['request']['method'] in self._method): p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, target)) p._raw_request = kwargs - if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): - p._raw_post_data = self._driver.run('Network.getRequestPostData', - requestId=rid)['postData'] break self._extra_info_ids.setdefault(kwargs['requestId'], {})['obj'] = p if p else False @@ -223,8 +240,6 @@ class NetworkListener(object): def _response_received(self, **kwargs): """接收到返回信息时处理方法""" - if kwargs.get('frameId', self._page._frame_id) != self._page._frame_id: - return request = self._request_ids.get(kwargs['requestId'], None) if request: request._raw_response = kwargs['response'] @@ -238,7 +253,7 @@ class NetworkListener(object): if obj is False: self._extra_info_ids.pop(kwargs['requestId'], None) elif isinstance(obj, DataPacket): - obj._requestExtraInfo = r['request'] + obj._requestExtraInfo = r.get('request', None) obj._responseExtraInfo = kwargs self._extra_info_ids.pop(kwargs['requestId'], None) else: @@ -246,16 +261,21 @@ class NetworkListener(object): def _loading_finished(self, **kwargs): """请求完成时处理方法""" - r_id = kwargs['requestId'] - dp = self._request_ids.get(r_id) - if dp: - r = self._driver.run('Network.getResponseBody', requestId=r_id) + rid = kwargs['requestId'] + packet = self._request_ids.get(rid) + if packet: + r = self._driver.run('Network.getResponseBody', requestId=rid) if 'body' in r: - dp._raw_body = r['body'] - dp._base64_body = r['base64Encoded'] + packet._raw_body = r['body'] + packet._base64_body = r['base64Encoded'] else: - dp._raw_body = '' - dp._base64_body = False + packet._raw_body = '' + packet._base64_body = False + + if (packet._raw_request['request'].get('hasPostData', None) + and not packet._raw_request['request'].get('postData', None)): + r = self._driver.run('Network.getRequestPostData', requestId=rid, _timeout=1) + packet._raw_post_data = r.get('postData', None) r = self._extra_info_ids.get(kwargs['requestId'], None) if r: @@ -268,10 +288,10 @@ class NetworkListener(object): obj._responseExtraInfo = response self._extra_info_ids.pop(kwargs['requestId'], None) - self._request_ids.pop(r_id, None) + self._request_ids.pop(rid, None) - if dp: - self._caught.put(dp) + if packet: + self._caught.put(packet) def _loading_failed(self, **kwargs): """请求失败时的回调方法""" @@ -299,6 +319,20 @@ class NetworkListener(object): self._caught.put(dp) +class FrameListener(Listener): + def _requestWillBeSent(self, **kwargs): + """接收到请求时的回调函数""" + if not self._page._is_diff_domain and kwargs.get('frameId', None) != self._page._frame_id: + return + super()._requestWillBeSent(**kwargs) + + def _response_received(self, **kwargs): + """接收到返回信息时处理方法""" + if not self._page._is_diff_domain and kwargs.get('frameId', None) != self._page._frame_id: + return + super()._response_received(**kwargs) + + class DataPacket(object): """返回的数据包管理类""" diff --git a/DrissionPage/_units/network_listener.pyi b/DrissionPage/_units/listener.pyi similarity index 93% rename from DrissionPage/_units/network_listener.pyi rename to DrissionPage/_units/listener.pyi index 8ef48cd..2c3a280 100644 --- a/DrissionPage/_units/network_listener.pyi +++ b/DrissionPage/_units/listener.pyi @@ -10,11 +10,14 @@ from requests.structures import CaseInsensitiveDict from .._base.chromium_driver import ChromiumDriver from .._pages.chromium_base import ChromiumBase +from .._pages.chromium_frame import ChromiumFrame -class NetworkListener(object): +class Listener(object): def __init__(self, page: ChromiumBase): self._page: ChromiumBase = ... + self._address: str = ... + self._target_id: str = ... self._targets: Union[str, dict] = ... self._method: set = ... self._caught: Queue = ... @@ -44,6 +47,8 @@ class NetworkListener(object): def clear(self) -> None: ... + def _to_target(self, target_id: str, address: str, page: ChromiumBase) -> None: ... + def start(self, targets: Union[str, List[str], Tuple, bool, None] = None, is_regex: bool = False, method: Union[str, list, tuple, set] = None) \ -> Union[DataPacket, Dict[str, List[DataPacket]], False]: ... @@ -66,6 +71,12 @@ class NetworkListener(object): def _set_callback(self) -> None: ... +class FrameListener(Listener): + def __init__(self, page: ChromiumFrame, is_diff: bool): + self._page: ChromiumFrame = ... + self._is_diff: bool = ... + + class DataPacket(object): """返回的数据包管理类""" diff --git a/DrissionPage/_units/select_element.py b/DrissionPage/_units/selector.py similarity index 100% rename from DrissionPage/_units/select_element.py rename to DrissionPage/_units/selector.py diff --git a/DrissionPage/_units/select_element.pyi b/DrissionPage/_units/selector.pyi similarity index 100% rename from DrissionPage/_units/select_element.pyi rename to DrissionPage/_units/selector.pyi diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index d51d2eb..d0bd70c 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -47,16 +47,17 @@ class ChromiumBaseSetter(BasePageSetter): """设置连接失败重连间隔""" self._page.retry_interval = interval - def timeouts(self, implicit=None, page_load=None, script=None): + def timeouts(self, base=None, page_load=None, script=None, implicit=None): """设置超时时间，单位为秒 - :param implicit: 查找元素超时时间 + :param base: 基本等待时间，除页面加载和脚本超时，其它等待默认使用 :param page_load: 页面加载超时时间 :param script: 脚本运行超时时间 :return: None """ - if implicit is not None: - self._page.timeouts.implicit = implicit - self._page._timeout = implicit + base = base if base is not None else implicit + if base is not None: + self._page.timeouts.base = base + self._page._timeout = base if page_load is not None: self._page.timeouts.page_load = page_load diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index b936565..a254288 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -45,7 +45,7 @@ class ChromiumBaseSetter(BasePageSetter): def retry_interval(self, interval: float) -> None: ... - def timeouts(self, implicit: float = None, page_load: float = None, script: float = None) -> None: ... + def timeouts(self, base: float = None, page_load: float = None, script: float = None) -> None: ... def user_agent(self, ua: str, platform: str = None) -> None: ... diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 3cbca57..0d09ed3 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -130,14 +130,6 @@ class BaseWaiter(object): """ return self._change('title', text, exclude, timeout, raise_err) - def data_packets(self, count=1, timeout=None, fix_count: bool = True): - """等待符合要求的数据包到达指定数量 - :param count: 需要捕捉的数据包数量 - :param timeout: 超时时间，为None无限等待 - :param fix_count: 是否必须满足总数要求，发生超时，为True返回False，为False返回已捕捉到的数据包 - :return: count为1时返回数据包对象，大于1时返回列表，超时且fix_count为True时返回False""" - return self._driver.listen.wait(count, timeout, fix_count) - def _change(self, arg, text, exclude=False, timeout=None, raise_err=None): """等待指定属性变成包含或不包含指定文本 :param arg: 要被匹配的属性 @@ -189,6 +181,16 @@ class BaseWaiter(object): else: return False + # -----------即将废弃----------- + + def data_packets(self, count=1, timeout=None, fix_count: bool = True): + """等待符合要求的数据包到达指定数量 + :param count: 需要捕捉的数据包数量 + :param timeout: 超时时间，为None无限等待 + :param fix_count: 是否必须满足总数要求，发生超时，为True返回False，为False返回已捕捉到的数据包 + :return: count为1时返回数据包对象，大于1时返回列表，超时且fix_count为True时返回False""" + return self._driver.listen.wait(count, timeout, fix_count) + class TabWaiter(BaseWaiter): diff --git a/DrissionPage/_units/waiter.pyi b/DrissionPage/_units/waiter.pyi index d353178..f8634b1 100644 --- a/DrissionPage/_units/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -5,8 +5,8 @@ """ from typing import Union, List -from .download_manager import DownloadMission -from .network_listener import DataPacket +from .downloader import DownloadMission +from .listener import DataPacket from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame @@ -47,9 +47,6 @@ class BaseWaiter(object): def title_change(self, text: str, exclude: bool = False, timeout: float = None, raise_err: bool = None) -> bool: ... - def data_packets(self, count: int = 1, timeout: float = None, - fix_count: bool = True) -> Union[List[DataPacket], DataPacket, None]: ... - def _change(self, arg: str, text: str, exclude: bool = False, timeout: float = None, raise_err: bool = None) -> bool: ... diff --git a/DrissionPage/common.py b/DrissionPage/common.py index 88f9e57..32fabb2 100644 --- a/DrissionPage/common.py +++ b/DrissionPage/common.py @@ -8,6 +8,6 @@ from ._commons.keys import Keys from ._commons.settings import Settings from ._commons.tools import wait_until, configs_to_here from ._elements.session_element import make_session_ele -from ._units.action_chains import ActionChains +from ._units.actions import Actions -__all__ = ['make_session_ele', 'ActionChains', 'Keys', 'By', 'Settings', 'wait_until', 'configs_to_here'] +__all__ = ['make_session_ele', 'Actions', 'Keys', 'By', 'Settings', 'wait_until', 'configs_to_here'] diff --git a/setup.py b/setup.py index 7b3053d..acb5c6a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b15", + version="4.0.0b16", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From bf245f7221057f43108c861425566b54df2fdca1 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 3 Dec 2023 23:18:27 +0800 Subject: [PATCH 117/182] =?UTF-8?q?4.0.0b17(+)=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E7=9B=B8=E5=AF=B9=E5=AE=9A=E4=BD=8Dtimeout=E5=A4=B1=E6=95=88?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=9B=20=E7=9B=B8=E5=AF=B9=E5=AE=9A?= =?UTF-8?q?=E4=BD=8Dtimeout=E6=94=B9=E4=B8=BANone=EF=BC=9B=20=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0listen.wait=5Fsilent()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_base/base.py | 8 +-- DrissionPage/_base/base.pyi | 65 +++++++--------------- DrissionPage/_base/chromium_driver.py | 4 -- DrissionPage/_elements/chromium_element.py | 6 +- DrissionPage/_pages/chromium_frame.py | 14 +++-- DrissionPage/_units/listener.py | 28 ++++++++++ DrissionPage/_units/listener.pyi | 5 +- setup.py | 2 +- 9 files changed, 69 insertions(+), 65 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index cd5855e..b1205d5 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b16' +__version__ = '4.0.0b17' diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index 2f017f2..23ef08e 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -172,7 +172,7 @@ class DrissionElement(BaseElement): return NoneElement(self.page, 'child()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) - def prev(self, filter_loc='', index=1, timeout=0, ele_only=True): + def prev(self, filter_loc='', index=1, timeout=None, ele_only=True): """返回前面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 :param filter_loc: 用于筛选的查询语法 :param index: 前面第几个查询结果，1开始 @@ -192,7 +192,7 @@ class DrissionElement(BaseElement): else: return NoneElement(self.page, 'prev()', {'filter_loc': filter_loc, 'index': index, 'ele_only': ele_only}) - def next(self, filter_loc='', index=1, timeout=0, ele_only=True): + def next(self, filter_loc='', index=1, timeout=None, ele_only=True): """返回后面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 :param filter_loc: 用于筛选的查询语法 :param index: 后面第几个查询结果，1开始 @@ -271,7 +271,7 @@ class DrissionElement(BaseElement): nodes = self._ele(loc, timeout=timeout, single=False, relative=True) return [e for e in nodes if not (isinstance(e, str) and sub('[ \n\t\r]', '', e) == '')] - def prevs(self, filter_loc='', timeout=0, ele_only=True): + def prevs(self, filter_loc='', timeout=None, ele_only=True): """返回前面全部兄弟元素或节点组成的列表，可用查询语法筛选 :param filter_loc: 用于筛选的查询语法 :param timeout: 查找节点的超时时间 @@ -280,7 +280,7 @@ class DrissionElement(BaseElement): """ return self._get_brothers(filter_loc=filter_loc, direction='preceding', timeout=timeout, ele_only=ele_only) - def nexts(self, filter_loc='', timeout=0, ele_only=True): + def nexts(self, filter_loc='', timeout=None, ele_only=True): """返回后面全部兄弟元素或节点组成的列表，可用查询语法筛选 :param filter_loc: 用于筛选的查询语法 :param timeout: 查找节点的超时时间 diff --git a/DrissionPage/_base/base.pyi b/DrissionPage/_base/base.pyi index fd18289..157909d 100644 --- a/DrissionPage/_base/base.pyi +++ b/DrissionPage/_base/base.pyi @@ -83,62 +83,39 @@ class DrissionElement(BaseElement): def parent(self, level_or_loc: Union[tuple, str, int] = 1, index: int = 1) -> Union[DrissionElement, None]: ... - def child(self, - filter_loc: Union[tuple, str, int] = '', - index: int = 1, - timeout: float = None, - ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]: ... + def child(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, + timeout: float = None, ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]: ... - def prev(self, - filter_loc: Union[tuple, str, int] = '', - index: int = 1, - timeout: float = 0, - ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]: ... + def prev(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, + timeout: float = None, ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]: ... - def next(self, - filter_loc: Union[tuple, str, int] = '', - index: int = 1, - timeout: float = 0, - ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]: ... + def next(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, + timeout: float = None, ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]: ... - def before(self, - filter_loc: Union[tuple, str, int] = '', - index: int = 1, - timeout: float = None, - ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]: ... + def before(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, + timeout: float = None, ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]: ... - def after(self, - filter_loc: Union[tuple, str, int] = '', - index: int = 1, - timeout: float = None, - ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]: ... + def after(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, + timeout: float = None, ele_only: bool = True) -> Union[DrissionElement, str, NoneElement]: ... - def children(self, filter_loc: Union[tuple, str] = '', - timeout: float = None, + def children(self, filter_loc: Union[tuple, str] = '', timeout: float = None, ele_only: bool = True) -> List[Union[DrissionElement, str]]: ... - def prevs(self, filter_loc: Union[tuple, str] = '', - timeout: float = 0, + def prevs(self, filter_loc: Union[tuple, str] = '', timeout: float = None, ele_only: bool = True) -> List[Union[DrissionElement, str]]: ... - def nexts(self, filter_loc: Union[tuple, str] = '', - timeout: float = 0, + def nexts(self, filter_loc: Union[tuple, str] = '', timeout: float = None, ele_only: bool = True) -> List[Union[DrissionElement, str]]: ... - def befores(self, filter_loc: Union[tuple, str] = '', - timeout: float = None, + def befores(self, filter_loc: Union[tuple, str] = '', timeout: float = None, ele_only: bool = True) -> List[Union[DrissionElement, str]]: ... - def afters(self, filter_loc: Union[tuple, str] = '', - timeout: float = None, + def afters(self, filter_loc: Union[tuple, str] = '', timeout: float = None, ele_only: bool = True) -> List[Union[DrissionElement, str]]: ... - def _get_brothers(self, index: int = None, - filter_loc: Union[tuple, str] = '', - direction: str = 'following', - brother: bool = True, - timeout: float = 0.5, - ele_only: bool = True) -> List[Union[DrissionElement, str]]: ... + def _get_brothers(self, index: int = None, filter_loc: Union[tuple, str] = '', + direction: str = 'following', brother: bool = True, + timeout: float = 0.5, ele_only: bool = True) -> List[Union[DrissionElement, str]]: ... # ----------------以下属性或方法由后代实现---------------- @property @@ -205,11 +182,7 @@ class BasePage(BaseParser): def get_cookies(self, as_dict: bool = False, all_info: bool = False) -> Union[list, dict]: ... @abstractmethod - def get(self, - url: str, - show_errmsg: bool = False, - retry: int = None, - interval: float = None): ... + def get(self, url: str, show_errmsg: bool = False, retry: int = None, interval: float = None): ... def _ele(self, loc_or_ele, timeout: float = None, single: bool = True, raise_err: bool = None, method: str = None): ... diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index f8f4b4f..44f3e8b 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -190,14 +190,10 @@ class ChromiumDriver(object): event = self.event_queue.get_nowait() function = self.event_handlers.get(event['method']) if function: - # if self._debug: - # print(f'开始执行 {function.__name__}') try: function(**event['params']) except: pass - # if self._debug: - # print(f'执行 {function.__name__}完毕') self.event_handlers.clear() self.method_results.clear() diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 25d9952..f680009 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -724,7 +724,7 @@ class ChromiumElement(DrissionElement): elif mode == 'css': txt1 = '' txt3 = '' - txt4 = '''path = '>' + el.tagName + ":nth-child(" + nth + ")" + path;''' + txt4 = '''path = '>' + el.tagName.toLowerCase() + ":nth-child(" + nth + ")" + path;''' txt5 = '''return path.substr(1);''' else: @@ -1150,8 +1150,8 @@ def find_by_xpath(ele, xpath, single, timeout, relative=True): raise SyntaxError(f'查询语句错误：\n{r}') end_time = perf_counter() + timeout - while (r['result']['subtype'] == 'null' - or r['result']['description'] == 'NodeList(0)') and perf_counter() < end_time: + while ((r['result']['subtype'] == 'null' or r['result']['description'] in ('NodeList(0)', 'Array(0)')) + and perf_counter() < end_time): r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, returnByValue=False, awaitPromise=True, userGesture=True) diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 5716eb6..55653c3 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -15,7 +15,7 @@ from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter from .._units.states import FrameStates from .._units.waiter import FrameWaiter -from ..errors import ContextLostError, ElementLostError, GetDocumentError, PageClosedError +from ..errors import ContextLostError, ElementLostError, GetDocumentError, PageClosedError, JavaScriptError class ChromiumFrame(ChromiumBase): @@ -54,10 +54,11 @@ class ChromiumFrame(ChromiumBase): obj_id = super().run_js('document;', as_expr=True)['objectId'] self.doc_ele = ChromiumElement(self, obj_id=obj_id) - end_time = perf_counter() + 5 - while perf_counter() < end_time and self.url == 'about:blank': - sleep(.1) self._rect = None + end_time = perf_counter() + 5 + while perf_counter() < end_time: + if self.url is None or self.url == 'about:blank': + sleep(.1) def __call__(self, loc_or_str, timeout=None): """在内部查找元素 @@ -292,7 +293,10 @@ class ChromiumFrame(ChromiumBase): @property def url(self): """返回frame当前访问的url""" - return self.doc_ele.run_js('return this.location.href;') + try: + return self.doc_ele.run_js('return this.location.href;') + except JavaScriptError: + return None @property def html(self): diff --git a/DrissionPage/_units/listener.py b/DrissionPage/_units/listener.py index 3323049..2e7e4d8 100644 --- a/DrissionPage/_units/listener.py +++ b/DrissionPage/_units/listener.py @@ -25,6 +25,7 @@ class Listener(object): self._address = page.address self._target_id = page._target_id self._driver = None + self._running_requests = 0 self._caught = None # 临存捕捉到的数据 self._request_ids = None # 暂存须要拦截的请求id @@ -86,6 +87,7 @@ class Listener(object): self._request_ids = {} self._extra_info_ids = {} self._caught = Queue(maxsize=0) + self._running_requests = 0 self._set_callback() @@ -180,6 +182,27 @@ class Listener(object): self._request_ids = {} self._extra_info_ids = {} self._caught.queue.clear() + self._running_requests = 0 + + def wait_silent(self, timeout=None): + """等待所有请求结束 + :param timeout: 超时，为None时无限等待 + :return: 返回是否等待成功 + """ + if not self.listening: + raise RuntimeError('监听未启动，用listen.start()启动。') + if timeout is None: + while self._running_requests > 0: + sleep(.1) + return True + + end_time = perf_counter() + timeout + while perf_counter() < end_time: + if self._running_requests <= 0: + return True + sleep(.1) + else: + return False def _to_target(self, target_id, address, page): """切换监听的页面对象 @@ -212,6 +235,7 @@ class Listener(object): def _requestWillBeSent(self, **kwargs): """接收到请求时的回调函数""" + self._running_requests += 1 p = None if not self._targets: if not self._method or kwargs['request']['method'] in self._method: @@ -236,6 +260,7 @@ class Listener(object): def _requestWillBeSentExtraInfo(self, **kwargs): """接收到请求额外信息时的回调函数""" + self._running_requests += 1 self._extra_info_ids.setdefault(kwargs['requestId'], {})['request'] = kwargs def _response_received(self, **kwargs): @@ -247,6 +272,7 @@ class Listener(object): def _responseReceivedExtraInfo(self, **kwargs): """接收到返回额外信息时的回调函数""" + self._running_requests -= 1 r = self._extra_info_ids.get(kwargs['requestId'], None) if r: obj = r.get('obj', None) @@ -261,6 +287,7 @@ class Listener(object): def _loading_finished(self, **kwargs): """请求完成时处理方法""" + self._running_requests -= 1 rid = kwargs['requestId'] packet = self._request_ids.get(rid) if packet: @@ -295,6 +322,7 @@ class Listener(object): def _loading_failed(self, **kwargs): """请求失败时的回调方法""" + self._running_requests -= 1 r_id = kwargs['requestId'] dp = self._request_ids.get(r_id, None) if dp: diff --git a/DrissionPage/_units/listener.pyi b/DrissionPage/_units/listener.pyi index 2c3a280..b3a2a93 100644 --- a/DrissionPage/_units/listener.pyi +++ b/DrissionPage/_units/listener.pyi @@ -26,6 +26,7 @@ class Listener(object): self._request_ids: dict = ... self._extra_info_ids: dict = ... self.listening: bool = ... + self._running_requests: int = ... @property def targets(self) -> Optional[set]: ... @@ -47,6 +48,8 @@ class Listener(object): def clear(self) -> None: ... + def wait_silent(self, timeout=None) -> bool: ... + def _to_target(self, target_id: str, address: str, page: ChromiumBase) -> None: ... def start(self, targets: Union[str, List[str], Tuple, bool, None] = None, is_regex: bool = False, @@ -72,7 +75,7 @@ class Listener(object): class FrameListener(Listener): - def __init__(self, page: ChromiumFrame, is_diff: bool): + def __init__(self, page: ChromiumFrame): self._page: ChromiumFrame = ... self._is_diff: bool = ... diff --git a/setup.py b/setup.py index acb5c6a..c2e223a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b16", + version="4.0.0b17", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From d7ec6179880b540fea05920d9b5953fd47d06d4e Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 5 Dec 2023 22:34:09 +0800 Subject: [PATCH 118/182] =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=8F=AA=E7=9B=91?= =?UTF-8?q?=E5=90=ACGET=E5=92=8CPOST=E8=AF=B7=E6=B1=82=EF=BC=9B=E4=BF=AE?= =?UTF-8?q?=E5=A4=8Dcookie=E8=AE=BE=E7=BD=AEexpires=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_commons/web.py | 15 ++++++++++++++- DrissionPage/_units/listener.py | 8 ++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/DrissionPage/_commons/web.py b/DrissionPage/_commons/web.py index 5c1d182..2a726b3 100644 --- a/DrissionPage/_commons/web.py +++ b/DrissionPage/_commons/web.py @@ -3,6 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ +from datetime import datetime from html import unescape from http.cookiejar import Cookie from re import sub @@ -250,7 +251,19 @@ def set_browser_cookies(page, cookies): cookie['expires'] = int(cookie['expiry']) cookie.pop('expiry') if 'expires' in cookie: - cookie['expires'] = int(cookie['expires']) + if cookie['expires'].isdigit(): + cookie['expires'] = int(cookie['expires']) + + elif cookie['expires'].replace('.', '').isdigit(): + cookie['expires'] = float(cookie['expires']) + + else: + try: + cookie['expires'] = datetime.strptime(cookie['expires'], + '%a, %d %b %Y %H:%M:%S GMT').timestamp() + except ValueError: + cookie['expires'] = datetime.strptime(cookie['expires'], + '%a, %d %b %y %H:%M:%S GMT').timestamp() if cookie['value'] is None: cookie['value'] = '' if cookie['name'].startswith('__Secure-'): diff --git a/DrissionPage/_units/listener.py b/DrissionPage/_units/listener.py index 2e7e4d8..a864db9 100644 --- a/DrissionPage/_units/listener.py +++ b/DrissionPage/_units/listener.py @@ -43,11 +43,11 @@ class Listener(object): """返回监听目标""" return self._targets - def set_targets(self, targets=True, is_regex=False, method=None): + def set_targets(self, targets=True, is_regex=False, method=('GET', 'POST')): """指定要等待的数据包 :param targets: 要匹配的数据包url特征，可用list等传入多个，为True时获取所有 :param is_regex: 设置的target是否正则表达式 - :param method: 设置监听的请求类型，可用list等指定多个，为None时监听全部 + :param method: 设置监听的请求类型，可指定多个，为None时监听全部 :return: None """ if targets is not None: @@ -68,11 +68,11 @@ class Listener(object): else: raise TypeError('method参数只能是str、list、tuple、set类型。') - def start(self, targets=None, is_regex=False, method=None): + def start(self, targets=None, is_regex=False, method=('GET', 'POST')): """拦截目标请求，每次拦截前清空结果 :param targets: 要匹配的数据包url特征，可用list等传入多个，为True时获取所有 :param is_regex: 设置的target是否正则表达式 - :param method: 设置监听的请求类型，可用list等指定多个，为None时监听全部 + :param method: 设置监听的请求类型，可指定多个，为None时监听全部 :return: None """ if targets or method: From 5ff303130d403ac1754cf505a14baf4e712e1eaf Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 6 Dec 2023 21:43:06 +0800 Subject: [PATCH 119/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8Diframe=E5=86=85?= =?UTF-8?q?=E5=85=83=E7=B4=A0link=E5=8F=82=E6=95=B0=E4=B8=8D=E6=AD=A3?= =?UTF-8?q?=E7=A1=AE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_commons/web.py | 10 +++++----- DrissionPage/_commons/web.pyi | 2 +- DrissionPage/_elements/chromium_element.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/DrissionPage/_commons/web.py b/DrissionPage/_commons/web.py index 2a726b3..a94bacf 100644 --- a/DrissionPage/_commons/web.py +++ b/DrissionPage/_commons/web.py @@ -128,10 +128,10 @@ def offset_scroll(ele, offset_x, offset_y): return cx, cy -def make_absolute_link(link, page=None): +def make_absolute_link(link, baseURI=None): """获取绝对url :param link: 超链接 - :param page: 页面对象 + :param baseURI: 页面或iframe的url :return: 绝对链接 """ if not link: @@ -142,11 +142,11 @@ def make_absolute_link(link, page=None): # 是相对路径，与页面url拼接并返回 if not parsed['netloc']: - return urljoin(page.url, link) if page else link + return urljoin(baseURI, link) if baseURI else link # 是绝对路径但缺少协议，从页面url获取协议并修复 - if not parsed['scheme'] and page: - parsed['scheme'] = urlparse(page.url).scheme + if not parsed['scheme'] and baseURI: + parsed['scheme'] = urlparse(baseURI).scheme parsed = tuple(v for v in parsed.values()) return urlunparse(parsed) diff --git a/DrissionPage/_commons/web.pyi b/DrissionPage/_commons/web.pyi index 7bf9a5f..128f48f 100644 --- a/DrissionPage/_commons/web.pyi +++ b/DrissionPage/_commons/web.pyi @@ -26,7 +26,7 @@ def location_in_viewport(page: ChromiumBase, loc_x: float, loc_y: float) -> bool def offset_scroll(ele: ChromiumElement, offset_x: float, offset_y: float) -> tuple: ... -def make_absolute_link(link: str, page: BasePage = None) -> str: ... +def make_absolute_link(link: str, baseURI: str = None) -> str: ... def is_js_func(func: str) -> bool: ... diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index f680009..4f9ff19 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -324,10 +324,10 @@ class ChromiumElement(DrissionElement): if not link or link.lower().startswith(('javascript:', 'mailto:')): return link else: - return make_absolute_link(link, self.page) + return make_absolute_link(link, self.prop('baseURI')) elif attr == 'src': - return make_absolute_link(attrs.get('src', None), self.page) + return make_absolute_link(attrs.get('src', None), self.prop('baseURI')) elif attr == 'text': return self.text From cee5af6d7ed5ed2fe185d15c51db2248bb36ba11 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 6 Dec 2023 23:30:34 +0800 Subject: [PATCH 120/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=9B=91=E5=90=ACsta?= =?UTF-8?q?rt()=E6=B2=A1=E6=B8=85=E7=A9=BA=E9=97=AE=E9=A2=98=EF=BC=9B?= =?UTF-8?q?=E5=8A=A8=E4=BD=9C=E9=93=BE=E5=A2=9E=E5=8A=A0input()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_commons/keys.py | 55 ++++++++++-- DrissionPage/_commons/keys.pyi | 97 +++++++++++++++++++++ DrissionPage/_elements/chromium_element.py | 52 +---------- DrissionPage/_elements/chromium_element.pyi | 6 -- DrissionPage/_units/actions.py | 12 ++- DrissionPage/_units/actions.pyi | 4 +- DrissionPage/_units/listener.py | 10 +-- 7 files changed, 165 insertions(+), 71 deletions(-) create mode 100644 DrissionPage/_commons/keys.pyi diff --git a/DrissionPage/_commons/keys.py b/DrissionPage/_commons/keys.py index e31590e..626470b 100644 --- a/DrissionPage/_commons/keys.py +++ b/DrissionPage/_commons/keys.py @@ -3,7 +3,6 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import List, Tuple, Dict class Keys: @@ -339,10 +338,10 @@ modifierBit = {'\ue00a': 1, '\ue008': 8} -def keys_to_typing(value) -> Tuple[int, str]: +def keys_to_typing(value): """把要输入的内容连成字符串，去掉其中 ctrl 等键。 返回的modifier表示是否有按下组合键""" - typing: List[str] = [] + typing = [] modifier = 0 for val in value: if val in ('\ue009', '\ue008', '\ue00a', '\ue03d'): @@ -359,7 +358,7 @@ def keys_to_typing(value) -> Tuple[int, str]: return modifier, ''.join(typing) -def keyDescriptionForString(_modifiers: int, keyString: str) -> Dict: # noqa: C901 +def keyDescriptionForString(_modifiers, keyString): # noqa: C901 shift = _modifiers & 8 description = {'key': '', 'keyCode': 0, @@ -367,7 +366,7 @@ def keyDescriptionForString(_modifiers: int, keyString: str) -> Dict: # noqa: C 'text': '', 'location': 0} - definition: Dict = keyDefinitions.get(keyString) # type: ignore + definition = keyDefinitions.get(keyString) # type: ignore if not definition: raise ValueError(f'未知按键：{keyString}') @@ -399,3 +398,49 @@ def keyDescriptionForString(_modifiers: int, keyString: str) -> Dict: # noqa: C description['text'] = '' return description + + +def send_key(page, modifier, key): + """发送一个字，在键盘中的字符触发按键，其它直接发送文本""" + if key not in keyDefinitions: + page.run_cdp('Input.insertText', text=key) + + else: + description = keyDescriptionForString(modifier, key) + text = description['text'] + data = {'type': 'keyDown' if text else 'rawKeyDown', + 'modifiers': modifier, + 'windowsVirtualKeyCode': description['keyCode'], + 'code': description['code'], + 'key': description['key'], + 'text': text, + 'autoRepeat': False, + 'unmodifiedText': text, + 'location': description['location'], + 'isKeypad': description['location'] == 3} + + page.run_cdp('Input.dispatchKeyEvent', **data) + data['type'] = 'keyUp' + page.run_cdp('Input.dispatchKeyEvent', **data) + + +def input_text_or_keys(page, text_or_keys): + """输入文本，也可输入组合键，组合键用tuple形式输入 + :param page: ChromiumBase对象 + :param text_or_keys: 文本值或按键组合 + :return: self + """ + if not isinstance(text_or_keys, (tuple, list)): + text_or_keys = (str(text_or_keys),) + modifier, text_or_keys = keys_to_typing(text_or_keys) + + if modifier != 0: # 包含修饰符 + for key in text_or_keys: + send_key(page, modifier, key) + return + + if text_or_keys.endswith(('\n', '\ue007')): + page.run_cdp('Input.insertText', text=text_or_keys[:-1]) + send_key(page, modifier, '\n') + else: + page.run_cdp('Input.insertText', text=text_or_keys) diff --git a/DrissionPage/_commons/keys.pyi b/DrissionPage/_commons/keys.pyi new file mode 100644 index 0000000..de2ca51 --- /dev/null +++ b/DrissionPage/_commons/keys.pyi @@ -0,0 +1,97 @@ +# -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" +from typing import Tuple, Dict, Union, Any + +from .._pages.chromium_base import ChromiumBase + + +class Keys: + """特殊按键""" + + NULL: str + CANCEL: str + HELP: str + BACKSPACE: str + BACK_SPACE: str + TAB: str + CLEAR: str + RETURN: str + ENTER: str + SHIFT: str + LEFT_SHIFT: str + CONTROL: str + CTRL: str + LEFT_CONTROL: str + ALT: str + LEFT_ALT: str + PAUSE: str + ESCAPE: str + SPACE: str + PAGE_UP: str + PAGE_DOWN: str + END: str + HOME: str + LEFT: str + ARROW_LEFT: str + UP: str + ARROW_UP: str + RIGHT: str + ARROW_RIGHT: str + DOWN: str + ARROW_DOWN: str + INSERT: str + DELETE: str + DEL: str + SEMICOLON: str + EQUALS: str + + NUMPAD0: str + NUMPAD1: str + NUMPAD2: str + NUMPAD3: str + NUMPAD4: str + NUMPAD5: str + NUMPAD6: str + NUMPAD7: str + NUMPAD8: str + NUMPAD9: str + MULTIPLY: str + ADD: str + SUBTRACT: str + DECIMAL: str + DIVIDE: str + + F1: str + F2: str + F3: str + F4: str + F5: str + F6: str + F7: str + F8: str + F9: str + F10: str + F11: str + F12: str + + META: str + COMMAND: str + + +keyDefinitions: dict = ... +modifierBit: dict = ... + + +def keys_to_typing(value: Union[str, int, list, tuple]) -> Tuple[int, str]: ... + + +def keyDescriptionForString(_modifiers: int, keyString: str) -> Dict: ... + + +def send_key(page: ChromiumBase, modifier: int, key: str) -> None: ... + + +def input_text_or_keys(page: ChromiumBase, text_or_keys: Any) -> None: ... diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 4f9ff19..a89ea63 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -11,7 +11,7 @@ from time import perf_counter, sleep from .none_element import NoneElement from .session_element import make_session_ele from .._base.base import DrissionElement, BaseElement -from .._commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions +from .._commons.keys import input_text_or_keys from .._commons.locator import get_loc from .._commons.settings import Settings from .._commons.tools import get_usable_path @@ -605,21 +605,7 @@ class ChromiumElement(DrissionElement): else: self._input_focus() - # ------------处理字符------------- - if not isinstance(vals, (tuple, list)): - vals = (str(vals),) - modifier, vals = keys_to_typing(vals) - - if modifier != 0: # 包含修饰符 - for key in vals: - send_key(self, modifier, key) - return - - if vals.endswith(('\n', '\ue007')): - self.page.run_cdp('Input.insertText', text=vals[:-1]) - send_key(self, modifier, '\n') - else: - self.page.run_cdp('Input.insertText', text=vals) + input_text_or_keys(self.page, vals) def clear(self, by_js=False): """清空元素文本 @@ -1388,40 +1374,6 @@ def convert_argument(arg): return {'unserializableValue': '-Infinity'} -def send_enter(ele): - """发送回车""" - data = {'type': 'keyDown', 'modifiers': 0, 'windowsVirtualKeyCode': 13, 'code': 'Enter', 'key': 'Enter', - 'text': '\r', 'autoRepeat': False, 'unmodifiedText': '\r', 'location': 0, 'isKeypad': False} - - ele.page.run_cdp('Input.dispatchKeyEvent', **data) - data['type'] = 'keyUp' - ele.page.run_cdp('Input.dispatchKeyEvent', **data) - - -def send_key(ele, modifier, key): - """发送一个字，在键盘中的字符触发按键，其它直接发送文本""" - if key not in keyDefinitions: - ele.page.run_cdp('Input.insertText', text=key) - - else: - description = keyDescriptionForString(modifier, key) - text = description['text'] - data = {'type': 'keyDown' if text else 'rawKeyDown', - 'modifiers': modifier, - 'windowsVirtualKeyCode': description['keyCode'], - 'code': description['code'], - 'key': description['key'], - 'text': text, - 'autoRepeat': False, - 'unmodifiedText': text, - 'location': description['location'], - 'isKeypad': description['location'] == 3} - - ele.page.run_cdp('Input.dispatchKeyEvent', **data) - data['type'] = 'keyUp' - ele.page.run_cdp('Input.dispatchKeyEvent', **data) - - class Pseudo(object): def __init__(self, ele): """ diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 7725ccd..4b04419 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -308,12 +308,6 @@ def parse_js_result(page: ChromiumBase, ele: ChromiumElement, result: dict): ... def convert_argument(arg: Any) -> dict: ... -def send_enter(ele: ChromiumElement) -> None: ... - - -def send_key(ele: ChromiumElement, modifier: int, key: str) -> None: ... - - class Pseudo(object): def __init__(self, ele: ChromiumElement): self._ele: ChromiumElement = ... diff --git a/DrissionPage/_units/actions.py b/DrissionPage/_units/actions.py index 42cee3c..308d102 100644 --- a/DrissionPage/_units/actions.py +++ b/DrissionPage/_units/actions.py @@ -5,7 +5,7 @@ """ from time import sleep, perf_counter -from .._commons.keys import modifierBit, keyDescriptionForString +from .._commons.keys import modifierBit, keyDescriptionForString, input_text_or_keys from .._commons.web import location_in_viewport @@ -267,7 +267,7 @@ class Actions: return self def type(self, text): - """输入文本 + """用模拟键盘按键方式输入文本，可输入字符串，只能输入键盘上有的字符 :param text: 要输入的文本，特殊字符和多个文本可用list或tuple传入 :return: self """ @@ -278,6 +278,14 @@ class Actions: self.key_up(character) return self + def input(self, text_or_keys): + """输入文本，也可输入组合键，组合键用tuple形式输入 + :param text_or_keys: 文本值或按键组合 + :return: self + """ + input_text_or_keys(self.page, text_or_keys) + return self + def wait(self, second): """等待若干秒""" sleep(second) diff --git a/DrissionPage/_units/actions.pyi b/DrissionPage/_units/actions.pyi index 2f5df4e..4c17d80 100644 --- a/DrissionPage/_units/actions.pyi +++ b/DrissionPage/_units/actions.pyi @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union, Tuple +from typing import Union, Tuple, Any from .._base.chromium_driver import ChromiumDriver from .._elements.chromium_element import ChromiumElement @@ -66,6 +66,8 @@ class Actions: def type(self, text: Union[str, list, tuple]) -> Actions: ... + def input(self, text_or_keys: Any) -> Actions: ... + def wait(self, second: float) -> Actions: ... def _get_key_data(self, key: str, action: str) -> dict: ... diff --git a/DrissionPage/_units/listener.py b/DrissionPage/_units/listener.py index a864db9..aa473ed 100644 --- a/DrissionPage/_units/listener.py +++ b/DrissionPage/_units/listener.py @@ -77,18 +77,14 @@ class Listener(object): """ if targets or method: self.set_targets(targets, is_regex, method) + self.clear() + if self.listening: return self._driver = ChromiumDriver(self._target_id, 'page', self._address) self._driver.run('Network.enable') - self.listening = True - self._request_ids = {} - self._extra_info_ids = {} - self._caught = Queue(maxsize=0) - self._running_requests = 0 - self._set_callback() def wait(self, count=1, timeout=None, fit_count=True): @@ -181,7 +177,7 @@ class Listener(object): """清空结果""" self._request_ids = {} self._extra_info_ids = {} - self._caught.queue.clear() + self._caught = Queue(maxsize=0) self._running_requests = 0 def wait_silent(self, timeout=None): From 294e5219c74d7c84bba5a77d09f3afbf36b84c23 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 8 Dec 2023 18:31:51 +0800 Subject: [PATCH 121/182] =?UTF-8?q?ChromiumOptions=E5=A2=9E=E5=8A=A0incogn?= =?UTF-8?q?ito()=E6=96=B9=E6=B3=95=EF=BC=9Bini=E6=96=87=E4=BB=B6=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E5=87=A0=E4=B8=AA=E5=90=AF=E5=8A=A8=E5=8F=82=E6=95=B0?= =?UTF-8?q?=EF=BC=9Bcookie=E8=AE=BE=E7=BD=AE=E6=97=B6value=E9=BB=98?= =?UTF-8?q?=E8=AE=A4=E4=B8=BAstr=EF=BC=9B=E4=BF=AE=E5=A4=8Ddriver=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/chromium_driver.py | 5 ++--- DrissionPage/_commons/web.py | 2 ++ DrissionPage/_configs/chromium_options.py | 8 ++++++++ DrissionPage/_configs/chromium_options.pyi | 2 ++ DrissionPage/_configs/configs.ini | 2 +- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/chromium_driver.py index 44f3e8b..422eeb4 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/chromium_driver.py @@ -9,8 +9,7 @@ from threading import Thread, Event from time import perf_counter from requests import get -from websocket import (WebSocketTimeoutException, WebSocketException, WebSocketConnectionClosedException, - create_connection) +from websocket import WebSocketTimeoutException, WebSocketConnectionClosedException, create_connection class ChromiumDriver(object): @@ -103,7 +102,7 @@ class ChromiumDriver(object): msg = loads(msg_json) except WebSocketTimeoutException: continue - except (WebSocketException, OSError, WebSocketConnectionClosedException): + except: self.stop() return diff --git a/DrissionPage/_commons/web.py b/DrissionPage/_commons/web.py index a94bacf..030c181 100644 --- a/DrissionPage/_commons/web.py +++ b/DrissionPage/_commons/web.py @@ -266,6 +266,8 @@ def set_browser_cookies(page, cookies): '%a, %d %b %y %H:%M:%S GMT').timestamp() if cookie['value'] is None: cookie['value'] = '' + elif not isinstance(cookie['value'], str): + cookie['value'] = str(cookie['value']) if cookie['name'].startswith('__Secure-'): cookie['secure'] = True diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index b8e2eec..fca7805 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -340,6 +340,14 @@ class ChromiumOptions(object): on_off = None if on_off else False return self.set_argument('--mute-audio', on_off) + def incognito(self, on_off=True): + """设置是否使用无痕模式启动 + :param on_off: 开或关 + :return: 当前对象 + """ + on_off = None if on_off else False + return self.set_argument('--incognito', on_off) + def ignore_certificate_errors(self, on_off=True): """设置是否忽略证书错误 :param on_off: 开或关 diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index ec843ca..8398182 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -113,6 +113,8 @@ class ChromiumOptions(object): def mute(self, on_off: bool = True) -> ChromiumOptions: ... + def incognito(self, on_off: bool = True) -> ChromiumOptions: ... + def set_user_agent(self, user_agent: str) -> ChromiumOptions: ... def set_proxy(self, proxy: str) -> ChromiumOptions: ... diff --git a/DrissionPage/_configs/configs.ini b/DrissionPage/_configs/configs.ini index 7a22f7a..4355442 100644 --- a/DrissionPage/_configs/configs.ini +++ b/DrissionPage/_configs/configs.ini @@ -4,7 +4,7 @@ download_path = [chromium_options] address = 127.0.0.1:9222 browser_path = chrome -arguments = ['--no-first-run', '--disable-infobars', '--disable-popup-blocking'] +arguments = ['--no-default-browser-check', '--disable-suggestions-ui', '--no-first-run', '--disable-infobars', '--disable-popup-blocking'] extensions = [] prefs = {'profile.default_content_settings.popups': 0, 'profile.default_content_setting_values': {'notifications': 2}} flags = {} From 30df1c8eb80ced5dac3a8da1d1df36d2c5b114cd Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 8 Dec 2023 19:53:11 +0800 Subject: [PATCH 122/182] =?UTF-8?q?=E5=A2=9E=E5=8A=A0WrongURLError?= =?UTF-8?q?=EF=BC=9Bget()=E4=BC=9A=E6=A3=80=E6=9F=A5url=E8=A7=84=E8=8C=83?= =?UTF-8?q?=EF=BC=9BSessionPage=E7=9A=84get()=E5=8F=AF=E6=8C=87=E5=90=91?= =?UTF-8?q?=E6=9C=AC=E5=9C=B0=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/base.py | 13 --------- DrissionPage/_base/base.pyi | 2 -- DrissionPage/_commons/tools.py | 5 +++- DrissionPage/_elements/session_element.py | 4 +-- DrissionPage/_pages/chromium_base.py | 21 ++++++++++++++- DrissionPage/_pages/chromium_base.pyi | 2 ++ DrissionPage/_pages/session_page.py | 32 ++++++++++++++++++++--- DrissionPage/_pages/session_page.pyi | 4 +++ DrissionPage/errors.py | 4 +++ 9 files changed, 64 insertions(+), 23 deletions(-) diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index 23ef08e..7568f90 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -5,7 +5,6 @@ """ from abc import abstractmethod from re import sub -from urllib.parse import quote from DownloadKit import DownloadKit @@ -421,18 +420,6 @@ class BasePage(BaseParser): self._DownloadKit = DownloadKit(driver=self, goal_path=self.download_path) return self._DownloadKit - def _before_connect(self, url, retry, interval): - """连接前的准备 - :param url: 要访问的url - :param retry: 重试次数 - :param interval: 重试间隔 - :return: 重试次数和间隔组成的tuple - """ - self._url = quote(url, safe='-_.~!*\'"();:@&=+$,/\\?#[]%') - retry = retry if retry is not None else self.retry_times - interval = interval if interval is not None else self.retry_interval - return retry, interval - # ----------------以下属性或方法由后代实现---------------- @property def url(self): diff --git a/DrissionPage/_base/base.pyi b/DrissionPage/_base/base.pyi index 157909d..13175e1 100644 --- a/DrissionPage/_base/base.pyi +++ b/DrissionPage/_base/base.pyi @@ -166,8 +166,6 @@ class BasePage(BaseParser): @property def download(self) -> DownloadKit: ... - def _before_connect(self, url: str, retry: int, interval: float) -> tuple: ... - # ----------------以下属性或方法由后代实现---------------- @property def url(self) -> str: ... diff --git a/DrissionPage/_commons/tools.py b/DrissionPage/_commons/tools.py index afcb270..0105b52 100644 --- a/DrissionPage/_commons/tools.py +++ b/DrissionPage/_commons/tools.py @@ -12,7 +12,8 @@ from time import perf_counter, sleep from psutil import process_iter, AccessDenied, NoSuchProcess, ZombieProcess from .._configs.options_manage import OptionsManager -from ..errors import ContextLostError, ElementLostError, CDPError, PageClosedError, NoRectError, AlertExistsError +from ..errors import (ContextLostError, ElementLostError, CDPError, PageClosedError, NoRectError, AlertExistsError, + WrongURLError) def get_usable_path(path, is_file=True, parents=True): @@ -273,6 +274,8 @@ def raise_error(r): raise AlertExistsError elif error in ('Node does not have a layout object', 'Could not compute box model.'): raise NoRectError + elif error == 'Cannot navigate to invalid URL': + raise WrongURLError(f'无效的url：{r["args"]["url"]}。也许要加上"http://"？') elif r['type'] == 'call_method_error': raise CDPError(f'\n错误：{r["error"]}\nmethod：{r["method"]}\nargs：{r["args"]}\n出现这个错误可能意味着程序有bug，' '请把错误信息和重现方法告知作者，谢谢。\n报告网站：https://gitee.com/g1879/DrissionPage/issues') diff --git a/DrissionPage/_elements/session_element.py b/DrissionPage/_elements/session_element.py index 5c0109e..bd19b87 100644 --- a/DrissionPage/_elements/session_element.py +++ b/DrissionPage/_elements/session_element.py @@ -195,10 +195,10 @@ class SessionElement(DrissionElement): return link else: # 其它情况直接返回绝对url - return make_absolute_link(link, self.page) + return make_absolute_link(link, self.page.url) elif attr == 'src': - return make_absolute_link(self.inner_ele.get('src'), self.page) + return make_absolute_link(self.inner_ele.get('src'), self.page.url) elif attr == 'text': return self.text diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 832c9de..1bf2f13 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -5,9 +5,10 @@ """ from json import loads, JSONDecodeError from os.path import sep -from re import findall +from re import findall, match from threading import Thread from time import perf_counter, sleep +from urllib.parse import quote from .._base.base import BasePage from .._commons.locator import get_loc, is_loc @@ -895,6 +896,24 @@ class ChromiumBase(BasePage): pass return False + def _before_connect(self, url, retry, interval): + """连接前的准备 + :param url: 要访问的url + :param retry: 重试次数 + :param interval: 重试间隔 + :return: 重试次数和间隔组成的tuple + """ + url = quote(url, safe='-_.~!*\'"();:@&=+$,/\\?#[]%') + if not url: + self._url = 'chrome://newtab/' + elif not match(r'.*?://', url): + self._url = f'http://{url}' + else: + self._url = url + retry = retry if retry is not None else self.retry_times + interval = interval if interval is not None else self.retry_interval + return retry, interval + def _d_connect(self, to_url, times=0, interval=1, show_errmsg=False, timeout=None): """尝试连接，重试若干次 :param to_url: 要访问的url diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 3ba30f9..ebc0a84 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -231,6 +231,8 @@ class ChromiumBase(BasePage): def _on_alert_open(self, **kwargs): ... + def _before_connect(self, url: str, retry: int, interval: float) -> tuple: ... + def _d_connect(self, to_url: str, times: int = 0, interval: float = 1, show_errmsg: bool = False, timeout: float = None) -> Union[bool, None]: ... diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index 7b5fe44..112db23 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -6,9 +6,9 @@ from pathlib import Path from re import search from time import sleep -from urllib.parse import urlparse +from urllib.parse import urlparse, quote -from requests import Session +from requests import Session, Response from requests.structures import CaseInsensitiveDict from tldextract import extract @@ -130,8 +130,8 @@ class SessionPage(BasePage): return self._set def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None, **kwargs): - """用get方式跳转到url - :param url: 目标url + """用get方式跳转到url，可输入文件路径 + :param url: 目标url，可指定本地文件路径 :param show_errmsg: 是否显示和抛出异常 :param retry: 重试次数 :param interval: 重试间隔（秒） @@ -139,6 +139,17 @@ class SessionPage(BasePage): :param kwargs: 连接参数 :return: url是否可用 """ + if not url.lower().startswith('http'): + if url.startswith('file:///'): + url = url[8:] + if Path(url).exists(): + with open(url, 'rb') as f: + r = Response() + r._content = f.read() + r.status_code = 200 + self._response = r + return + retry, interval = self._before_connect(url, retry, interval) return self._s_connect(url, 'get', None, show_errmsg, retry, interval, **kwargs) def ele(self, loc_or_ele, timeout=None): @@ -220,6 +231,7 @@ class SessionPage(BasePage): :param kwargs: 连接参数 :return: url是否可用 """ + retry, interval = self._before_connect(url, retry, interval) return self._s_connect(url, 'post', data, show_errmsg, retry, interval, **kwargs) def close(self): @@ -228,6 +240,18 @@ class SessionPage(BasePage): if self._response is not None: self._response.close() + def _before_connect(self, url, retry, interval): + """连接前的准备 + :param url: 要访问的url + :param retry: 重试次数 + :param interval: 重试间隔 + :return: 重试次数和间隔组成的tuple + """ + self._url = quote(url, safe='-_.~!*\'"();:@&=+$,/\\?#[]%') + retry = retry if retry is not None else self.retry_times + interval = interval if interval is not None else self.retry_interval + return retry, interval + def _s_connect(self, url, mode, data=None, show_errmsg=False, retry=None, interval=None, **kwargs): """执行get或post连接 :param url: 目标url diff --git a/DrissionPage/_pages/session_page.pyi b/DrissionPage/_pages/session_page.pyi index 7a214f1..a0e8775 100644 --- a/DrissionPage/_pages/session_page.pyi +++ b/DrissionPage/_pages/session_page.pyi @@ -136,6 +136,10 @@ class SessionPage(BasePage): verify: Any | None = ..., cert: Any | None = ...) -> bool: ... + def close(self) -> None: ... + + def _before_connect(self, url: str, retry: int, interval: float) -> tuple: ... + def _s_connect(self, url: str, mode: str, diff --git a/DrissionPage/errors.py b/DrissionPage/errors.py index 7423445..30894b0 100644 --- a/DrissionPage/errors.py +++ b/DrissionPage/errors.py @@ -75,3 +75,7 @@ class GetDocumentError(BaseError): class WaitTimeoutError(BaseError): _info = '等待失败。' + + +class WrongURLError(BaseError): + _info = '无效的url。' From 1a5ec884f185a0681bc392e2619fdfafdf6bb968 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 8 Dec 2023 23:31:25 +0800 Subject: [PATCH 123/182] =?UTF-8?q?=5Fcommon=E6=94=B9=E5=90=8D=E4=B8=BA=5F?= =?UTF-8?q?functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_base/base.py | 6 +++--- DrissionPage/_base/browser.py | 2 +- DrissionPage/_configs/chromium_options.py | 2 +- DrissionPage/_configs/session_options.py | 2 +- DrissionPage/_elements/chromium_element.py | 10 +++++----- DrissionPage/_elements/session_element.py | 4 ++-- DrissionPage/{_commons => _functions}/browser.py | 0 DrissionPage/{_commons => _functions}/browser.pyi | 0 DrissionPage/{_commons => _functions}/by.py | 0 DrissionPage/{_commons => _functions}/cli.py | 2 +- DrissionPage/{_commons => _functions}/keys.py | 0 DrissionPage/{_commons => _functions}/keys.pyi | 0 DrissionPage/{_commons => _functions}/locator.py | 0 DrissionPage/{_commons => _functions}/locator.pyi | 0 DrissionPage/{_commons => _functions}/settings.py | 0 DrissionPage/{_commons => _functions}/tools.py | 0 DrissionPage/{_commons => _functions}/tools.pyi | 0 DrissionPage/{_commons => _functions}/web.py | 0 DrissionPage/{_commons => _functions}/web.pyi | 0 DrissionPage/_pages/chromium_base.py | 8 ++++---- DrissionPage/_pages/chromium_page.py | 2 +- DrissionPage/_pages/chromium_tab.py | 2 +- DrissionPage/_pages/session_page.py | 4 +++- DrissionPage/_pages/session_page.pyi | 3 ++- DrissionPage/_pages/web_page.py | 2 +- DrissionPage/_units/actions.py | 4 ++-- DrissionPage/_units/clicker.py | 4 ++-- DrissionPage/_units/downloader.py | 2 +- DrissionPage/_units/screencast.py | 2 +- DrissionPage/_units/setter.py | 4 ++-- DrissionPage/_units/states.py | 2 +- DrissionPage/_units/waiter.py | 2 +- DrissionPage/common.py | 8 ++++---- setup.py | 2 +- 35 files changed, 42 insertions(+), 39 deletions(-) rename DrissionPage/{_commons => _functions}/browser.py (100%) rename DrissionPage/{_commons => _functions}/browser.pyi (100%) rename DrissionPage/{_commons => _functions}/by.py (100%) rename DrissionPage/{_commons => _functions}/cli.py (96%) rename DrissionPage/{_commons => _functions}/keys.py (100%) rename DrissionPage/{_commons => _functions}/keys.pyi (100%) rename DrissionPage/{_commons => _functions}/locator.py (100%) rename DrissionPage/{_commons => _functions}/locator.pyi (100%) rename DrissionPage/{_commons => _functions}/settings.py (100%) rename DrissionPage/{_commons => _functions}/tools.py (100%) rename DrissionPage/{_commons => _functions}/tools.pyi (100%) rename DrissionPage/{_commons => _functions}/web.py (100%) rename DrissionPage/{_commons => _functions}/web.pyi (100%) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index b1205d5..01c6967 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b17' +__version__ = '4.0.0b18' diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index 7568f90..22ced5a 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -8,9 +8,9 @@ from re import sub from DownloadKit import DownloadKit -from .._commons.settings import Settings -from .._commons.locator import get_loc -from .._commons.web import format_html +from .._functions.settings import Settings +from .._functions.locator import get_loc +from .._functions.web import format_html from .._elements.none_element import NoneElement from ..errors import ElementNotFoundError diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 5d2ed0c..88da578 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -6,7 +6,7 @@ from time import sleep, perf_counter from .chromium_driver import BrowserDriver, ChromiumDriver -from .._commons.tools import stop_process_on_port, raise_error +from .._functions.tools import stop_process_on_port, raise_error from .._units.downloader import DownloadManager __ERROR__ = 'error' diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index fca7805..3d4515d 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -10,7 +10,7 @@ from tempfile import gettempdir, TemporaryDirectory from threading import Lock from .options_manage import OptionsManager -from .._commons.tools import port_is_using, clean_folder +from .._functions.tools import port_is_using, clean_folder class ChromiumOptions(object): diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index 88b396f..3331d98 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -9,7 +9,7 @@ from requests import Session from requests.structures import CaseInsensitiveDict from .options_manage import OptionsManager -from .._commons.web import cookies_to_tuple, set_session_cookies +from .._functions.web import cookies_to_tuple, set_session_cookies class SessionOptions(object): diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index a89ea63..f09b089 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -11,11 +11,11 @@ from time import perf_counter, sleep from .none_element import NoneElement from .session_element import make_session_ele from .._base.base import DrissionElement, BaseElement -from .._commons.keys import input_text_or_keys -from .._commons.locator import get_loc -from .._commons.settings import Settings -from .._commons.tools import get_usable_path -from .._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, offset_scroll +from .._functions.keys import input_text_or_keys +from .._functions.locator import get_loc +from .._functions.settings import Settings +from .._functions.tools import get_usable_path +from .._functions.web import make_absolute_link, get_ele_txt, format_html, is_js_func, offset_scroll from .._units.clicker import Clicker from .._units.rect import ElementRect from .._units.scroller import ElementScroller diff --git a/DrissionPage/_elements/session_element.py b/DrissionPage/_elements/session_element.py index bd19b87..6bd6326 100644 --- a/DrissionPage/_elements/session_element.py +++ b/DrissionPage/_elements/session_element.py @@ -11,8 +11,8 @@ from lxml.html import HtmlElement, fromstring from .none_element import NoneElement from .._base.base import DrissionElement, BasePage, BaseElement -from .._commons.locator import get_loc -from .._commons.web import get_ele_txt, make_absolute_link +from .._functions.locator import get_loc +from .._functions.web import get_ele_txt, make_absolute_link class SessionElement(DrissionElement): diff --git a/DrissionPage/_commons/browser.py b/DrissionPage/_functions/browser.py similarity index 100% rename from DrissionPage/_commons/browser.py rename to DrissionPage/_functions/browser.py diff --git a/DrissionPage/_commons/browser.pyi b/DrissionPage/_functions/browser.pyi similarity index 100% rename from DrissionPage/_commons/browser.pyi rename to DrissionPage/_functions/browser.pyi diff --git a/DrissionPage/_commons/by.py b/DrissionPage/_functions/by.py similarity index 100% rename from DrissionPage/_commons/by.py rename to DrissionPage/_functions/by.py diff --git a/DrissionPage/_commons/cli.py b/DrissionPage/_functions/cli.py similarity index 96% rename from DrissionPage/_commons/cli.py rename to DrissionPage/_functions/cli.py index ec3bec8..a06b228 100644 --- a/DrissionPage/_commons/cli.py +++ b/DrissionPage/_functions/cli.py @@ -5,7 +5,7 @@ """ from click import command, option -from .._commons.tools import configs_to_here as ch +from .._functions.tools import configs_to_here as ch from .._configs.chromium_options import ChromiumOptions from .._pages.chromium_page import ChromiumPage diff --git a/DrissionPage/_commons/keys.py b/DrissionPage/_functions/keys.py similarity index 100% rename from DrissionPage/_commons/keys.py rename to DrissionPage/_functions/keys.py diff --git a/DrissionPage/_commons/keys.pyi b/DrissionPage/_functions/keys.pyi similarity index 100% rename from DrissionPage/_commons/keys.pyi rename to DrissionPage/_functions/keys.pyi diff --git a/DrissionPage/_commons/locator.py b/DrissionPage/_functions/locator.py similarity index 100% rename from DrissionPage/_commons/locator.py rename to DrissionPage/_functions/locator.py diff --git a/DrissionPage/_commons/locator.pyi b/DrissionPage/_functions/locator.pyi similarity index 100% rename from DrissionPage/_commons/locator.pyi rename to DrissionPage/_functions/locator.pyi diff --git a/DrissionPage/_commons/settings.py b/DrissionPage/_functions/settings.py similarity index 100% rename from DrissionPage/_commons/settings.py rename to DrissionPage/_functions/settings.py diff --git a/DrissionPage/_commons/tools.py b/DrissionPage/_functions/tools.py similarity index 100% rename from DrissionPage/_commons/tools.py rename to DrissionPage/_functions/tools.py diff --git a/DrissionPage/_commons/tools.pyi b/DrissionPage/_functions/tools.pyi similarity index 100% rename from DrissionPage/_commons/tools.pyi rename to DrissionPage/_functions/tools.pyi diff --git a/DrissionPage/_commons/web.py b/DrissionPage/_functions/web.py similarity index 100% rename from DrissionPage/_commons/web.py rename to DrissionPage/_functions/web.py diff --git a/DrissionPage/_commons/web.pyi b/DrissionPage/_functions/web.pyi similarity index 100% rename from DrissionPage/_commons/web.pyi rename to DrissionPage/_functions/web.pyi diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 1bf2f13..c51228c 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -11,10 +11,10 @@ from time import perf_counter, sleep from urllib.parse import quote from .._base.base import BasePage -from .._commons.locator import get_loc, is_loc -from .._commons.settings import Settings -from .._commons.tools import get_usable_path, raise_error -from .._commons.web import location_in_viewport +from .._functions.locator import get_loc, is_loc +from .._functions.settings import Settings +from .._functions.tools import get_usable_path, raise_error +from .._functions.web import location_in_viewport from .._elements.chromium_element import ChromiumElement, run_js, make_chromium_ele from .._elements.none_element import NoneElement from .._elements.session_element import make_session_ele diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 0fd998b..ee3282a 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -9,7 +9,7 @@ from time import sleep, perf_counter from requests import get from .._base.browser import Browser -from .._commons.browser import connect_browser +from .._functions.browser import connect_browser from .._configs.chromium_options import ChromiumOptions from .._pages.chromium_base import ChromiumBase, Timeout from .._pages.chromium_tab import ChromiumTab diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 126153f..2890dc0 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -6,7 +6,7 @@ from copy import copy from .._base.base import BasePage -from .._commons.web import set_session_cookies, set_browser_cookies +from .._functions.web import set_session_cookies, set_browser_cookies from .._configs.session_options import SessionOptions from .._pages.chromium_base import ChromiumBase from .._pages.session_page import SessionPage diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index 112db23..e94a899 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -13,9 +13,9 @@ from requests.structures import CaseInsensitiveDict from tldextract import extract from .._base.base import BasePage -from .._commons.web import cookie_to_dict from .._configs.session_options import SessionOptions from .._elements.session_element import SessionElement, make_session_ele +from .._functions.web import cookie_to_dict from .._units.setter import SessionPageSetter @@ -139,6 +139,8 @@ class SessionPage(BasePage): :param kwargs: 连接参数 :return: url是否可用 """ + if isinstance(url, Path): + url = str(url.absolute()) if not url.lower().startswith('http'): if url.startswith('file:///'): url = url[8:] diff --git a/DrissionPage/_pages/session_page.pyi b/DrissionPage/_pages/session_page.pyi index a0e8775..75c73fe 100644 --- a/DrissionPage/_pages/session_page.pyi +++ b/DrissionPage/_pages/session_page.pyi @@ -3,6 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ +from pathlib import Path from typing import Any, Union, Tuple, List, Optional from requests import Session, Response @@ -66,7 +67,7 @@ class SessionPage(BasePage): def download_path(self) -> str: ... def get(self, - url: str, + url: Union[Path, str], show_errmsg: bool | None = False, retry: int | None = None, interval: float | None = None, diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 2f75d8e..d7b9b59 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -7,8 +7,8 @@ from .chromium_page import ChromiumPage from .chromium_tab import WebPageTab from .session_page import SessionPage from .._base.base import BasePage -from .._commons.web import set_session_cookies, set_browser_cookies from .._configs.chromium_options import ChromiumOptions +from .._functions.web import set_session_cookies, set_browser_cookies from .._units.setter import WebPageSetter diff --git a/DrissionPage/_units/actions.py b/DrissionPage/_units/actions.py index 308d102..4d6a0ff 100644 --- a/DrissionPage/_units/actions.py +++ b/DrissionPage/_units/actions.py @@ -5,8 +5,8 @@ """ from time import sleep, perf_counter -from .._commons.keys import modifierBit, keyDescriptionForString, input_text_or_keys -from .._commons.web import location_in_viewport +from .._functions.keys import modifierBit, keyDescriptionForString, input_text_or_keys +from .._functions.web import location_in_viewport class Actions: diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index 18359ff..e297862 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -5,8 +5,8 @@ """ from time import perf_counter, sleep -from .._commons.settings import Settings -from .._commons.web import offset_scroll +from .._functions.settings import Settings +from .._functions.web import offset_scroll from ..errors import CanNotClickError, CDPError, NoRectError diff --git a/DrissionPage/_units/downloader.py b/DrissionPage/_units/downloader.py index a81df80..f3bb017 100644 --- a/DrissionPage/_units/downloader.py +++ b/DrissionPage/_units/downloader.py @@ -8,7 +8,7 @@ from pathlib import Path from shutil import move from time import sleep, perf_counter -from .._commons.tools import get_usable_path +from .._functions.tools import get_usable_path class DownloadManager(object): diff --git a/DrissionPage/_units/screencast.py b/DrissionPage/_units/screencast.py index 592fc89..4b927a8 100644 --- a/DrissionPage/_units/screencast.py +++ b/DrissionPage/_units/screencast.py @@ -9,7 +9,7 @@ from pathlib import Path from threading import Thread from time import sleep, time -from .._commons.tools import clean_folder +from .._functions.tools import clean_folder class Screencast(object): diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index d0bd70c..fae9f5f 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -7,8 +7,8 @@ from pathlib import Path from requests.structures import CaseInsensitiveDict -from .._commons.tools import show_or_hide_browser -from .._commons.web import set_browser_cookies, set_session_cookies +from .._functions.tools import show_or_hide_browser +from .._functions.web import set_browser_cookies, set_session_cookies class BasePageSetter(object): diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index c178d5e..dc4a020 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from .._commons.web import location_in_viewport +from .._functions.web import location_in_viewport from ..errors import CDPError, NoRectError, PageClosedError, ElementLostError diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 0d09ed3..b27328d 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -1,7 +1,7 @@ # -*- coding:utf-8 -*- from time import sleep, perf_counter -from .._commons.settings import Settings +from .._functions.settings import Settings from ..errors import WaitTimeoutError, NoRectError diff --git a/DrissionPage/common.py b/DrissionPage/common.py index 32fabb2..44ab030 100644 --- a/DrissionPage/common.py +++ b/DrissionPage/common.py @@ -3,11 +3,11 @@ @Author : g1879 @Contact : g1879@qq.com """ -from ._commons.by import By -from ._commons.keys import Keys -from ._commons.settings import Settings -from ._commons.tools import wait_until, configs_to_here from ._elements.session_element import make_session_ele +from ._functions.by import By +from ._functions.keys import Keys +from ._functions.settings import Settings +from ._functions.tools import wait_until, configs_to_here from ._units.actions import Actions __all__ = ['make_session_ele', 'Actions', 'Keys', 'By', 'Settings', 'wait_until', 'configs_to_here'] diff --git a/setup.py b/setup.py index c2e223a..1180e6d 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b17", + version="4.0.0b18", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From ae417542b7f419677112505fd59e0ed677a88f74 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 10 Dec 2023 01:07:37 +0800 Subject: [PATCH 124/182] =?UTF-8?q?4.0.0b18(+)=20SessionPage=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E8=AE=BE=E7=BD=AE=E7=BC=96=E7=A0=81=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=EF=BC=9B=20listen.wait()=E5=A2=9E=E5=8A=A0raise=5Ferr=E5=8F=82?= =?UTF-8?q?=E6=95=B0=EF=BC=9B=20=E4=BC=98=E5=8C=96set.local=5Fstorage()?= =?UTF-8?q?=E3=80=81set.session=5Fstorage()=E5=92=8Cclear=5Fcache()?= =?UTF-8?q?=EF=BC=9B=20=E5=90=84=E7=B1=BB=E5=A2=9E=E5=8A=A0=5F=5Frepr=5F?= =?UTF-8?q?=5F=EF=BC=9B=20=E6=8C=87=E5=AE=9A=E6=96=B0=E7=89=88DownloadKit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_configs/chromium_options.py | 3 +++ DrissionPage/_configs/session_options.py | 3 +++ DrissionPage/_pages/chromium_base.py | 14 +++++++--- DrissionPage/_pages/chromium_page.py | 3 +++ DrissionPage/_pages/chromium_tab.py | 8 +++++- DrissionPage/_pages/session_page.py | 16 ++++++++--- DrissionPage/_pages/session_page.pyi | 4 +++ DrissionPage/_pages/web_page.py | 3 +++ DrissionPage/_units/listener.py | 10 +++++-- DrissionPage/_units/listener.pyi | 4 +-- DrissionPage/_units/setter.py | 33 ++++++++++++++++++++--- DrissionPage/_units/setter.pyi | 2 ++ requirements.txt | 2 +- setup.py | 2 +- 14 files changed, 89 insertions(+), 18 deletions(-) diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index 3d4515d..8c71fa8 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -593,6 +593,9 @@ class ChromiumOptions(object): on_off = None if on_off else False return self.set_argument('--mute-audio', on_off) + def __repr__(self): + return f'' + class PortFinder(object): used_port = {} diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index 3331d98..14525b8 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -457,6 +457,9 @@ class SessionOptions(object): self._download_path = str(download_path) return self + def __repr__(self): + return f'' + def session_options_to_dict(options): """把session配置对象转换为字典 diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index c51228c..eb31600 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -820,12 +820,18 @@ class ChromiumBase(BasePage): :param cookies: 是否清除cookies :return: None """ - if session_storage: - self.run_js('sessionStorage.clear();', as_expr=True) - if local_storage: - self.run_js('localStorage.clear();', as_expr=True) + if session_storage or local_storage: + self.run_cdp_loaded('DOMStorage.enable') + i = self.run_cdp('Storage.getStorageKeyForFrame', frameId=self._frame_id)['storageKey'] + if session_storage: + self.run_cdp('DOMStorage.clear', storageId={'storageKey': i, 'isLocalStorage': False}) + if local_storage: + self.run_cdp('DOMStorage.clear', storageId={'storageKey': i, 'isLocalStorage': True}) + self.run_cdp_loaded('DOMStorage.disable') + if cache: self.run_cdp_loaded('Network.clearBrowserCache') + if cookies: self.run_cdp_loaded('Network.clearBrowserCookies') diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index ee3282a..61d9104 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -248,6 +248,9 @@ class ChromiumPage(ChromiumBase): """ self.browser.quit(timeout, force) + def __repr__(self): + return f'' + def get_rename(original, rename): if '.' in rename: diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 2890dc0..907c60b 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -58,6 +58,9 @@ class ChromiumTab(ChromiumBase): self._wait = TabWaiter(self) return self._wait + def __repr__(self): + return f'' + class WebPageTab(SessionPage, ChromiumTab, BasePage): def __init__(self, page, tab_id): @@ -352,4 +355,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): return super()._find_elements(loc_or_ele, single=single) elif self._mode == 'd': return super(SessionPage, self)._find_elements(loc_or_ele, timeout=timeout, single=single, - relative=relative) \ No newline at end of file + relative=relative) + + def __repr__(self): + return f'' diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index e94a899..60d49f5 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -32,6 +32,7 @@ class SessionPage(BasePage): self._response = None self._session = None self._set = None + self._encoding = None self._s_set_start_options(session_or_options) self._s_set_runtime_settings() self._create_session() @@ -122,6 +123,11 @@ class SessionPage(BasePage): """返回访问url得到的response对象""" return self._response + @property + def encoding(self): + """返回设置的编码""" + return self._encoding + @property def set(self): """返回用于等待的对象""" @@ -151,7 +157,6 @@ class SessionPage(BasePage): r.status_code = 200 self._response = r return - retry, interval = self._before_connect(url, retry, interval) return self._s_connect(url, 'get', None, show_errmsg, retry, interval, **kwargs) def ele(self, loc_or_ele, timeout=None): @@ -233,7 +238,6 @@ class SessionPage(BasePage): :param kwargs: 连接参数 :return: url是否可用 """ - retry, interval = self._before_connect(url, retry, interval) return self._s_connect(url, 'post', data, show_errmsg, retry, interval, **kwargs) def close(self): @@ -289,7 +293,7 @@ class SessionPage(BasePage): :param data: post方式要提交的数据 :param show_errmsg: 是否显示和抛出异常 :param kwargs: 其它参数 - :return: tuple，第一位为Response或None，第二位为出错信息或'Success' + :return: tuple，第一位为Response或None，第二位为出错信息或 'Success' """ kwargs = CaseInsensitiveDict(kwargs) if 'headers' not in kwargs: @@ -322,6 +326,9 @@ class SessionPage(BasePage): r = self.session.post(url, data=data, **kwargs) if r and r.content: + if self._encoding: + r.encoding = self._encoding + return r, 'Success' return set_charset(r), 'Success' except Exception as e: @@ -348,6 +355,9 @@ class SessionPage(BasePage): raise ConnectionError(f'状态码：{r.status_code}') return r, f'状态码：{r.status_code}' + def __repr__(self): + return f'' + def check_headers(kwargs, headers, arg): """检查kwargs或headers中是否有arg所示属性""" diff --git a/DrissionPage/_pages/session_page.pyi b/DrissionPage/_pages/session_page.pyi index 75c73fe..017bc13 100644 --- a/DrissionPage/_pages/session_page.pyi +++ b/DrissionPage/_pages/session_page.pyi @@ -30,6 +30,7 @@ class SessionPage(BasePage): self.retry_times: int = ... self.retry_interval: float = ... self._set: SessionPageSetter = ... + self._encoding: str = ... def _s_set_start_options(self, session_or_options: Union[Session, SessionOptions]) -> None: ... @@ -114,6 +115,9 @@ class SessionPage(BasePage): @property def response(self) -> Response: ... + @property + def encoding(self) -> str: ... + @property def set(self) -> SessionPageSetter: ... diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index d7b9b59..748ccf7 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -387,3 +387,6 @@ class WebPage(SessionPage, ChromiumPage, BasePage): super(SessionPage, self).quit(timeout, force) self._driver = None self._has_driver = None + + def __repr__(self): + return f'' diff --git a/DrissionPage/_units/listener.py b/DrissionPage/_units/listener.py index aa473ed..72346a6 100644 --- a/DrissionPage/_units/listener.py +++ b/DrissionPage/_units/listener.py @@ -12,6 +12,8 @@ from time import perf_counter, sleep from requests.structures import CaseInsensitiveDict from .._base.chromium_driver import ChromiumDriver +from .._functions.settings import Settings +from ..errors import WaitTimeoutError class Listener(object): @@ -87,11 +89,12 @@ class Listener(object): self._set_callback() - def wait(self, count=1, timeout=None, fit_count=True): + def wait(self, count=1, timeout=None, fit_count=True, raise_err=None): """等待符合要求的数据包到达指定数量 :param count: 需要捕捉的数据包数量 :param timeout: 超时时间，为None无限等待 :param fit_count: 是否必须满足总数要求，发生超时，为True返回False，为False返回已捕捉到的数据包 + :param raise_err: 超时时是否抛出错误，为None时根据Settings设置 :return: count为1时返回数据包对象，大于1时返回列表，超时且fit_count为True时返回False """ if not self.listening: @@ -113,7 +116,10 @@ class Listener(object): if fail: if fit_count or not self._caught.qsize(): - return False + if raise_err is True or Settings.raise_when_wait_failed is True: + raise WaitTimeoutError('等待数据包失败。') + else: + return False else: return [self._caught.get_nowait() for _ in range(self._caught.qsize())] diff --git a/DrissionPage/_units/listener.pyi b/DrissionPage/_units/listener.pyi index b3a2a93..12eb344 100644 --- a/DrissionPage/_units/listener.pyi +++ b/DrissionPage/_units/listener.pyi @@ -40,8 +40,8 @@ class Listener(object): def go_on(self) -> None: ... - def wait(self, count: int = 1, timeout: float = None, - fit_count: bool = True) -> Union[List[DataPacket], DataPacket, None]: ... + def wait(self, count: int = 1, timeout: float = None, fit_count: bool = True, + raise_err: bool = None) -> Union[List[DataPacket], DataPacket, None]: ... @property def results(self) -> Union[DataPacket, Dict[str, List[DataPacket]], False]: ... diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index fae9f5f..705bc1d 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -82,8 +82,15 @@ class ChromiumBaseSetter(BasePageSetter): :param value: 项的值，设置为False时，删除该项 :return: None """ - js = f'sessionStorage.removeItem("{item}");' if item is False else f'sessionStorage.setItem("{item}","{value}");' - return self._page.run_js_loaded(js, as_expr=True) + self._page.run_cdp_loaded('DOMStorage.enable') + i = self._page.run_cdp('Storage.getStorageKeyForFrame', frameId=self._page._frame_id)['storageKey'] + if value is False: + self._page.run_cdp('DOMStorage.removeDOMStorageItem', + storageId={'storageKey': i, 'isLocalStorage': False}, key=item) + else: + self._page.run_cdp('DOMStorage.setDOMStorageItem', storageId={'storageKey': i, 'isLocalStorage': False}, + key=item, value=value) + self._page.run_cdp_loaded('DOMStorage.disable') def local_storage(self, item, value): """设置或删除某项localStorage信息 @@ -91,8 +98,15 @@ class ChromiumBaseSetter(BasePageSetter): :param value: 项的值，设置为False时，删除该项 :return: None """ - js = f'localStorage.removeItem("{item}");' if item is False else f'localStorage.setItem("{item}","{value}");' - return self._page.run_js_loaded(js, as_expr=True) + self._page.run_cdp_loaded('DOMStorage.enable') + i = self._page.run_cdp('Storage.getStorageKeyForFrame', frameId=self._page._frame_id)['storageKey'] + if value is False: + self._page.run_cdp('DOMStorage.removeDOMStorageItem', + storageId={'storageKey': i, 'isLocalStorage': True}, key=item) + else: + self._page.run_cdp('DOMStorage.setDOMStorageItem', storageId={'storageKey': i, 'isLocalStorage': True}, + key=item, value=value) + self._page.run_cdp_loaded('DOMStorage.disable') def cookie(self, cookie): """设置单个cookie @@ -237,6 +251,17 @@ class SessionPageSetter(BasePageSetter): """ self._page.timeout = second + def encoding(self, encoding, set_all=True): + """设置编码 + :param encoding: 编码名称 + :param set_all: 是否设置对象参数，为False则只设置当前response + :return: None + """ + if set_all: + self._page._encoding = encoding + if self._page.response: + self._page.response.encoding = encoding + def cookie(self, cookie): """为Session对象设置单个cookie :param cookie: cookie信息 diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index a254288..a82eaeb 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -100,6 +100,8 @@ class SessionPageSetter(BasePageSetter): def timeout(self, second: float) -> None: ... + def encoding(self, encoding: str, set_all: bool = True) -> None: ... + def cookie(self, cookie: Union[Cookie, str, dict]) -> None: ... def cookies(self, cookies: Union[RequestsCookieJar, list, tuple, str, dict]) -> None: ... diff --git a/requirements.txt b/requirements.txt index ba1160f..dd6c6eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ requests lxml cssselect -DownloadKit>=1.0.0 +DownloadKit>=2.0.0b0 FlowViewer>=0.3.0 websocket-client click diff --git a/setup.py b/setup.py index 1180e6d..5bf2bc8 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup( 'lxml', 'requests', 'cssselect', - 'DownloadKit>=1.0.0', + 'DownloadKit>=2.0.0b0', 'FlowViewer>=0.3.0', 'websocket-client', 'click', From 173a323a428f1b887caed64f051eac00d13aafc7 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 10 Dec 2023 10:53:50 +0800 Subject: [PATCH 125/182] =?UTF-8?q?4.0.0b19=20=E4=B8=8B=E6=8B=89=E9=A1=B9?= =?UTF-8?q?=E5=8F=AF=E7=82=B9=E5=87=BB=E9=80=89=E6=8B=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_elements/chromium_element.py | 8 +++- DrissionPage/_units/clicker.py | 7 ++++ DrissionPage/_units/listener.py | 1 + DrissionPage/_units/selector.py | 49 +++++++++++++++------- DrissionPage/_units/selector.pyi | 10 ++++- requirements.txt | 1 - setup.py | 3 +- 8 files changed, 59 insertions(+), 22 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 01c6967..5861945 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b18' +__version__ = '4.0.0b19' diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index f09b089..ebf0796 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -688,14 +688,18 @@ class ChromiumElement(DrissionElement): if obj_id: return self.page.run_cdp('DOM.requestNode', objectId=obj_id)['nodeId'] else: - return self.page.run_cdp('DOM.describeNode', backendNodeId=backend_id)['node']['nodeId'] + n = self.page.run_cdp('DOM.describeNode', backendNodeId=backend_id)['node'] + self._tag = n['localName'] + return n['nodeId'] def _get_backend_id(self, node_id): """根据传入node id获取backend id :param node_id: :return: backend id """ - return self.page.run_cdp('DOM.describeNode', nodeId=node_id)['node']['backendNodeId'] + n = self.page.run_cdp('DOM.describeNode', nodeId=node_id)['node'] + self._tag = n['localName'] + return n['backendNodeId'] def _get_ele_path(self, mode): """返获取绝对的css路径或xpath路径""" diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index e297862..943e49c 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -34,6 +34,13 @@ class Clicker(object): :param wait_stop: 是否等待元素运动结束再执行点击 :return: 是否点击成功 """ + if self._ele.tag == 'option': + if self._ele.states.is_selected: + self._ele.parent('t:select').select.cancel_by_option(self._ele) + else: + self._ele.parent('t:select').select.by_option(self._ele) + return + if not by_js: # 模拟点击 can_click = False timeout = self._ele.page.timeout if timeout is None else timeout diff --git a/DrissionPage/_units/listener.py b/DrissionPage/_units/listener.py index 72346a6..5dd1638 100644 --- a/DrissionPage/_units/listener.py +++ b/DrissionPage/_units/listener.py @@ -88,6 +88,7 @@ class Listener(object): self._driver.run('Network.enable') self._set_callback() + self.listening = True def wait(self, count=1, timeout=None, fit_count=True, raise_err=None): """等待符合要求的数据包到达指定数量 diff --git a/DrissionPage/_units/selector.py b/DrissionPage/_units/selector.py index 9494d16..b3d9d21 100644 --- a/DrissionPage/_units/selector.py +++ b/DrissionPage/_units/selector.py @@ -109,6 +109,13 @@ class SelectElement(object): """ return self._by_loc(loc, timeout) + def by_option(self, option): + """选中单个或多个选项元素 + :param option: 定位符 + :return: None + """ + self._select_options(option, 'true') + def cancel_by_text(self, text, timeout=None): """此方法用于根据text值取消选择项。当元素是多选列表时，可以接收list或tuple :param text: 文本，传入list或tuple可取消多项 @@ -141,6 +148,13 @@ class SelectElement(object): """ return self._by_loc(loc, timeout, True) + def cancel_by_option(self, option): + """选中单个或多个选项元素 + :param option: 定位符 + :return: None + """ + self._select_options(option, 'false') + def _by_loc(self, loc, timeout=None, cancel=False): """用定位符取消选择指定的项 :param loc: 定位符 @@ -154,13 +168,9 @@ class SelectElement(object): mode = 'false' if cancel else 'true' if self.is_multi: - for ele in eles: - ele.run_js(f'this.selected={mode};') - self._dispatch_change() - return True - - eles[0].run_js(f'this.selected={mode};') - self._dispatch_change() + self._select_options(eles, mode) + else: + self._select_options(eles[0], mode) return True def _select(self, condition, para_type='text', cancel=False, timeout=None): @@ -205,10 +215,7 @@ class SelectElement(object): break if ok: - for i in eles: - i.run_js(f'this.selected={mode};') - - self._dispatch_change() + self._select_options(eles, mode) return True return False @@ -231,14 +238,26 @@ class SelectElement(object): if ok: eles = self.options - for i in condition: - eles[i - 1].run_js(f'this.selected={mode};') - - self._dispatch_change() + eles = [eles[i - 1] for i in condition] + self._select_options(eles, mode) return True return False + def _select_options(self, option, mode): + """选中或取消某个选项 + :param option: options元素对象 + :param mode: 选中还是取消 + :return: None + """ + if isinstance(option, (list, tuple, set)): + for o in option: + o.run_js(f'this.selected={mode};') + self._dispatch_change() + else: + option.run_js(f'this.selected={mode};') + self._dispatch_change() + def _dispatch_change(self): """触发修改动作""" self._ele.run_js('this.dispatchEvent(new Event("change", {bubbles: true}));') diff --git a/DrissionPage/_units/selector.pyi b/DrissionPage/_units/selector.pyi index 55d07b6..f0d2fcc 100644 --- a/DrissionPage/_units/selector.pyi +++ b/DrissionPage/_units/selector.pyi @@ -38,6 +38,8 @@ class SelectElement(object): def by_loc(self, loc: Union[str, Tuple[str, str]], timeout: float = None) -> bool: ... + def by_option(self, option: Union[ChromiumElement, List[ChromiumElement], Tuple[ChromiumElement]]) -> None: ... + def cancel_by_text(self, text: Union[str, list, tuple], timeout: float = None) -> bool: ... def cancel_by_value(self, value: Union[str, list, tuple], timeout: float = None) -> bool: ... @@ -46,6 +48,9 @@ class SelectElement(object): def cancel_by_loc(self, loc: Union[str, Tuple[str, str]], timeout: float = None) -> bool: ... + def cancel_by_option(self, + option: Union[ChromiumElement, List[ChromiumElement], Tuple[ChromiumElement]]) -> None: ... + def invert(self) -> None: ... def _by_loc(self, loc: Union[str, Tuple[str, str]], timeout: float = None, cancel: bool = False) -> bool: ... @@ -60,4 +65,7 @@ class SelectElement(object): def _index(self, condition: set, mode: str, timeout: float) -> bool: ... - def _dispatch_change(self) -> None: ... \ No newline at end of file + def _select_options(self, option: Union[ChromiumElement, List[ChromiumElement], Tuple[ChromiumElement]], + mode: str) -> None: ... + + def _dispatch_change(self) -> None: ... diff --git a/requirements.txt b/requirements.txt index dd6c6eb..ea3dad4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,6 @@ requests lxml cssselect DownloadKit>=2.0.0b0 -FlowViewer>=0.3.0 websocket-client click tldextract diff --git a/setup.py b/setup.py index 5bf2bc8..1ca634c 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b18", + version="4.0.0b19", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", @@ -23,7 +23,6 @@ setup( 'requests', 'cssselect', 'DownloadKit>=2.0.0b0', - 'FlowViewer>=0.3.0', 'websocket-client', 'click', 'tldextract', From 66e52f6fbec02c9b6fa3a15b9cfd65e465af66ff Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 11 Dec 2023 01:09:04 +0800 Subject: [PATCH 126/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=88=87=E6=8D=A2?= =?UTF-8?q?=E6=A8=A1=E5=BC=8Fheaders=E8=AE=BE=E7=BD=AE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_pages/chromium_tab.py | 4 ++-- DrissionPage/_pages/session_page.py | 6 ++++-- DrissionPage/_pages/web_page.py | 2 +- DrissionPage/_units/setter.py | 6 +++--- DrissionPage/_units/setter.pyi | 4 ++-- setup.py | 2 +- 7 files changed, 14 insertions(+), 12 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 5861945..1c99f50 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b19' +__version__ = '4.0.0b20' diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 907c60b..983ee4a 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -304,7 +304,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): self.get(url) def cookies_to_session(self, copy_user_agent=True): - """把driver对象的cookies复制到session对象 + """把浏览器的cookies复制到session对象 :param copy_user_agent: 是否复制ua信息 :return: None """ @@ -313,7 +313,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): if copy_user_agent: user_agent = self.run_cdp('Runtime.evaluate', expression='navigator.userAgent;')['result']['value'] - self.session.headers.update({"User-Agent": user_agent}) + self._headers.update({"User-Agent": user_agent}) set_session_cookies(self.session, super(SessionPage, self).get_cookies()) diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index 60d49f5..038bf38 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -49,6 +49,8 @@ class SessionPage(BasePage): elif isinstance(session_or_options, Session): self._session_options = SessionOptions() + self._headers = session_or_options.headers + session_or_options.headers = None self._session = session_or_options def _s_set_runtime_settings(self): @@ -305,12 +307,12 @@ class SessionPage(BasePage): parsed_url = urlparse(url) hostname = parsed_url.hostname scheme = parsed_url.scheme - if not check_headers(kwargs, self.session.headers, 'Referer'): + if not check_headers(kwargs, self._headers, 'Referer'): kwargs['headers']['Referer'] = self.url if self.url else f'{scheme}://{hostname}' if 'Host' not in kwargs['headers']: kwargs['headers']['Host'] = hostname - if not check_headers(kwargs, self.session.headers, 'timeout'): + if not check_headers(kwargs, self._headers, 'timeout'): kwargs['timeout'] = self.timeout kwargs['headers'] = {**self._headers, **kwargs['headers']} diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 748ccf7..3b08d1a 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -274,7 +274,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): if copy_user_agent: user_agent = self.run_cdp('Runtime.evaluate', expression='navigator.userAgent;')['result']['value'] - self.session.headers.update({"User-Agent": user_agent}) + self._headers.update({"User-Agent": user_agent}) set_session_cookies(self.session, super(SessionPage, self).get_cookies()) diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index 705bc1d..cfca821 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -253,12 +253,12 @@ class SessionPageSetter(BasePageSetter): def encoding(self, encoding, set_all=True): """设置编码 - :param encoding: 编码名称 - :param set_all: 是否设置对象参数，为False则只设置当前response + :param encoding: 编码名称，如果要取消之前的设置，传入None + :param set_all: 是否设置对象参数，为False则只设置当前Response :return: None """ if set_all: - self._page._encoding = encoding + self._page._encoding = encoding if encoding else None if self._page.response: self._page.response.encoding = encoding diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index a82eaeb..45faf59 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -5,7 +5,7 @@ """ from http.cookiejar import Cookie from pathlib import Path -from typing import Union, Tuple, Literal, Any +from typing import Union, Tuple, Literal, Any, Optional from requests.adapters import HTTPAdapter from requests.auth import HTTPBasicAuth @@ -100,7 +100,7 @@ class SessionPageSetter(BasePageSetter): def timeout(self, second: float) -> None: ... - def encoding(self, encoding: str, set_all: bool = True) -> None: ... + def encoding(self, encoding: Optional[str, None], set_all: bool = True) -> None: ... def cookie(self, cookie: Union[Cookie, str, dict]) -> None: ... diff --git a/setup.py b/setup.py index 1ca634c..ed2a7d6 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b19", + version="4.0.0b20", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From cea156fc862b7fdb4eeff4cd2bf093014e5c711d Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 11 Dec 2023 18:15:24 +0800 Subject: [PATCH 127/182] =?UTF-8?q?4.0.0b20(+)=20=E5=A2=9E=E5=8A=A0Storage?= =?UTF-8?q?Error=EF=BC=9B=20=E5=8A=A8=E4=BD=9C=E9=93=BEkey=5Fdown()?= =?UTF-8?q?=E5=92=8Ckey=5Fup()=E6=94=AF=E6=8C=81=E6=8E=A5=E6=94=B6?= =?UTF-8?q?=E6=8C=89=E9=94=AE=E5=90=8D=E7=A7=B0=E6=96=87=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.pyi | 8 ++++-- DrissionPage/_functions/tools.py | 4 ++- DrissionPage/_pages/chromium_base.pyi | 14 ++++++---- DrissionPage/_units/actions.py | 8 ++++-- DrissionPage/_units/actions.pyi | 31 +++++++++++++++++++-- DrissionPage/errors.py | 4 +++ requirements.txt | 2 +- setup.py | 2 +- 8 files changed, 55 insertions(+), 18 deletions(-) diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 4b04419..95efc22 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from pathlib import Path -from typing import Union, Tuple, List, Any +from typing import Union, Tuple, List, Any, Literal from .none_element import NoneElement from .._base.base import DrissionElement, BaseElement @@ -21,6 +21,8 @@ from .._units.setter import ChromiumElementSetter from .._units.states import ShadowRootStates, ElementStates from .._units.waiter import ElementWaiter +PIC_TYPE = Literal['jpg', 'jpeg', 'png', 'webp', True] + class ChromiumElement(DrissionElement): @@ -175,8 +177,8 @@ class ChromiumElement(DrissionElement): def save(self, path: [str, bool] = None, name: str = None, timeout: float = None) -> str: ... - def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, - as_base64: [bool, str] = None, scroll_to_center: bool = True) -> Union[str, bytes]: ... + def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: PIC_TYPE = None, + as_base64: PIC_TYPE = None, scroll_to_center: bool = True) -> Union[str, bytes]: ... def input(self, vals: Any, clear: bool = True, by_js: bool = False) -> None: ... diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index 0105b52..a7cc6fb 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -13,7 +13,7 @@ from psutil import process_iter, AccessDenied, NoSuchProcess, ZombieProcess from .._configs.options_manage import OptionsManager from ..errors import (ContextLostError, ElementLostError, CDPError, PageClosedError, NoRectError, AlertExistsError, - WrongURLError) + WrongURLError, StorageError) def get_usable_path(path, is_file=True, parents=True): @@ -276,6 +276,8 @@ def raise_error(r): raise NoRectError elif error == 'Cannot navigate to invalid URL': raise WrongURLError(f'无效的url：{r["args"]["url"]}。也许要加上"http://"？') + elif error == 'Frame corresponds to an opaque origin and its storage key cannot be serialized': + raise StorageError elif r['type'] == 'call_method_error': raise CDPError(f'\n错误：{r["error"]}\nmethod：{r["method"]}\nargs：{r["args"]}\n出现这个错误可能意味着程序有bug，' '请把错误信息和重现方法告知作者，谢谢。\n报告网站：https://gitee.com/g1879/DrissionPage/issues') diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index ebc0a84..24c9154 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from pathlib import Path -from typing import Union, Tuple, List, Any, Optional +from typing import Union, Tuple, List, Any, Optional, Literal from .._base.base import BasePage from .._base.browser import Browser @@ -23,6 +23,8 @@ from .._units.setter import ChromiumBaseSetter from .._units.states import PageStates from .._units.waiter import BaseWaiter +PIC_TYPE = Literal['jpg', 'jpeg', 'png', 'webp', True] + class ChromiumBase(BasePage): def __init__(self, @@ -214,12 +216,12 @@ class ChromiumBase(BasePage): def get_local_storage(self, item: str = None) -> Union[str, dict, None]: ... - def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, - as_base64: [bool, str] = None, full_page: bool = False, - left_top: Tuple[int, int] = None, right_bottom: Tuple[int, int] = None) -> Union[str, bytes]: ... + def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: PIC_TYPE = None, + as_base64: PIC_TYPE = None, full_page: bool = False, left_top: Tuple[int, int] = None, + right_bottom: Tuple[int, int] = None) -> Union[str, bytes]: ... - def _get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, - as_base64: [bool, str] = None, full_page: bool = False, left_top: Tuple[float, float] = None, + def _get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: PIC_TYPE = None, + as_base64: PIC_TYPE = None, full_page: bool = False, left_top: Tuple[float, float] = None, right_bottom: Tuple[float, float] = None, ele: ChromiumElement = None) -> Union[str, bytes]: ... def clear_cache(self, session_storage: bool = True, local_storage: bool = True, cache: bool = True, diff --git a/DrissionPage/_units/actions.py b/DrissionPage/_units/actions.py index 4d6a0ff..33dc4c9 100644 --- a/DrissionPage/_units/actions.py +++ b/DrissionPage/_units/actions.py @@ -5,7 +5,7 @@ """ from time import sleep, perf_counter -from .._functions.keys import modifierBit, keyDescriptionForString, input_text_or_keys +from .._functions.keys import modifierBit, keyDescriptionForString, input_text_or_keys, Keys from .._functions.web import location_in_viewport @@ -241,10 +241,11 @@ class Actions: return self.move(pixel, 0) def key_down(self, key): - """按下键盘上的按键 - :param key: 按键，特殊字符见Keys + """按下键盘上的按键， + :param key: 使用Keys获取的按键，或'DEL'形式按键名称 :return: self """ + key = getattr(Keys, key.upper(), key) if key in ('\ue009', '\ue008', '\ue00a', '\ue03d'): # 如果上修饰符，添加到变量 self.modifier |= modifierBit.get(key, 0) return self @@ -258,6 +259,7 @@ class Actions: :param key: 按键，特殊字符见Keys :return: self """ + key = getattr(Keys, key.upper(), key) if key in ('\ue009', '\ue008', '\ue00a', '\ue03d'): # 如果上修饰符，添加到变量 self.modifier ^= modifierBit.get(key, 0) return self diff --git a/DrissionPage/_units/actions.pyi b/DrissionPage/_units/actions.pyi index 4c17d80..9f8f717 100644 --- a/DrissionPage/_units/actions.pyi +++ b/DrissionPage/_units/actions.pyi @@ -3,12 +3,37 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union, Tuple, Any +from typing import Union, Tuple, Any, Literal from .._base.chromium_driver import ChromiumDriver from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase +KEYS = Literal['NULL', 'CANCEL', 'HELP', 'BACKSPACE', 'BACK_SPACE', +'TAB', 'CLEAR', 'RETURN', 'ENTER', 'SHIFT', 'LEFT_SHIFT', 'CONTROL', +'CTRL', 'LEFT_CONTROL', 'ALT', 'LEFT_ALT', 'PAUSE', 'ESCAPE', 'SPACE', +'PAGE_UP', 'PAGE_DOWN', 'END', 'HOME', 'LEFT', 'ARROW_LEFT', 'UP', +'ARROW_UP', 'RIGHT', 'ARROW_RIGHT', 'DOWN', 'ARROW_DOWN', 'INSERT', +'DELETE', 'DEL', 'SEMICOLON', 'EQUALS', 'NUMPAD0', 'NUMPAD1', 'NUMPAD2', +'NUMPAD3', 'NUMPAD4', 'NUMPAD5', 'NUMPAD6', 'NUMPAD7', 'NUMPAD8', 'NUMPAD9', +'MULTIPLY', 'ADD', 'SUBTRACT', 'DECIMAL', 'DIVIDE', 'F1', 'F2', +'F3', 'F4', 'F5', 'F6', 'F7', 'F8', 'F9', 'F10', 'F11', 'F12', 'META', 'COMMAND ', +'null', 'cancel', 'help', 'backspace', 'back_space', 'tab', 'clear', 'return', 'enter', +'shift', 'left_shift', 'control', 'ctrl', 'left_control', 'alt', 'left_alt', 'pause', +'escape', 'space', 'page_up', 'page_down', 'end', 'home', 'left', 'arrow_left', 'up', +'arrow_up', 'right', 'arrow_right', 'down', 'arrow_down', 'insert', 'delete', 'del', +'semicolon', 'equals', 'numpad0', 'numpad1', 'numpad2', 'numpad3', 'numpad4', 'numpad5', +'numpad6', 'numpad7', 'numpad8', 'numpad9', 'multiply', 'add', 'subtract', 'decimal', +'divide', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12', +'meta', 'command ', +'\ue000', '\ue002', '\ue003', '\ue004', '\ue005', '\ue006', '\ue007', '\ue008', '\ue009', +'\ue009', '\ue00a', '\ue00b', '\ue00c', '\ue00d', '\ue00e', '\ue00f', '\ue010', '\ue011', +'\ue012', '\ue013', '\ue014', '\ue015', '\ue016', '\ue017', '\ue017', '\ue018', '\ue019', +'\ue01a', '\ue01b', '\ue01c', '\ue01d', '\ue01e', '\ue01f', '\ue020', '\ue021', '\ue022', +'\ue023', '\ue024', '\ue025', '\ue027', '\ue028', '\ue029', '\ue031', '\ue032', '\ue033', '\ue034', +'\ue035', '\ue036', '\ue037', '\ue038', '\ue039', '\ue03a', '\ue03b', '\ue03c', '\ue03d', '\ue03d' +] + class Actions: @@ -60,9 +85,9 @@ class Actions: def right(self, pixel: int) -> Actions: ... - def key_down(self, key: str) -> Actions: ... + def key_down(self, key: KEYS) -> Actions: ... - def key_up(self, key: str) -> Actions: ... + def key_up(self, key: KEYS) -> Actions: ... def type(self, text: Union[str, list, tuple]) -> Actions: ... diff --git a/DrissionPage/errors.py b/DrissionPage/errors.py index 30894b0..14e4907 100644 --- a/DrissionPage/errors.py +++ b/DrissionPage/errors.py @@ -79,3 +79,7 @@ class WaitTimeoutError(BaseError): class WrongURLError(BaseError): _info = '无效的url。' + + +class StorageError(BaseError): + _info = '无法操作当前存储数据。' diff --git a/requirements.txt b/requirements.txt index ea3dad4..b195908 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ requests lxml cssselect -DownloadKit>=2.0.0b0 +DownloadKit>=2.0.0b1 websocket-client click tldextract diff --git a/setup.py b/setup.py index ed2a7d6..800c5f1 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup( 'lxml', 'requests', 'cssselect', - 'DownloadKit>=2.0.0b0', + 'DownloadKit>=2.0.0b1', 'websocket-client', 'click', 'tldextract', From a11267c8a34a4bfab0db633c9f7fc22a8fca4390 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 12 Dec 2023 17:09:17 +0800 Subject: [PATCH 128/182] =?UTF-8?q?=E5=BE=AE=E8=B0=83=EF=BC=8C=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E5=B0=8F=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_configs/chromium_options.py | 24 +++++++++++----------- DrissionPage/_configs/chromium_options.pyi | 4 ++-- DrissionPage/_configs/session_options.py | 2 +- DrissionPage/_functions/tools.py | 2 +- DrissionPage/_units/actions.pyi | 14 ++++++++----- 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index 8c71fa8..ddbbece 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -124,16 +124,6 @@ class ChromiumOptions(object): """返回代理设置""" return self._proxy - @property - def debugger_address(self): - """返回浏览器地址，ip:port""" - return self._address - - @debugger_address.setter - def debugger_address(self, address): - """设置浏览器地址，格式ip:port""" - self.set_address(address) - @property def address(self): """返回浏览器地址，ip:port""" @@ -278,7 +268,7 @@ class ChromiumOptions(object): return self def clear_flags_in_file(self): - """删除浏览器设置文件中已设置的实验项""" + """删除浏览器配置文件中已设置的实验项""" self.clear_file_flags = True return self @@ -370,7 +360,7 @@ class ChromiumOptions(object): """ if search(r'.*?:.*?@.*?\..*', proxy): print('你似乎在设置使用账号密码的代理，暂时不支持这种代理，可自行用插件实现需求。') - if not proxy.lower().startswith('socks'): + if proxy.lower().startswith('socks'): print('你似乎在设置使用socks代理，暂时不支持这种代理，可自行用插件实现需求。') self._proxy = proxy return self.set_argument('--proxy-server', proxy) @@ -558,6 +548,16 @@ class ChromiumOptions(object): # ---------------即将废弃-------------- + @property + def debugger_address(self): + """返回浏览器地址，ip:port""" + return self._address + + @debugger_address.setter + def debugger_address(self, address): + """设置浏览器地址，格式ip:port""" + self.set_address(address) + def set_page_load_strategy(self, value): return self.set_load_mode(value) diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index 8398182..7a0d7c8 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from pathlib import Path -from typing import Union, Tuple, Any +from typing import Union, Tuple, Any, Literal class ChromiumOptions(object): @@ -121,7 +121,7 @@ class ChromiumOptions(object): def ignore_certificate_errors(self, on_off=True) -> ChromiumOptions: ... - def set_load_mode(self, value: str) -> ChromiumOptions: ... + def set_load_mode(self, value: Literal['normal', 'eager', 'none']) -> ChromiumOptions: ... def set_browser_path(self, path: Union[str, Path]) -> ChromiumOptions: ... diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index 14525b8..f68d6b0 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -89,7 +89,7 @@ class SessionOptions(object): """返回默认下载路径属性信息""" return self._download_path - def set_download_path(self, path=None): + def set_download_path(self, path): """设置默认下载路径 :param path: 下载路径 :return: 返回当前对象 diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index a7cc6fb..77a9d56 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -232,7 +232,7 @@ def stop_process_on_port(port): for proc in process_iter(['pid', 'connections']): try: connections = proc.connections() - except AccessDenied: + except (AccessDenied, NoSuchProcess): continue for conn in connections: if conn.laddr.port == int(port): diff --git a/DrissionPage/_units/actions.pyi b/DrissionPage/_units/actions.pyi index 9f8f717..94663e0 100644 --- a/DrissionPage/_units/actions.pyi +++ b/DrissionPage/_units/actions.pyi @@ -9,8 +9,8 @@ from .._base.chromium_driver import ChromiumDriver from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase -KEYS = Literal['NULL', 'CANCEL', 'HELP', 'BACKSPACE', 'BACK_SPACE', -'TAB', 'CLEAR', 'RETURN', 'ENTER', 'SHIFT', 'LEFT_SHIFT', 'CONTROL', +KEYS = Literal['NULL', 'CANCEL', 'HELP', 'BACKSPACE', 'BACK_SPACE', 'meta', +'TAB', 'CLEAR', 'RETURN', 'ENTER', 'SHIFT', 'LEFT_SHIFT', 'CONTROL', 'command ', 'CTRL', 'LEFT_CONTROL', 'ALT', 'LEFT_ALT', 'PAUSE', 'ESCAPE', 'SPACE', 'PAGE_UP', 'PAGE_DOWN', 'END', 'HOME', 'LEFT', 'ARROW_LEFT', 'UP', 'ARROW_UP', 'RIGHT', 'ARROW_RIGHT', 'DOWN', 'ARROW_DOWN', 'INSERT', @@ -25,14 +25,18 @@ KEYS = Literal['NULL', 'CANCEL', 'HELP', 'BACKSPACE', 'BACK_SPACE', 'semicolon', 'equals', 'numpad0', 'numpad1', 'numpad2', 'numpad3', 'numpad4', 'numpad5', 'numpad6', 'numpad7', 'numpad8', 'numpad9', 'multiply', 'add', 'subtract', 'decimal', 'divide', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12', -'meta', 'command ', '\ue000', '\ue002', '\ue003', '\ue004', '\ue005', '\ue006', '\ue007', '\ue008', '\ue009', '\ue009', '\ue00a', '\ue00b', '\ue00c', '\ue00d', '\ue00e', '\ue00f', '\ue010', '\ue011', '\ue012', '\ue013', '\ue014', '\ue015', '\ue016', '\ue017', '\ue017', '\ue018', '\ue019', '\ue01a', '\ue01b', '\ue01c', '\ue01d', '\ue01e', '\ue01f', '\ue020', '\ue021', '\ue022', '\ue023', '\ue024', '\ue025', '\ue027', '\ue028', '\ue029', '\ue031', '\ue032', '\ue033', '\ue034', -'\ue035', '\ue036', '\ue037', '\ue038', '\ue039', '\ue03a', '\ue03b', '\ue03c', '\ue03d', '\ue03d' -] +'\ue035', '\ue036', '\ue037', '\ue038', '\ue039', '\ue03a', '\ue03b', '\ue03c', '\ue03d', '\ue03d', +'`', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '-', '=', 'q', 'w', +'e', 'r', 't', 'y', 'u', 'i', 'o', 'p', '[', ']', '\\', 'a', 's', 'd', 'f', +'g', 'h', 'j', 'k', 'l', ';', '\'', 'z', 'x', 'c', 'v', 'b', 'n', 'm', ',', +'.', '/', '~', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '_', '+', +'Q', 'W', 'E', 'R', 'T', 'Y', 'U', 'I', 'O', 'P', '{', '}', 'A', 'S', 'D', +'F', 'G', 'H', 'J', 'K', 'L', ':', '"', 'Z', 'X', 'C', 'V', 'B', 'N', 'M', '<', '>', '?'] class Actions: From 6a063787ee83a3b67b68b912ca1ff90819453de8 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 12 Dec 2023 21:24:08 +0800 Subject: [PATCH 129/182] =?UTF-8?q?=E5=A2=9E=E5=8A=A0set.auto=5Fhandle=5Fa?= =?UTF-8?q?lert()=EF=BC=9Bhandle=5Falert()=E5=A2=9E=E5=8A=A0next=5Fone?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 37 +++++++++++++++++++-------- DrissionPage/_pages/chromium_base.pyi | 6 ++++- DrissionPage/_units/setter.py | 8 ++++++ DrissionPage/_units/setter.pyi | 2 ++ 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index eb31600..a48db70 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -835,13 +835,19 @@ class ChromiumBase(BasePage): if cookies: self.run_cdp_loaded('Network.clearBrowserCookies') - def handle_alert(self, accept=True, send=None, timeout=None): + def handle_alert(self, accept=True, send=None, timeout=None, next_one=False): """处理提示框，可以自动等待提示框出现 :param accept: True表示确认，False表示取消，其它值不会按按钮但依然返回文本值 :param send: 处理prompt提示框时可输入文本 :param timeout: 等待提示框出现的超时时间，为None则使用self.timeout属性的值 + :param next_one: 是否处理下一个出现的提示框，为True时timeout参数无效 :return: 提示框内容文本，未等到提示框则返回False """ + if next_one: + self._alert.handle_next = accept + self._alert.next_text = send + return + timeout = self.timeout if timeout is None else timeout timeout = .1 if timeout <= 0 else timeout end_time = perf_counter() + timeout @@ -857,16 +863,6 @@ class ChromiumBase(BasePage): self.driver.run('Page.handleJavaScriptDialog', accept=accept) return res_text - def _on_alert_close(self, **kwargs): - """alert关闭时触发的方法""" - self._alert.activated = False - self._alert.text = None - self._alert.type = None - self._alert.defaultPrompt = None - self._alert.response_accept = kwargs.get('result') - self._alert.response_text = kwargs['userInput'] - self._has_alert = False - def _on_alert_open(self, **kwargs): """alert出现时触发的方法""" self._alert.activated = True @@ -877,6 +873,22 @@ class ChromiumBase(BasePage): self._alert.response_text = None self._has_alert = True + if self._alert.auto is not None: + self.handle_alert(self._alert.auto) + elif self._alert.handle_next is not None: + self.handle_alert(self._alert.handle_next, self._alert.next_text) + self._alert.handle_next = None + + def _on_alert_close(self, **kwargs): + """alert关闭时触发的方法""" + self._alert.activated = False + self._alert.text = None + self._alert.type = None + self._alert.defaultPrompt = None + self._alert.response_accept = kwargs.get('result') + self._alert.response_text = kwargs['userInput'] + self._has_alert = False + def _wait_loaded(self, timeout=None): """等待页面加载完成，超时触发停止加载 :param timeout: 超时时间 @@ -1107,6 +1119,9 @@ class Alert(object): self.defaultPrompt = None self.response_accept = None self.response_text = None + self.handle_next = None + self.next_text = None + self.auto = None def close_privacy_dialog(page, tid): diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 24c9154..3624a20 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -227,7 +227,8 @@ class ChromiumBase(BasePage): def clear_cache(self, session_storage: bool = True, local_storage: bool = True, cache: bool = True, cookies: bool = True) -> None: ... - def handle_alert(self, accept: bool = True, send: str = None, timeout: float = None) -> Union[str, False]: ... + def handle_alert(self, accept: bool = True, send: str = None, timeout: float = None, + next_one: bool = False) -> Union[str, False]: ... def _on_alert_close(self, **kwargs): ... @@ -257,3 +258,6 @@ class Alert(object): self.defaultPrompt: str = ... self.response_accept: str = ... self.response_text: str = ... + self.handle_next: Optional[bool] = ... + self.next_text: str = ... + self.auto: Optional[bool] = ... diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index cfca821..4b5b952 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -146,6 +146,14 @@ class ChromiumBaseSetter(BasePageSetter): self._page.run_cdp('Network.enable') self._page.run_cdp('Network.setExtraHTTPHeaders', headers=headers) + def auto_handle_alert(self, on_off=True, accept=True): + """设置是否启用自动处理弹窗 + :param on_off: bool表示开或关 + :param accept: bool表示确定还是取消 + :return: None + """ + self._page._alert.auto = accept if on_off else None + # --------------即将废弃--------------- @property diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index 45faf59..19e2196 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -59,6 +59,8 @@ class ChromiumBaseSetter(BasePageSetter): def headers(self, headers: dict) -> None: ... + def auto_handle_alert(self, on_off: bool = True, accept: bool = True) -> None: ... + def upload_files(self, files: Union[str, list, tuple]) -> None: ... From 5d192997ba5684bf873a6d5d36696da2af27e86a Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 12 Dec 2023 23:09:32 +0800 Subject: [PATCH 130/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B0=8F=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_units/selector.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/DrissionPage/_units/selector.py b/DrissionPage/_units/selector.py index b3d9d21..0c168e6 100644 --- a/DrissionPage/_units/selector.py +++ b/DrissionPage/_units/selector.py @@ -110,8 +110,8 @@ class SelectElement(object): return self._by_loc(loc, timeout) def by_option(self, option): - """选中单个或多个选项元素 - :param option: 定位符 + """选中单个或多个option元素 + :param option: option元素或它们组成的列表 :return: None """ self._select_options(option, 'true') @@ -149,8 +149,8 @@ class SelectElement(object): return self._by_loc(loc, timeout, True) def cancel_by_option(self, option): - """选中单个或多个选项元素 - :param option: 定位符 + """取消选中单个或多个option元素 + :param option: option元素或它们组成的列表 :return: None """ self._select_options(option, 'false') @@ -251,6 +251,8 @@ class SelectElement(object): :return: None """ if isinstance(option, (list, tuple, set)): + if not self.is_multi: + raise TypeError("只能对多项选框执行多选。") for o in option: o.run_js(f'this.selected={mode};') self._dispatch_change() From ce2d14c34e3698732684825ec7d745bd1d16775a Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 12 Dec 2023 23:53:29 +0800 Subject: [PATCH 131/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B0=8F=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index ebf0796..71353d8 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -1374,9 +1374,11 @@ def convert_argument(arg): from math import inf if arg == inf: return {'unserializableValue': 'Infinity'} - if arg == -inf: + elif arg == -inf: return {'unserializableValue': '-Infinity'} + raise TypeError(f'不支持参数{arg}的类型：{type(arg)}') + class Pseudo(object): def __init__(self, ele): From 731d40a8915733e71517c37cc7a7a7b9f2a9d7cb Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 13 Dec 2023 15:34:58 +0800 Subject: [PATCH 132/182] =?UTF-8?q?=E5=A2=9E=E5=8A=A0CookieFormatError?= =?UTF-8?q?=EF=BC=9B=E5=8A=A8=E4=BD=9C=E9=93=BEtype()text=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E6=94=B9=E4=B8=BAkeys=EF=BC=8Cinput()=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E6=94=B9=E4=B8=BAtext?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_functions/tools.py | 4 +++- DrissionPage/_units/actions.py | 14 +++++++------- DrissionPage/_units/actions.pyi | 4 ++-- DrissionPage/errors.py | 4 ++++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index 77a9d56..c36af3d 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -13,7 +13,7 @@ from psutil import process_iter, AccessDenied, NoSuchProcess, ZombieProcess from .._configs.options_manage import OptionsManager from ..errors import (ContextLostError, ElementLostError, CDPError, PageClosedError, NoRectError, AlertExistsError, - WrongURLError, StorageError) + WrongURLError, StorageError, CookieFormatError) def get_usable_path(path, is_file=True, parents=True): @@ -278,6 +278,8 @@ def raise_error(r): raise WrongURLError(f'无效的url：{r["args"]["url"]}。也许要加上"http://"？') elif error == 'Frame corresponds to an opaque origin and its storage key cannot be serialized': raise StorageError + elif error == 'Sanitizing cookie failed': + raise CookieFormatError(f'cookie格式不正确：{r["args"]}') elif r['type'] == 'call_method_error': raise CDPError(f'\n错误：{r["error"]}\nmethod：{r["method"]}\nargs：{r["args"]}\n出现这个错误可能意味着程序有bug，' '请把错误信息和重现方法告知作者，谢谢。\n报告网站：https://gitee.com/g1879/DrissionPage/issues') diff --git a/DrissionPage/_units/actions.py b/DrissionPage/_units/actions.py index 33dc4c9..e933913 100644 --- a/DrissionPage/_units/actions.py +++ b/DrissionPage/_units/actions.py @@ -268,24 +268,24 @@ class Actions: self.page.run_cdp('Input.dispatchKeyEvent', **data) return self - def type(self, text): - """用模拟键盘按键方式输入文本，可输入字符串，只能输入键盘上有的字符 - :param text: 要输入的文本，特殊字符和多个文本可用list或tuple传入 + def type(self, keys): + """用模拟键盘按键方式输入文本，可输入字符串，也可输入组合键，只能输入键盘上有的字符 + :param keys: 要按下的按键，特殊字符和多个文本可用list或tuple传入 :return: self """ - for i in text: + for i in keys: for character in i: self.key_down(character) sleep(.05) self.key_up(character) return self - def input(self, text_or_keys): + def input(self, text): """输入文本，也可输入组合键，组合键用tuple形式输入 - :param text_or_keys: 文本值或按键组合 + :param text: 文本值或按键组合 :return: self """ - input_text_or_keys(self.page, text_or_keys) + input_text_or_keys(self.page, text) return self def wait(self, second): diff --git a/DrissionPage/_units/actions.pyi b/DrissionPage/_units/actions.pyi index 94663e0..3b6045c 100644 --- a/DrissionPage/_units/actions.pyi +++ b/DrissionPage/_units/actions.pyi @@ -93,9 +93,9 @@ class Actions: def key_up(self, key: KEYS) -> Actions: ... - def type(self, text: Union[str, list, tuple]) -> Actions: ... + def type(self, keys: Union[str, list, tuple]) -> Actions: ... - def input(self, text_or_keys: Any) -> Actions: ... + def input(self, text: Any) -> Actions: ... def wait(self, second: float) -> Actions: ... diff --git a/DrissionPage/errors.py b/DrissionPage/errors.py index 14e4907..5e8677a 100644 --- a/DrissionPage/errors.py +++ b/DrissionPage/errors.py @@ -83,3 +83,7 @@ class WrongURLError(BaseError): class StorageError(BaseError): _info = '无法操作当前存储数据。' + + +class CookieFormatError(BaseError): + _info = 'cookie格式不正确。' From 545b7a0732ca387a3f5d8b810a2b62ead94962cd Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 14 Dec 2023 00:45:13 +0800 Subject: [PATCH 133/182] =?UTF-8?q?=E4=B8=80=E4=BA=9B=E4=BF=AE=E6=94=B9(+)?= =?UTF-8?q?=20fullscreen()=E6=94=B9=E4=B8=BAfull()=EF=BC=9B=20corners?= =?UTF-8?q?=E8=BF=94=E5=9B=9Etuple=EF=BC=9B=20Frame=E5=A2=9E=E5=8A=A0state?= =?UTF-8?q?s.is=5Fdisplayed=E5=92=8Cstates.is=5Floading=EF=BC=9B=20has=5Fa?= =?UTF-8?q?lert=E7=A7=BB=E5=88=B0states=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 4 ++-- DrissionPage/_pages/chromium_base.py | 5 ----- DrissionPage/_pages/chromium_base.pyi | 3 --- DrissionPage/_pages/web_page.py | 2 +- DrissionPage/_pages/web_page.pyi | 2 ++ DrissionPage/_units/rect.py | 9 ++++----- DrissionPage/_units/rect.pyi | 8 ++++---- DrissionPage/_units/setter.py | 6 +++++- DrissionPage/_units/setter.pyi | 2 +- DrissionPage/_units/states.py | 18 ++++++++++++++++++ DrissionPage/_units/states.pyi | 6 ++++++ 11 files changed, 43 insertions(+), 22 deletions(-) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 71353d8..ebc6f7d 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -1287,7 +1287,7 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): obj_id = page_or_ele._root_id is_page = True - if page.has_alert: + if page.states.has_alert: raise AlertExistsError try: @@ -1309,7 +1309,7 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): else: raise ElementLostError('原来获取到的元素对象已不在页面内。') - if res is None and page.driver.has_alert: # 存在alert的情况 + if res is None and page.states.has_alert: # 存在alert的情况 return None exceptionDetails = res.get('exceptionDetails') diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index a48db70..437502c 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -426,11 +426,6 @@ class ChromiumBase(BasePage): """返回等待上传文件列表""" return self._upload_list - @property - def has_alert(self): - """返回是否存在提示框""" - return self._has_alert - @property def _js_ready_state(self): """返回js获取的ready state信息""" diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 3624a20..e9acefc 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -162,9 +162,6 @@ class ChromiumBase(BasePage): @property def states(self) -> PageStates: ... - @property - def has_alert(self) -> bool: ... - def run_js(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> Any: ... def run_js_loaded(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> Any: ... diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 3b08d1a..505f06d 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -349,7 +349,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): self._has_session = None def close(self): - """关闭标签页""" + """关闭标签页和Session""" if self._has_driver: self.close_tabs(self.tab_id) if self._session: diff --git a/DrissionPage/_pages/web_page.pyi b/DrissionPage/_pages/web_page.pyi index 7931c75..6b238e4 100644 --- a/DrissionPage/_pages/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -131,6 +131,8 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def close_session(self) -> None: ... + def close(self) -> None: ... + # ----------------重写SessionPage的函数----------------------- def post(self, url: str, diff --git a/DrissionPage/_units/rect.py b/DrissionPage/_units/rect.py index 0b9f0f0..93ac879 100644 --- a/DrissionPage/_units/rect.py +++ b/DrissionPage/_units/rect.py @@ -19,14 +19,13 @@ class ElementRect(object): r = self._ele.page.run_cdp_loaded('Page.getLayoutMetrics')['visualViewport'] sx = r['pageX'] sy = r['pageY'] - return [(vr[0] + sx, vr[1] + sy), (vr[2] + sx, vr[3] + sy), - (vr[4] + sx, vr[5] + sy), (vr[6] + sx, vr[7] + sy)] + return [(vr[0] + sx, vr[1] + sy), (vr[2] + sx, vr[3] + sy), (vr[4] + sx, vr[5] + sy), (vr[6] + sx, vr[7] + sy)] @property def viewport_corners(self): """返回元素四个角视口坐标，顺序：坐上、右上、右下、左下，没有大小的元素抛出NoRectError""" r = self._get_viewport_rect('border') - return [(r[0], r[1]), (r[2], r[3]), (r[4], r[5]), (r[6], r[7])] + return (r[0], r[1]), (r[2], r[3]), (r[4], r[5]), (r[6], r[7]) @property def size(self): @@ -188,12 +187,12 @@ class FrameRect(object): @property def location(self): - """返回元素左上角的绝对坐标""" + """返回iframe元素左上角的绝对坐标""" return self._frame.frame_ele.rect.location @property def viewport_location(self): - """返回视口在屏幕中坐标，左上角为(0, 0)""" + """返回元素在视口中坐标，左上角为(0, 0)""" return self._frame.frame_ele.rect.viewport_location @property diff --git a/DrissionPage/_units/rect.pyi b/DrissionPage/_units/rect.pyi index b45ef98..deb346f 100644 --- a/DrissionPage/_units/rect.pyi +++ b/DrissionPage/_units/rect.pyi @@ -49,10 +49,10 @@ class ElementRect(object): def screen_click_point(self) -> Tuple[float, float]: ... @property - def corners(self) -> List[Tuple[float, float], ...]: ... + def corners(self) -> Tuple[Tuple[float, float], ...]: ... @property - def viewport_corners(self) -> List[Tuple[float, float], ...]: ... + def viewport_corners(self) -> Tuple[Tuple[float, float], ...]: ... def _get_viewport_rect(self, quad: str) -> Union[list, None]: ... @@ -112,7 +112,7 @@ class FrameRect(object): def viewport_size(self) -> Tuple[float, float]: ... @property - def corners(self) -> List[Tuple[float, float], ...]: ... + def corners(self) -> Tuple[Tuple[float, float], ...]: ... @property - def viewport_corners(self) -> List[Tuple[float, float], ...]: ... + def viewport_corners(self) -> Tuple[Tuple[float, float], ...]: ... diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index 4b5b952..7d5486b 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -572,7 +572,7 @@ class WindowSetter(object): self._perform({'windowState': 'normal'}) self._perform({'windowState': 'minimized'}) - def fullscreen(self): + def full(self): """设置窗口为全屏""" s = self._get_info()['bounds']['windowState'] if s == 'minimized': @@ -635,6 +635,10 @@ class WindowSetter(object): """窗口最小化""" self.mini() + def fullscreen(self): + """设置窗口为全屏""" + self.full() + class PageWindowSetter(WindowSetter): def hide(self): diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index 19e2196..582e1af 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -207,7 +207,7 @@ class WindowSetter(object): def mini(self) -> None: ... - def fullscreen(self) -> None: ... + def full(self) -> None: ... def normal(self) -> None: ... diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index dc4a020..a53da2a 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -132,6 +132,11 @@ class PageStates(object): """返回当前页面加载状态，'loading' 'interactive' 'complete'""" return self._page._ready_state + @property + def has_alert(self): + """返回当前页面加载状态，'loading' 'interactive' 'complete'""" + return self._page._has_alert + class FrameStates(object): def __init__(self, frame): @@ -140,6 +145,11 @@ class FrameStates(object): """ self._frame = frame + @property + def is_loading(self): + """返回页面是否在加载状态""" + return self._frame._is_loading + @property def is_alive(self): """返回frame元素是否可用，且里面仍挂载有frame""" @@ -152,4 +162,12 @@ class FrameStates(object): @property def ready_state(self): + """返回加载状态""" return self._frame._ready_state + + @property + def is_displayed(self): + """返回iframe是否显示""" + return not (self._frame.frame_ele.style('visibility') == 'hidden' + or self._frame.frame_ele.run_js('return this.offsetParent === null;') + or self._frame.frame_ele.style('display') == 'none') diff --git a/DrissionPage/_units/states.pyi b/DrissionPage/_units/states.pyi index e3c303c..0bad17b 100644 --- a/DrissionPage/_units/states.pyi +++ b/DrissionPage/_units/states.pyi @@ -69,11 +69,17 @@ class PageStates(object): @property def ready_state(self) -> Optional[str]: ... + @property + def has_alert(self) -> bool: ... + class FrameStates(object): def __init__(self, frame: ChromiumFrame): self._frame: ChromiumFrame = ... + @property + def is_loading(self) -> bool: ... + @property def is_alive(self) -> bool: ... From 0cc81621defdfe4053557d15e4035825df10af0a Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 14 Dec 2023 16:23:55 +0800 Subject: [PATCH 134/182] =?UTF-8?q?=E4=B8=80=E4=BA=9B=E4=BF=AE=E6=94=B9(+)?= =?UTF-8?q?=20=E4=BF=AE=E5=A4=8Dauto=5Fport()=E9=97=AE=E9=A2=98=EF=BC=9B?= =?UTF-8?q?=20=E4=BF=AE=E5=A4=8Dcookie=E8=AE=BE=E7=BD=AE=E9=97=AE=E9=A2=98?= =?UTF-8?q?=20go=5Fon=E6=94=B9=E6=88=90resume=EF=BC=9B=20ready=5Fstate?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0'connecting'=EF=BC=9B=20ChromiumShadowRoot?= =?UTF-8?q?=E6=94=B9=E4=B8=BAShadowRoot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/base.py | 2 +- DrissionPage/_configs/chromium_options.py | 4 ++-- DrissionPage/_configs/chromium_options.pyi | 2 ++ DrissionPage/_elements/chromium_element.py | 12 ++++++------ DrissionPage/_elements/chromium_element.pyi | 8 ++++---- DrissionPage/_elements/session_element.py | 2 +- DrissionPage/_functions/web.py | 3 ++- DrissionPage/_pages/chromium_base.py | 2 +- DrissionPage/_units/listener.py | 2 +- DrissionPage/_units/listener.pyi | 2 +- DrissionPage/_units/states.py | 4 ++-- DrissionPage/_units/states.pyi | 6 +++--- 12 files changed, 26 insertions(+), 23 deletions(-) diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index 22ced5a..b71730a 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -85,7 +85,7 @@ class BaseElement(BaseParser): class DrissionElement(BaseElement): """ChromiumElement 和 SessionElement的基类 - 但不是ShadowRootElement的基类""" + 但不是ShadowRoot的基类""" @property def link(self): diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index ddbbece..c055d0b 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -599,19 +599,19 @@ class ChromiumOptions(object): class PortFinder(object): used_port = {} + lock = Lock() def __init__(self): self.tmp_dir = Path(gettempdir()) / 'DrissionPage' / 'TempFolder' self.tmp_dir.mkdir(parents=True, exist_ok=True) if not PortFinder.used_port: clean_folder(self.tmp_dir) - self._lock = Lock() def get_port(self): """查找一个可用端口 :return: 可以使用的端口和用户文件夹路径组成的元组 """ - with self._lock: + with PortFinder.lock: for i in range(9600, 19600): if i in PortFinder.used_port: continue diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index 7a0d7c8..5531f57 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -4,6 +4,7 @@ @Contact : g1879@qq.com """ from pathlib import Path +from threading import Lock from typing import Union, Tuple, Any, Literal @@ -152,6 +153,7 @@ class ChromiumOptions(object): class PortFinder(object): used_port: dict = ... + lock: Lock = ... @staticmethod def get_port() -> Tuple[int, str]: ... diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index ebc6f7d..9769a80 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -159,7 +159,7 @@ class ChromiumElement(DrissionElement): if not info.get('shadowRoots', None): return None - return ChromiumShadowRoot(self, backend_id=info['shadowRoots'][0]['backendNodeId']) + return ShadowRoot(self, backend_id=info['shadowRoots'][0]['backendNodeId']) @property def sr(self): @@ -764,8 +764,8 @@ class ChromiumElement(DrissionElement): return self.rect.size -class ChromiumShadowRoot(BaseElement): - """ChromiumShadowRoot是用于处理ShadowRoot的类，使用方法和ChromiumElement基本一致""" +class ShadowRoot(BaseElement): + """ShadowRoot是用于处理ShadowRoot的类，使用方法和ChromiumElement基本一致""" def __init__(self, parent_ele, obj_id=None, backend_id=None): """ @@ -786,7 +786,7 @@ class ChromiumShadowRoot(BaseElement): self._states = None def __repr__(self): - return f'' + return f'' def __call__(self, loc_or_str, timeout=None): """在内部查找元素 @@ -1278,7 +1278,7 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :return: js执行结果 """ - if isinstance(page_or_ele, (ChromiumElement, ChromiumShadowRoot)): + if isinstance(page_or_ele, (ChromiumElement, ShadowRoot)): page = page_or_ele.page obj_id = page_or_ele._obj_id is_page = False @@ -1337,7 +1337,7 @@ def parse_js_result(page, ele, result): elif sub_type == 'node': class_name = result['className'] if class_name == 'ShadowRoot': - return ChromiumShadowRoot(ele, obj_id=result['objectId']) + return ShadowRoot(ele, obj_id=result['objectId']) elif class_name == 'HTMLDocument': return result else: diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 95efc22..e4598d0 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -80,10 +80,10 @@ class ChromiumElement(DrissionElement): def pseudo(self) -> Pseudo: ... @property - def shadow_root(self) -> Union[None, ChromiumShadowRoot]: ... + def shadow_root(self) -> Union[None, ShadowRoot]: ... @property - def sr(self) -> Union[None, ChromiumShadowRoot]: ... + def sr(self) -> Union[None, ShadowRoot]: ... @property def scroll(self) -> ElementScroller: ... @@ -205,7 +205,7 @@ class ChromiumElement(DrissionElement): def _get_ele_path(self, mode: str) -> str: ... -class ChromiumShadowRoot(BaseElement): +class ShadowRoot(BaseElement): def __init__(self, parent_ele: ChromiumElement, obj_id: str = None, backend_id: str = None): self._obj_id: str = ... @@ -300,7 +300,7 @@ def make_chromium_ele(page: ChromiumBase, node_id: str = ..., obj_id: str = ...) def make_js_for_find_ele_by_xpath(xpath: str, type_txt: str, node_txt: str) -> str: ... -def run_js(page_or_ele: Union[ChromiumBase, ChromiumElement, ChromiumShadowRoot], script: str, +def run_js(page_or_ele: Union[ChromiumBase, ChromiumElement, ShadowRoot], script: str, as_expr: bool = False, timeout: float = None, args: tuple = ...) -> Any: ... diff --git a/DrissionPage/_elements/session_element.py b/DrissionPage/_elements/session_element.py index 6bd6326..be1402f 100644 --- a/DrissionPage/_elements/session_element.py +++ b/DrissionPage/_elements/session_element.py @@ -349,7 +349,7 @@ def make_session_ele(html_or_ele, loc=None, single=True): page = None html_or_ele = fromstring(html_or_ele) - # ShadowRootElement, ChromiumShadowRoot, ChromiumFrame + # ShadowRoot, ChromiumFrame elif isinstance(html_or_ele, BaseElement) or the_type.endswith(".ChromiumFrame'>"): page = html_or_ele.page html_or_ele = fromstring(html_or_ele.html) diff --git a/DrissionPage/_functions/web.py b/DrissionPage/_functions/web.py index 030c181..96258dd 100644 --- a/DrissionPage/_functions/web.py +++ b/DrissionPage/_functions/web.py @@ -250,7 +250,8 @@ def set_browser_cookies(page, cookies): if 'expiry' in cookie: cookie['expires'] = int(cookie['expiry']) cookie.pop('expiry') - if 'expires' in cookie: + + if 'expires' in cookie and isinstance(cookie['expires'], str): if cookie['expires'].isdigit(): cookie['expires'] = int(cookie['expires']) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 437502c..4843b65 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -190,7 +190,7 @@ class ChromiumBase(BasePage): print('在FrameStartedLoading变成loading') self._doc_got = False - self._ready_state = 'loading' + self._ready_state = 'connecting' self._is_loading = True self._load_end_time = perf_counter() + self.timeouts.page_load if self._load_mode == 'eager': diff --git a/DrissionPage/_units/listener.py b/DrissionPage/_units/listener.py index 5dd1638..830f3c1 100644 --- a/DrissionPage/_units/listener.py +++ b/DrissionPage/_units/listener.py @@ -173,7 +173,7 @@ class Listener(object): if clear: self.clear() - def go_on(self): + def resume(self): """继续暂停的监听""" if self.listening: return diff --git a/DrissionPage/_units/listener.pyi b/DrissionPage/_units/listener.pyi index 12eb344..d38d8be 100644 --- a/DrissionPage/_units/listener.pyi +++ b/DrissionPage/_units/listener.pyi @@ -38,7 +38,7 @@ class Listener(object): def pause(self, clear: bool = True) -> None: ... - def go_on(self) -> None: ... + def resume(self) -> None: ... def wait(self, count: int = 1, timeout: float = None, fit_count: bool = True, raise_err: bool = None) -> Union[List[DataPacket], DataPacket, None]: ... diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index a53da2a..00dfde9 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -129,12 +129,12 @@ class PageStates(object): @property def ready_state(self): - """返回当前页面加载状态，'loading' 'interactive' 'complete'""" + """返回当前页面加载状态，'connecting' 'loading' 'interactive' 'complete'""" return self._page._ready_state @property def has_alert(self): - """返回当前页面加载状态，'loading' 'interactive' 'complete'""" + """返回当前页面是否存在弹窗""" return self._page._has_alert diff --git a/DrissionPage/_units/states.pyi b/DrissionPage/_units/states.pyi index 0bad17b..102b1ea 100644 --- a/DrissionPage/_units/states.pyi +++ b/DrissionPage/_units/states.pyi @@ -5,7 +5,7 @@ """ from typing import Union, Tuple, List, Optional -from .._elements.chromium_element import ChromiumShadowRoot, ChromiumElement +from .._elements.chromium_element import ShadowRoot, ChromiumElement from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame @@ -43,11 +43,11 @@ class ElementStates(object): class ShadowRootStates(object): - def __init__(self, ele: ChromiumShadowRoot): + def __init__(self, ele: ShadowRoot): """ :param ele: ChromiumElement """ - self._ele: ChromiumShadowRoot = ... + self._ele: ShadowRoot = ... @property def is_enabled(self) -> bool: ... From cd9439be5cfa7f7e4269b19dad8a28e07f2c94c3 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 14 Dec 2023 23:25:11 +0800 Subject: [PATCH 135/182] =?UTF-8?q?4.0.0b21run=5Fjs()=E7=AD=89=E5=BE=85?= =?UTF-8?q?=E7=8E=AF=E5=A2=83=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_elements/chromium_element.py | 12 +++++++++--- DrissionPage/_pages/chromium_base.py | 11 ++++++++--- setup.py | 2 +- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 1c99f50..edd6064 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b20' +__version__ = '4.0.0b21' diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 9769a80..c12333c 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -1279,13 +1279,19 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): :return: js执行结果 """ if isinstance(page_or_ele, (ChromiumElement, ShadowRoot)): + is_page = False page = page_or_ele.page obj_id = page_or_ele._obj_id - is_page = False else: - page = page_or_ele - obj_id = page_or_ele._root_id is_page = True + page = page_or_ele + end_time = perf_counter() + 5 + while perf_counter() < end_time: + obj_id = page_or_ele._root_id + if obj_id is not None: + break + else: + raise RuntimeError('js运行环境出错。') if page.states.has_alert: raise AlertExistsError diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 4843b65..50bc643 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -155,9 +155,12 @@ class ChromiumBase(BasePage): return timeout = timeout if timeout >= .5 else .5 self._is_reading = True + end_time = perf_counter() + timeout try: b_id = self.run_cdp('DOM.getDocument', _timeout=timeout)['root']['backendNodeId'] - self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id, _timeout=1)['object']['objectId'] + timeout = end_time - perf_counter() + timeout = .5 if timeout < 0 else timeout + self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id, _timeout=timeout)['object']['objectId'] r = self.run_cdp('Page.getFrameTree') for i in findall(r"'id': '(.*?)'", str(r)): @@ -168,8 +171,10 @@ class ChromiumBase(BasePage): except: if self._debug: - print('获取文档失败') - return False + print('获取文档失败。') + print('请把报错信息和重现方法告知作者，感谢。\nhttps://gitee.com/g1879/DrissionPage/issues/new') + raise + # return False finally: self._is_loading = False diff --git a/setup.py b/setup.py index 800c5f1..7fc5b6b 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b20", + version="4.0.0b21", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From 7284b108ef6f2ba1128cfd490e1f6bc4391fb77d Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 15 Dec 2023 09:15:52 +0800 Subject: [PATCH 136/182] =?UTF-8?q?4.0.0b22=E4=BF=AE=E5=A4=8D=E5=B0=8F?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_pages/chromium_frame.py | 5 +++-- DrissionPage/_units/states.py | 5 +++++ DrissionPage/_units/states.pyi | 6 ++++++ setup.py | 2 +- 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index edd6064..c4cb7e4 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b21' +__version__ = '4.0.0b22' diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 55653c3..2c77f01 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -57,8 +57,9 @@ class ChromiumFrame(ChromiumBase): self._rect = None end_time = perf_counter() + 5 while perf_counter() < end_time: - if self.url is None or self.url == 'about:blank': - sleep(.1) + if self.url not in (None, 'about:blank'): + break + sleep(.1) def __call__(self, loc_or_str, timeout=None): """在内部查找元素 diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index 00dfde9..f1744f9 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -171,3 +171,8 @@ class FrameStates(object): return not (self._frame.frame_ele.style('visibility') == 'hidden' or self._frame.frame_ele.run_js('return this.offsetParent === null;') or self._frame.frame_ele.style('display') == 'none') + + @property + def has_alert(self): + """返回当前页面是否存在弹窗""" + return self._frame._has_alert diff --git a/DrissionPage/_units/states.pyi b/DrissionPage/_units/states.pyi index 102b1ea..fa8888c 100644 --- a/DrissionPage/_units/states.pyi +++ b/DrissionPage/_units/states.pyi @@ -85,3 +85,9 @@ class FrameStates(object): @property def ready_state(self) -> str: ... + + @property + def is_displayed(self) -> bool: ... + + @property + def has_alert(self) -> bool: ... diff --git a/setup.py b/setup.py index 7fc5b6b..9df76dc 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b21", + version="4.0.0b22", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From 35c25fa4542dc28edd82877ed66b678161922eb2 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 15 Dec 2023 16:52:53 +0800 Subject: [PATCH 137/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dre=E6=B2=A1=E6=9C=89?= =?UTF-8?q?=E9=80=82=E9=85=8D=E6=8D=A2=E8=A1=8C=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_functions/tools.py | 2 +- DrissionPage/_pages/chromium_base.py | 2 ++ DrissionPage/_pages/chromium_frame.py | 4 ++-- DrissionPage/_pages/session_page.py | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index c36af3d..d5e140f 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -266,7 +266,7 @@ def raise_error(r): 'No node with given id found', 'Node with given id does not belong to the document', 'No node found for given backend id'): raise ElementLostError - elif error == 'tab closed': + elif error == ('tab closed', 'No target with given id found'): raise PageClosedError elif error == 'timeout': raise TimeoutError diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 50bc643..63f7c19 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -172,6 +172,8 @@ class ChromiumBase(BasePage): except: if self._debug: print('获取文档失败。') + from traceback import print_exc + print_exc() print('请把报错信息和重现方法告知作者，感谢。\nhttps://gitee.com/g1879/DrissionPage/issues/new') raise # return False diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 2c77f01..228d96f 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from copy import copy -from re import search, findall +from re import search, findall, DOTALL from time import sleep, perf_counter from .._elements.chromium_element import ChromiumElement @@ -304,7 +304,7 @@ class ChromiumFrame(ChromiumBase): """返回元素outerHTML文本""" tag = self.tag out_html = self._target_page.run_cdp('DOM.getOuterHTML', backendNodeId=self.frame_ele._backend_id)['outerHTML'] - sign = search(rf'<{tag}.*?>', out_html).group(0) + sign = search(rf'<{tag}.*?>', out_html, DOTALL).group(0) return f'{sign}{self.inner_html}' @property diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index 038bf38..9dfcbf5 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from pathlib import Path -from re import search +from re import search, DOTALL from time import sleep from urllib.parse import urlparse, quote @@ -379,7 +379,7 @@ def set_charset(response): # 在headers中获取不到编码，且如果是网页 elif content_type.replace(' ', '').startswith('text/html'): - re_result = search(b']+).*?>', response.content) + re_result = search(b']+).*?>', response.content, DOTALL) if re_result: charset = re_result.group(1).decode() From 05cac3b69a5402320b5b83569c3f0dc083ad1c0d Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 17 Dec 2023 19:16:41 +0800 Subject: [PATCH 138/182] =?UTF-8?q?=E4=B8=8D=E5=B0=91=E4=BF=AE=E6=94=B9(+)?= =?UTF-8?q?=20=E4=BF=AE=E5=A4=8D=E4=B8=80=E4=B8=AAcookies=E9=97=AE?= =?UTF-8?q?=E9=A2=98=EF=BC=9B=20=E5=85=83=E7=B4=A0=E8=A2=AB=E8=A6=86?= =?UTF-8?q?=E7=9B=96=E6=97=B6states.is=5Fcovered=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E8=A6=86=E7=9B=96=E5=85=83=E7=B4=A0id=EF=BC=9B=20click()by=5Fj?= =?UTF-8?q?s=E9=BB=98=E8=AE=A4=E6=94=B9=E4=B8=BAFalse=EF=BC=9B=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8Dhtml=E5=B8=A6xml=E6=8F=8F=E8=BF=B0=E6=97=B6SessionPage?= =?UTF-8?q?=E6=8A=A5=E9=94=99=E9=97=AE=E9=A2=98=EF=BC=9B=20get()=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E9=81=BF=E5=85=8D=E6=B5=8F=E8=A7=88=E5=99=A8=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E9=87=8D=E8=AF=95=EF=BC=9B=20=E5=88=A0=E9=99=A4set.co?= =?UTF-8?q?okie()=EF=BC=9B=20=E5=A2=9E=E5=8A=A0set.cookies.clear()?= =?UTF-8?q?=E5=92=8Cset.cookies.remove()=EF=BC=9B=20set.cookies()=E5=8F=AF?= =?UTF-8?q?=E6=8E=A5=E6=94=B6=E5=8D=95=E4=B8=AAcookie=EF=BC=9B=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8Dselect=E9=97=AE=E9=A2=98=EF=BC=9B=20ChromiumDriver?= =?UTF-8?q?=E6=94=B9=E4=B8=BADriver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 12 ++-- DrissionPage/_base/browser.pyi | 4 +- .../_base/{chromium_driver.py => driver.py} | 6 +- .../_base/{chromium_driver.pyi => driver.pyi} | 8 +-- DrissionPage/_elements/chromium_element.pyi | 14 ++-- DrissionPage/_elements/session_element.py | 11 ++-- DrissionPage/_functions/web.py | 29 +++++---- DrissionPage/_pages/chromium_base.py | 56 ++++++++-------- DrissionPage/_pages/chromium_base.pyi | 6 +- DrissionPage/_pages/chromium_frame.pyi | 2 +- DrissionPage/_pages/chromium_page.py | 2 +- DrissionPage/_pages/web_page.py | 2 +- DrissionPage/_pages/web_page.pyi | 6 +- DrissionPage/_units/actions.pyi | 4 +- DrissionPage/_units/clicker.py | 4 +- DrissionPage/_units/clicker.pyi | 6 +- DrissionPage/_units/cookies_setter.py | 64 +++++++++++++++++++ DrissionPage/_units/cookies_setter.pyi | 30 +++++++++ DrissionPage/_units/listener.py | 6 +- DrissionPage/_units/listener.pyi | 4 +- DrissionPage/_units/selector.py | 2 +- DrissionPage/_units/setter.py | 52 +++++---------- DrissionPage/_units/setter.pyi | 19 +++--- DrissionPage/_units/states.py | 10 +-- DrissionPage/_units/states.pyi | 4 +- 25 files changed, 221 insertions(+), 142 deletions(-) rename DrissionPage/_base/{chromium_driver.py => driver.py} (98%) rename DrissionPage/_base/{chromium_driver.pyi => driver.pyi} (89%) create mode 100644 DrissionPage/_units/cookies_setter.py create mode 100644 DrissionPage/_units/cookies_setter.pyi diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 88da578..4f12011 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -5,7 +5,7 @@ """ from time import sleep, perf_counter -from .chromium_driver import BrowserDriver, ChromiumDriver +from .driver import BrowserDriver, Driver from .._functions.tools import stop_process_on_port, raise_error from .._units.downloader import DownloadManager @@ -42,7 +42,7 @@ class Browser(object): self.id = browser_id self._frames = {} self._drivers = {} - # self._drivers = {t: ChromiumDriver(t, 'page', address) for t in self.tabs} + # self._drivers = {t: Driver(t, 'page', address) for t in self.tabs} self._connected = False self._process_id = None @@ -57,16 +57,16 @@ class Browser(object): self._driver.set_callback('Target.targetCreated', self._onTargetCreated) def _get_driver(self, tab_id): - """获取对应tab id的ChromiumDriver + """获取对应tab id的Driver :param tab_id: 标签页id - :return: ChromiumDriver对象 + :return: Driver对象 """ - return self._drivers.pop(tab_id, ChromiumDriver(tab_id, 'page', self.address)) + return self._drivers.pop(tab_id, Driver(tab_id, 'page', self.address)) def _onTargetCreated(self, **kwargs): """标签页创建时执行""" if kwargs['targetInfo']['type'] == 'page' and not kwargs['targetInfo']['url'].startswith('devtools://'): - self._drivers[kwargs['targetInfo']['targetId']] = ChromiumDriver(kwargs['targetInfo']['targetId'], 'page', + self._drivers[kwargs['targetInfo']['targetId']] = Driver(kwargs['targetInfo']['targetId'], 'page', self.address) def _onTargetDestroyed(self, **kwargs): diff --git a/DrissionPage/_base/browser.pyi b/DrissionPage/_base/browser.pyi index 3bf21be..b5d0161 100644 --- a/DrissionPage/_base/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -5,7 +5,7 @@ """ from typing import List, Optional, Union -from .chromium_driver import BrowserDriver, ChromiumDriver +from .driver import BrowserDriver, Driver from .._pages.chromium_page import ChromiumPage from .._units.downloader import DownloadManager @@ -26,7 +26,7 @@ class Browser(object): def __init__(self, address: str, browser_id: str, page: ChromiumPage): ... - def _get_driver(self, tab_id: str) -> ChromiumDriver: ... + def _get_driver(self, tab_id: str) -> Driver: ... def run_cdp(self, cmd, **cmd_args) -> dict: ... diff --git a/DrissionPage/_base/chromium_driver.py b/DrissionPage/_base/driver.py similarity index 98% rename from DrissionPage/_base/chromium_driver.py rename to DrissionPage/_base/driver.py index 422eeb4..77b1329 100644 --- a/DrissionPage/_base/chromium_driver.py +++ b/DrissionPage/_base/driver.py @@ -12,7 +12,7 @@ from requests import get from websocket import WebSocketTimeoutException, WebSocketConnectionClosedException, create_connection -class ChromiumDriver(object): +class Driver(object): def __init__(self, tab_id, tab_type, address): """ :param tab_id: 标签页id @@ -211,12 +211,12 @@ class ChromiumDriver(object): self.event_handlers.pop(event, None) def __str__(self): - return f"" + return f"" __repr__ = __str__ -class BrowserDriver(ChromiumDriver): +class BrowserDriver(Driver): BROWSERS = {} def __new__(cls, tab_id, tab_type, address, browser): diff --git a/DrissionPage/_base/chromium_driver.pyi b/DrissionPage/_base/driver.pyi similarity index 89% rename from DrissionPage/_base/chromium_driver.pyi rename to DrissionPage/_base/driver.pyi index 28d8692..e2833da 100644 --- a/DrissionPage/_base/chromium_driver.pyi +++ b/DrissionPage/_base/driver.pyi @@ -14,14 +14,14 @@ from .browser import Browser class GenericAttr(object): - def __init__(self, name: str, tab: ChromiumDriver): ... + def __init__(self, name: str, tab: Driver): ... def __getattr__(self, item: str) -> Callable: ... def __setattr__(self, key: str, value: Callable) -> None: ... -class ChromiumDriver(object): +class Driver(object): id: str address: str type: str @@ -58,8 +58,8 @@ class ChromiumDriver(object): def __str__(self) -> str: ... -class BrowserDriver(ChromiumDriver): - BROWSERS: Dict[str, ChromiumDriver] = ... +class BrowserDriver(Driver): + BROWSERS: Dict[str, Driver] = ... browser: Browser = ... def __new__(cls, tab_id: str, tab_type: str, address: str, browser: Browser): ... diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index e4598d0..bb39b75 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -26,12 +26,12 @@ PIC_TYPE = Literal['jpg', 'jpeg', 'png', 'webp', True] class ChromiumElement(DrissionElement): - def __init__(self, page: ChromiumBase, node_id: str = None, obj_id: str = None, backend_id: str = None): + def __init__(self, page: ChromiumBase, node_id: str = None, obj_id: str = None, backend_id: int = None): self._tag: str = ... self.page: Union[ChromiumPage, WebPage] = ... self._node_id: str = ... self._obj_id: str = ... - self._backend_id: str = ... + self._backend_id: int = ... self._doc_id: str = ... self._scroll: ElementScroller = ... self._clicker: Clicker = ... @@ -196,9 +196,9 @@ class ChromiumElement(DrissionElement): def drag_to(self, ele_or_loc: Union[tuple, ChromiumElement], duration: float = 0.5) -> None: ... - def _get_obj_id(self, node_id: str = None, backend_id: str = None) -> str: ... + def _get_obj_id(self, node_id: str = None, backend_id: int = None) -> str: ... - def _get_node_id(self, obj_id: str = None, backend_id: str = None) -> str: ... + def _get_node_id(self, obj_id: str = None, backend_id: int = None) -> str: ... def _get_backend_id(self, node_id: str) -> str: ... @@ -207,10 +207,10 @@ class ChromiumElement(DrissionElement): class ShadowRoot(BaseElement): - def __init__(self, parent_ele: ChromiumElement, obj_id: str = None, backend_id: str = None): + def __init__(self, parent_ele: ChromiumElement, obj_id: str = None, backend_id: int = None): self._obj_id: str = ... self._node_id: str = ... - self._backend_id: str = ... + self._backend_id: int = ... self.page: ChromiumPage = ... self.parent_ele: ChromiumElement = ... self._states: ShadowRootStates = ... @@ -277,7 +277,7 @@ class ShadowRoot(BaseElement): def _get_obj_id(self, back_id: str) -> str: ... - def _get_backend_id(self, node_id: str) -> str: ... + def _get_backend_id(self, node_id: str) -> int: ... def find_in_chromium_ele(ele: ChromiumElement, loc: Union[str, Tuple[str, str]], diff --git a/DrissionPage/_elements/session_element.py b/DrissionPage/_elements/session_element.py index be1402f..430e6c4 100644 --- a/DrissionPage/_elements/session_element.py +++ b/DrissionPage/_elements/session_element.py @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from html import unescape -from re import match, DOTALL +from re import match, sub, DOTALL from lxml.etree import tostring from lxml.html import HtmlElement, fromstring @@ -342,15 +342,18 @@ def make_session_ele(html_or_ele, loc=None, single=True): # 各种页面对象 elif isinstance(html_or_ele, BasePage): page = html_or_ele - html_or_ele = fromstring(html_or_ele.html) + html = html_or_ele.html + if html.startswith('', '', html) + html_or_ele = fromstring(html) # 直接传入html文本 elif isinstance(html_or_ele, str): page = None html_or_ele = fromstring(html_or_ele) - # ShadowRoot, ChromiumFrame - elif isinstance(html_or_ele, BaseElement) or the_type.endswith(".ChromiumFrame'>"): + # ShadowRoot + elif isinstance(html_or_ele, BaseElement): page = html_or_ele.page html_or_ele = fromstring(html_or_ele.html) diff --git a/DrissionPage/_functions/web.py b/DrissionPage/_functions/web.py index 96258dd..4e4cee7 100644 --- a/DrissionPage/_functions/web.py +++ b/DrissionPage/_functions/web.py @@ -251,20 +251,25 @@ def set_browser_cookies(page, cookies): cookie['expires'] = int(cookie['expiry']) cookie.pop('expiry') - if 'expires' in cookie and isinstance(cookie['expires'], str): - if cookie['expires'].isdigit(): - cookie['expires'] = int(cookie['expires']) + if 'expires' in cookie: + if not cookie['expires']: + cookie.pop('expires') - elif cookie['expires'].replace('.', '').isdigit(): - cookie['expires'] = float(cookie['expires']) + elif isinstance(cookie['expires'], str): + if cookie['expires'].isdigit(): + cookie['expires'] = int(cookie['expires']) + + elif cookie['expires'].replace('.', '').isdigit(): + cookie['expires'] = float(cookie['expires']) + + else: + try: + cookie['expires'] = datetime.strptime(cookie['expires'], + '%a, %d %b %Y %H:%M:%S GMT').timestamp() + except ValueError: + cookie['expires'] = datetime.strptime(cookie['expires'], + '%a, %d %b %y %H:%M:%S GMT').timestamp() - else: - try: - cookie['expires'] = datetime.strptime(cookie['expires'], - '%a, %d %b %Y %H:%M:%S GMT').timestamp() - except ValueError: - cookie['expires'] = datetime.strptime(cookie['expires'], - '%a, %d %b %y %H:%M:%S GMT').timestamp() if cookie['value'] is None: cookie['value'] = '' elif not isinstance(cookie['value'], str): diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 63f7c19..e0d2a3e 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -156,31 +156,31 @@ class ChromiumBase(BasePage): timeout = timeout if timeout >= .5 else .5 self._is_reading = True end_time = perf_counter() + timeout - try: - b_id = self.run_cdp('DOM.getDocument', _timeout=timeout)['root']['backendNodeId'] - timeout = end_time - perf_counter() - timeout = .5 if timeout < 0 else timeout - self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id, _timeout=timeout)['object']['objectId'] + while perf_counter() < end_time: + try: + b_id = self.run_cdp('DOM.getDocument', _timeout=timeout)['root']['backendNodeId'] + timeout = end_time - perf_counter() + timeout = .5 if timeout < 0 else timeout + self._root_id = self.run_cdp('DOM.resolveNode', + backendNodeId=b_id, _timeout=timeout)['object']['objectId'] + r = self.run_cdp('Page.getFrameTree') + for i in findall(r"'id': '(.*?)'", str(r)): + self.browser._frames[i] = self.tab_id + if self._debug: + print('获取文档结束') + result = True + break - r = self.run_cdp('Page.getFrameTree') - for i in findall(r"'id': '(.*?)'", str(r)): - self.browser._frames[i] = self.tab_id - if self._debug: - print('获取文档结束') - return True + except: + timeout = end_time - perf_counter() + timeout = .5 if timeout < 0 else timeout - except: - if self._debug: - print('获取文档失败。') - from traceback import print_exc - print_exc() - print('请把报错信息和重现方法告知作者，感谢。\nhttps://gitee.com/g1879/DrissionPage/issues/new') - raise - # return False + else: + result = False - finally: - self._is_loading = False - self._is_reading = False + self._is_loading = False + self._is_reading = False + return result def _onFrameDetached(self, **kwargs): self.browser._frames.pop(kwargs['frameId'], None) @@ -230,8 +230,8 @@ class ChromiumBase(BasePage): if self._load_mode == 'eager': self.run_cdp('Page.stopLoading') - self._get_document(self._load_end_time - perf_counter() - .1) - self._doc_got = True + if self._get_document(self._load_end_time - perf_counter() - .1): + self._doc_got = True self._ready_state = 'interactive' if self._debug: @@ -243,8 +243,7 @@ class ChromiumBase(BasePage): print(f'{self._frame_id}触发LoadEventFired') print('在LoadEventFired变成complete') - if self._doc_got is False: - self._get_document(self._load_end_time - perf_counter() - .1) + if self._doc_got is False and self._get_document(self._load_end_time - perf_counter() - .1): self._doc_got = True self._ready_state = 'complete' @@ -369,7 +368,7 @@ class ChromiumBase(BasePage): @property def driver(self): - """返回用于控制浏览器的ChromiumDriver对象""" + """返回用于控制浏览器的Driver对象""" if self._driver is None: raise RuntimeError('浏览器已关闭或链接已断开。') return self._driver @@ -960,6 +959,9 @@ class ChromiumBase(BasePage): sleep(interval) if self._debug or show_errmsg: print(f'重试{t + 1} {to_url}') + end_time1 = end_time - perf_counter() + while self._ready_state not in ('loading', 'complete') and perf_counter() < end_time1: # 等待出错信息显示 + sleep(.1) self.stop_loading() continue diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index e9acefc..db025c0 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -8,7 +8,7 @@ from typing import Union, Tuple, List, Any, Optional, Literal from .._base.base import BasePage from .._base.browser import Browser -from .._base.chromium_driver import ChromiumDriver +from .._base.driver import Driver from .._elements.chromium_element import ChromiumElement from .._elements.none_element import NoneElement from .._elements.session_element import SessionElement @@ -34,7 +34,7 @@ class ChromiumBase(BasePage): self._browser: Browser = ... self._page: ChromiumPage = ... self.address: str = ... - self._driver: ChromiumDriver = ... + self._driver: Driver = ... self._frame_id: str = ... self._is_reading: bool = ... self._is_timeout: bool = ... @@ -103,7 +103,7 @@ class ChromiumBase(BasePage): def title(self) -> str: ... @property - def driver(self) -> ChromiumDriver: ... + def driver(self) -> Driver: ... @property def url(self) -> str: ... diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index b7d6892..37df5a6 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -27,7 +27,7 @@ class ChromiumFrame(ChromiumBase): self.tab: ChromiumTab = ... self._tab_id: str = ... self._frame_ele: ChromiumElement = ... - self._backend_id: str = ... + self._backend_id: int = ... self._doc_ele: ChromiumElement = ... self._is_diff_domain: bool = ... self.doc_ele: ChromiumElement = ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 61d9104..2c53511 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -37,7 +37,7 @@ class ChromiumPage(ChromiumBase): def _handle_options(self, addr_or_opts): """设置浏览器启动属性 - :param addr_or_opts: 'ip:port'、ChromiumOptions、ChromiumDriver + :param addr_or_opts: 'ip:port'、ChromiumOptions、Driver :return: 返回浏览器地址 """ if not addr_or_opts: diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 505f06d..a62d30c 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -19,7 +19,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): """初始化函数 :param mode: 'd' 或 's'，即driver模式和session模式 :param timeout: 超时时间，d模式时为寻找元素时间，s模式时为连接时间，默认10秒 - :param chromium_options: ChromiumDriver对象，只使用s模式时应传入False + :param chromium_options: Driver对象，只使用s模式时应传入False :param session_or_options: Session对象或SessionOptions对象，只使用d模式时应传入False """ chromium_options = chromium_options or driver_or_options diff --git a/DrissionPage/_pages/web_page.pyi b/DrissionPage/_pages/web_page.pyi index 6b238e4..a0b78d5 100644 --- a/DrissionPage/_pages/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -12,7 +12,7 @@ from .chromium_page import ChromiumPage from .chromium_tab import WebPageTab from .session_page import SessionPage from .._base.base import BasePage -from .._base.chromium_driver import ChromiumDriver +from .._base.driver import Driver from .._configs.chromium_options import ChromiumOptions from .._configs.session_options import SessionOptions from .._elements.chromium_element import ChromiumElement @@ -162,9 +162,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): -> Union[ChromiumElement, SessionElement, ChromiumFrame, str, None, List[Union[SessionElement, str]], List[ Union[ChromiumElement, str, ChromiumFrame]]]: ... - def _set_start_options(self, dr_opt: Union[ChromiumDriver, bool, None], + def _set_start_options(self, dr_opt: Union[Driver, bool, None], se_opt: Union[Session, SessionOptions, bool, None]) -> None: ... def quit(self, timeout: float = 5, force: bool = True) -> None: ... - - def _on_download_begin(self, **kwargs): ... diff --git a/DrissionPage/_units/actions.pyi b/DrissionPage/_units/actions.pyi index 3b6045c..982ead1 100644 --- a/DrissionPage/_units/actions.pyi +++ b/DrissionPage/_units/actions.pyi @@ -5,7 +5,7 @@ """ from typing import Union, Tuple, Any, Literal -from .._base.chromium_driver import ChromiumDriver +from .._base.driver import Driver from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase @@ -43,7 +43,7 @@ class Actions: def __init__(self, page: ChromiumBase): self.page: ChromiumBase = ... - self._dr: ChromiumDriver = ... + self._dr: Driver = ... self.modifier: int = ... self.curr_x: int = ... self.curr_y: int = ... diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index 943e49c..27d6719 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -17,7 +17,7 @@ class Clicker(object): """ self._ele = ele - def __call__(self, by_js=None, timeout=1.5, wait_stop=True): + def __call__(self, by_js=False, timeout=1.5, wait_stop=True): """点击元素 如果遇到遮挡，可选择是否用js点击 :param by_js: 是否用js点击，为None时先用模拟点击，遇到遮挡改用js，为True时直接用js点击，为False时只用模拟点击 @@ -27,7 +27,7 @@ class Clicker(object): """ return self.left(by_js, timeout, wait_stop) - def left(self, by_js=None, timeout=1.5, wait_stop=True): + def left(self, by_js=False, timeout=1.5, wait_stop=True): """点击元素，可选择是否用js点击 :param by_js: 是否用js点击，为None时先用模拟点击，遇到遮挡改用js，为True时直接用js点击，为False时只用模拟点击 :param timeout: 模拟点击的超时时间，等待元素可见、可用、进入视口 diff --git a/DrissionPage/_units/clicker.pyi b/DrissionPage/_units/clicker.pyi index d62b99f..34286a9 100644 --- a/DrissionPage/_units/clicker.pyi +++ b/DrissionPage/_units/clicker.pyi @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union +from typing import Union, Optional from .._elements.chromium_element import ChromiumElement @@ -12,9 +12,9 @@ class Clicker(object): def __init__(self, ele: ChromiumElement): self._ele: ChromiumElement = ... - def __call__(self, by_js: Union[None, bool] = None, timeout: float = 1.5, wait_stop: bool = True) -> bool: ... + def __call__(self, by_js: Optional[bool] = False, timeout: float = 1.5, wait_stop: bool = True) -> bool: ... - def left(self, by_js: Union[None, bool] = None, timeout: float = 1.5, wait_stop: bool = True) -> bool: ... + def left(self, by_js: Optional[bool] = False, timeout: float = 1.5, wait_stop: bool = True) -> bool: ... def right(self) -> None: ... diff --git a/DrissionPage/_units/cookies_setter.py b/DrissionPage/_units/cookies_setter.py new file mode 100644 index 0000000..160077e --- /dev/null +++ b/DrissionPage/_units/cookies_setter.py @@ -0,0 +1,64 @@ +# -*- coding:utf-8 -*- +from http.cookiejar import Cookie + +from .._functions.web import set_browser_cookies, set_session_cookies + + +class CookiesSetter(object): + def __init__(self, page): + self._page = page + + def __call__(self, cookies): + """设置一个或多个cookie + :param cookies: cookies信息 + :return: None + """ + if (isinstance(cookies, dict) and 'name' in cookies and 'value' in cookies) or isinstance(cookies, Cookie): + cookies = [cookies] + set_browser_cookies(self._page, cookies) + + def remove(self, name, url=None, domain=None, path=None): + """删除一个cookie + :param name: cookie的name字段 + :param url: cookie的url字段，可选 + :param domain: cookie的domain字段，可选 + :param path: cookie的path字段，可选 + :return: None + """ + d = {'name': name} + if url is not None: + d['url'] = url + if domain is not None: + d['domain'] = domain + if path is not None: + d['path'] = path + self._page.run_cdp('Network.deleteCookies', **d) + + def clear(self): + """清除cookies""" + self._page.run_cdp('Network.clearBrowserCookies') + + +class SessionCookiesSetter(object): + def __init__(self, page): + self._page = page + + def __call__(self, cookies): + """设置多个cookie，注意不要传入单个 + :param cookies: cookies信息 + :return: None + """ + if (isinstance(cookies, dict) and 'name' in cookies and 'value' in cookies) or isinstance(cookies, Cookie): + cookies = [cookies] + set_session_cookies(self._page.session, cookies) + + def remove(self, name): + """删除一个cookie + :param name: cookie的name字段 + :return: None + """ + self._page.session.cookies.set(name, None) + + def clear(self): + """清除cookies""" + self._page.session.cookies.clear() diff --git a/DrissionPage/_units/cookies_setter.pyi b/DrissionPage/_units/cookies_setter.pyi new file mode 100644 index 0000000..4bfc3e9 --- /dev/null +++ b/DrissionPage/_units/cookies_setter.pyi @@ -0,0 +1,30 @@ +# -*- coding:utf-8 -*- +from http.cookiejar import Cookie +from typing import Union + +from requests.cookies import RequestsCookieJar + +from .._pages.session_page import SessionPage +from .._pages.chromium_base import ChromiumBase + + +class CookiesSetter(object): + def __init__(self, page: ChromiumBase): + self._page: ChromiumBase = ... + + def __call__(self, cookies: Union[RequestsCookieJar, Cookie, list, tuple, str, dict]) -> None: ... + + def remove(self, name: str, url: str = None, domain: str = None, path: str = None) -> None: ... + + def clear(self) -> None: ... + + +class SessionCookiesSetter(object): + def __init__(self, page: SessionPage): + self._page: SessionPage = ... + + def __call__(self, cookies: Union[RequestsCookieJar, Cookie, list, tuple, str, dict]) -> None: ... + + def remove(self, name: str) -> None: ... + + def clear(self) -> None: ... diff --git a/DrissionPage/_units/listener.py b/DrissionPage/_units/listener.py index 830f3c1..d18759f 100644 --- a/DrissionPage/_units/listener.py +++ b/DrissionPage/_units/listener.py @@ -11,7 +11,7 @@ from time import perf_counter, sleep from requests.structures import CaseInsensitiveDict -from .._base.chromium_driver import ChromiumDriver +from .._base.driver import Driver from .._functions.settings import Settings from ..errors import WaitTimeoutError @@ -84,7 +84,7 @@ class Listener(object): if self.listening: return - self._driver = ChromiumDriver(self._target_id, 'page', self._address) + self._driver = Driver(self._target_id, 'page', self._address) self._driver.run('Network.enable') self._set_callback() @@ -222,7 +222,7 @@ class Listener(object): debug = self._driver._debug self._driver.stop() if self.listening: - self._driver = ChromiumDriver(self._target_id, 'page', self._address) + self._driver = Driver(self._target_id, 'page', self._address) self._driver._debug = debug self._driver.run('Network.enable') self._set_callback() diff --git a/DrissionPage/_units/listener.pyi b/DrissionPage/_units/listener.pyi index d38d8be..c281c68 100644 --- a/DrissionPage/_units/listener.pyi +++ b/DrissionPage/_units/listener.pyi @@ -8,7 +8,7 @@ from typing import Union, Dict, List, Iterable, Tuple, Optional from requests.structures import CaseInsensitiveDict -from .._base.chromium_driver import ChromiumDriver +from .._base.driver import Driver from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame @@ -22,7 +22,7 @@ class Listener(object): self._method: set = ... self._caught: Queue = ... self._is_regex: bool = ... - self._driver: ChromiumDriver = ... + self._driver: Driver = ... self._request_ids: dict = ... self._extra_info_ids: dict = ... self.listening: bool = ... diff --git a/DrissionPage/_units/selector.py b/DrissionPage/_units/selector.py index 0c168e6..8d945d2 100644 --- a/DrissionPage/_units/selector.py +++ b/DrissionPage/_units/selector.py @@ -251,7 +251,7 @@ class SelectElement(object): :return: None """ if isinstance(option, (list, tuple, set)): - if not self.is_multi: + if not self.is_multi and len(option) > 1: raise TypeError("只能对多项选框执行多选。") for o in option: o.run_js(f'this.selected={mode};') diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index 7d5486b..5c27059 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -7,8 +7,8 @@ from pathlib import Path from requests.structures import CaseInsensitiveDict +from .cookies_setter import SessionCookiesSetter, CookiesSetter from .._functions.tools import show_or_hide_browser -from .._functions.web import set_browser_cookies, set_session_cookies class BasePageSetter(object): @@ -28,6 +28,7 @@ class BasePageSetter(object): class ChromiumBaseSetter(BasePageSetter): def __init__(self, page): super().__init__(page) + self._cookies_setter = None @property def load_mode(self): @@ -39,6 +40,13 @@ class ChromiumBaseSetter(BasePageSetter): """返回用于设置页面滚动设置的对象""" return PageScrollSetter(self._page.scroll) + @property + def cookies(self): + """返回用于设置cookies的对象""" + if self._cookies_setter is None: + self._cookies_setter = CookiesSetter(self._page) + return self._cookies_setter + def retry_times(self, times): """设置连接失败重连次数""" self._page.retry_times = times @@ -108,23 +116,6 @@ class ChromiumBaseSetter(BasePageSetter): key=item, value=value) self._page.run_cdp_loaded('DOMStorage.disable') - def cookie(self, cookie): - """设置单个cookie - :param cookie: cookie信息 - :return: None - """ - if isinstance(cookie, str): - self.cookies(cookie) - else: - self.cookies([cookie]) - - def cookies(self, cookies): - """设置多个cookie，注意不要传入单个 - :param cookies: cookies信息 - :return: None - """ - set_browser_cookies(self._page, cookies) - def upload_files(self, files): """等待上传的文件路径 :param files: 文件路径列表或字符串，字符串时多个文件用回车分隔 @@ -233,6 +224,14 @@ class SessionPageSetter(BasePageSetter): :param page: SessionPage对象 """ super().__init__(page) + self._cookies_setter = None + + @property + def cookies(self): + """返回用于设置cookies的对象""" + if self._cookies_setter is None: + self._cookies_setter = SessionCookiesSetter(self._page) + return self._cookies_setter def retry_times(self, times): """设置连接失败时重连次数""" @@ -270,23 +269,6 @@ class SessionPageSetter(BasePageSetter): if self._page.response: self._page.response.encoding = encoding - def cookie(self, cookie): - """为Session对象设置单个cookie - :param cookie: cookie信息 - :return: None - """ - if isinstance(cookie, str): - self.cookies(cookie) - else: - self.cookies([cookie]) - - def cookies(self, cookies): - """为Session对象设置多个cookie，注意不要传入单个 - :param cookies: cookies信息 - :return: None - """ - set_session_cookies(self._page.session, cookies) - def headers(self, headers): """设置通用的headers :param headers: dict形式的headers diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index 582e1af..415decb 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -3,14 +3,13 @@ @Author : g1879 @Contact : g1879@qq.com """ -from http.cookiejar import Cookie from pathlib import Path from typing import Union, Tuple, Literal, Any, Optional from requests.adapters import HTTPAdapter from requests.auth import HTTPBasicAuth -from requests.cookies import RequestsCookieJar +from .cookies_setter import SessionCookiesSetter, CookiesSetter from .scroller import PageScroller from .._base.base import BasePage from .._elements.chromium_element import ChromiumElement @@ -34,6 +33,7 @@ class BasePageSetter(object): class ChromiumBaseSetter(BasePageSetter): def __init__(self, page): self._page: ChromiumBase = ... + self._cookies_setter: CookiesSetter = ... @property def load_mode(self) -> LoadMode: ... @@ -41,6 +41,9 @@ class ChromiumBaseSetter(BasePageSetter): @property def scroll(self) -> PageScrollSetter: ... + @property + def cookies(self) -> CookiesSetter: ... + def retry_times(self, times: int) -> None: ... def retry_interval(self, interval: float) -> None: ... @@ -53,10 +56,6 @@ class ChromiumBaseSetter(BasePageSetter): def local_storage(self, item: str, value: Union[str, bool]) -> None: ... - def cookie(self, cookies: Union[RequestsCookieJar, str, dict]) -> None: ... - - def cookies(self, cookies: Union[RequestsCookieJar, list, tuple, str, dict]) -> None: ... - def headers(self, headers: dict) -> None: ... def auto_handle_alert(self, on_off: bool = True, accept: bool = True) -> None: ... @@ -93,6 +92,10 @@ class ChromiumPageSetter(TabSetter): class SessionPageSetter(BasePageSetter): def __init__(self, page: SessionPage): self._page: SessionPage = ... + self._cookies_setter: SessionCookiesSetter = ... + + @property + def cookies(self) -> SessionCookiesSetter: ... def retry_times(self, times: int) -> None: ... @@ -104,10 +107,6 @@ class SessionPageSetter(BasePageSetter): def encoding(self, encoding: Optional[str, None], set_all: bool = True) -> None: ... - def cookie(self, cookie: Union[Cookie, str, dict]) -> None: ... - - def cookies(self, cookies: Union[RequestsCookieJar, list, tuple, str, dict]) -> None: ... - def headers(self, headers: dict) -> None: ... def header(self, attr: str, value: str) -> None: ... diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index f1744f9..8294fab 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -61,18 +61,14 @@ class ElementStates(object): @property def is_covered(self): - """返回元素是否被覆盖，与是否在视口中无关""" + """返回元素是否被覆盖，与是否在视口中无关，如被覆盖返回覆盖元素的backend id，否则返回False""" lx, ly = self._ele.rect.click_point try: - r = self._ele.page.run_cdp('DOM.getNodeForLocation', x=lx, y=ly) + bid = self._ele.page.run_cdp('DOM.getNodeForLocation', x=lx, y=ly).get('backendNodeId') + return bid if bid != self._ele._backend_id else False except CDPError: return False - if r.get('backendNodeId') != self._ele._backend_id: - return True - - return False - @property def has_rect(self): """返回元素是否拥有位置和大小，没有返回False，有返回四个角在页面中坐标组成的列表""" diff --git a/DrissionPage/_units/states.pyi b/DrissionPage/_units/states.pyi index fa8888c..dcd57e5 100644 --- a/DrissionPage/_units/states.pyi +++ b/DrissionPage/_units/states.pyi @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union, Tuple, List, Optional +from typing import Union, Tuple, List, Optional, Literal from .._elements.chromium_element import ShadowRoot, ChromiumElement from .._pages.chromium_base import ChromiumBase @@ -36,7 +36,7 @@ class ElementStates(object): def is_whole_in_viewport(self) -> bool: ... @property - def is_covered(self) -> bool: ... + def is_covered(self) -> Union[Literal[False], int]: ... @property def has_rect(self) -> Union[bool, List[Tuple[float, float]]]: ... From 13cb1d7f53b4e8726b3f2298f93f04877381c679 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 18 Dec 2023 01:07:11 +0800 Subject: [PATCH 139/182] =?UTF-8?q?=E5=AE=8C=E5=96=84=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E9=98=B6=E6=AE=B5=E9=80=BB=E8=BE=91=E5=92=8Calert=E5=A4=84?= =?UTF-8?q?=E7=90=86=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/driver.py | 45 ++++++++++++++------------- DrissionPage/_base/driver.pyi | 3 +- DrissionPage/_functions/tools.py | 3 +- DrissionPage/_pages/chromium_base.py | 20 +++++++----- DrissionPage/_pages/chromium_base.pyi | 3 ++ DrissionPage/_units/clicker.py | 4 +-- 6 files changed, 46 insertions(+), 32 deletions(-) diff --git a/DrissionPage/_base/driver.py b/DrissionPage/_base/driver.py index 77b1329..3b94f88 100644 --- a/DrissionPage/_base/driver.py +++ b/DrissionPage/_base/driver.py @@ -37,6 +37,7 @@ class Driver(object): self._stopped = Event() self.event_handlers = {} + self.immediate_event_handlers = {} self.method_results = {} self.event_queue = Queue() @@ -63,12 +64,14 @@ class Driver(object): print(f'发> {message_json}') break - if timeout is not None: - timeout = perf_counter() + timeout - + end_time = perf_counter() + timeout if timeout is not None else None self.method_results[ws_id] = Queue() try: self._ws.send(message_json) + if timeout == 0: + self.method_results.pop(ws_id, None) + return {'id': ws_id, 'result': {}} + except (OSError, WebSocketConnectionClosedException): self.method_results.pop(ws_id, None) return None @@ -82,14 +85,12 @@ class Driver(object): except Empty: if self.alert_flag: self.alert_flag = False - result = {'result': {'message': 'alert exists.'}} self.method_results.pop(ws_id, None) - return result + return {'result': {'message': 'alert exists.'}} - elif timeout is not None and perf_counter() > timeout: - result = {'error': {'message': 'timeout'}} + elif timeout is not None and perf_counter() > end_time: self.method_results.pop(ws_id, None) - return result + return {'error': {'message': 'timeout'}} continue @@ -119,8 +120,12 @@ class Driver(object): if 'method' in msg: if msg['method'].startswith('Page.javascriptDialog'): self.alert_flag = msg['method'].endswith('Opening') - - self.event_queue.put(msg) + if msg['method'] in self.immediate_event_handlers: + function = self.immediate_event_handlers.get(msg['method']) + if function: + function(**msg['params']) + else: + self.event_queue.put(msg) elif msg.get('id') in self.method_results: self.method_results[msg['id']].put(msg) @@ -138,11 +143,7 @@ class Driver(object): function = self.event_handlers.get(event['method']) if function: - # if self._debug: - # print(f'开始执行 {function.__name__}') function(**event['params']) - # if self._debug: - # print(f'执行 {function.__name__}完毕') self.event_queue.task_done() @@ -156,8 +157,8 @@ class Driver(object): if self._stopped.is_set(): return {'error': 'tab closed', 'type': 'tab_closed'} - timeout = kwargs.pop("_timeout", 20) - result = self._send({"method": _method, "params": kwargs}, timeout=timeout) + timeout = kwargs.pop('_timeout', 20) + result = self._send({'method': _method, 'params': kwargs}, timeout=timeout) if result is None: return {'error': 'tab closed', 'type': 'tab_closed'} if 'result' not in result and 'error' in result: @@ -199,19 +200,21 @@ class Driver(object): self.event_queue.queue.clear() return True - def set_callback(self, event, callback): + def set_callback(self, event, callback, immediate=False): """绑定cdp event和回调方法 :param event: cdp event :param callback: 绑定到cdp event的回调方法 + :param immediate: 是否要立即处理的动作 :return: None """ + handler = self.immediate_event_handlers if immediate else self.event_handlers if callback: - self.event_handlers[event] = callback + handler[event] = callback else: - self.event_handlers.pop(event, None) + handler.pop(event, None) def __str__(self): - return f"" + return f'' __repr__ = __str__ @@ -233,7 +236,7 @@ class BrowserDriver(Driver): self.browser = browser def __repr__(self): - return f"" + return f'' def get(self, url): r = get(url, headers={'Connection': 'close'}) diff --git a/DrissionPage/_base/driver.pyi b/DrissionPage/_base/driver.pyi index e2833da..6ea93b4 100644 --- a/DrissionPage/_base/driver.pyi +++ b/DrissionPage/_base/driver.pyi @@ -34,6 +34,7 @@ class Driver(object): _handle_event_th: Thread _stopped: Event event_handlers: dict + immediate_event_handlers: dict method_results: dict event_queue: Queue @@ -53,7 +54,7 @@ class Driver(object): def stop(self) -> bool: ... - def set_callback(self, event: str, callback: Union[Callable, None]) -> None: ... + def set_callback(self, event: str, callback: Union[Callable, None], immediate: bool = False) -> None: ... def __str__(self) -> str: ... diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index d5e140f..ead86eb 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -269,7 +269,8 @@ def raise_error(r): elif error == ('tab closed', 'No target with given id found'): raise PageClosedError elif error == 'timeout': - raise TimeoutError + return + # raise TimeoutError elif error == 'alert exists.': raise AlertExistsError elif error in ('Node does not have a layout object', 'Could not compute box model.'): diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index e0d2a3e..554ad32 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -123,7 +123,7 @@ class ChromiumBase(BasePage): self._driver = self.browser._get_driver(tab_id) self._alert = Alert() - self._driver.set_callback('Page.javascriptDialogOpening', self._on_alert_open) + self._driver.set_callback('Page.javascriptDialogOpening', self._on_alert_open, immediate=True) self._driver.set_callback('Page.javascriptDialogClosed', self._on_alert_close) self._driver.run('DOM.enable') @@ -837,6 +837,12 @@ class ChromiumBase(BasePage): self.run_cdp_loaded('Network.clearBrowserCookies') def handle_alert(self, accept=True, send=None, timeout=None, next_one=False): + r = self._handle_alert(accept=accept, send=send, timeout=timeout, next_one=next_one) + while self._has_alert: + sleep(.1) + return r + + def _handle_alert(self, accept=True, send=None, timeout=None, next_one=False): """处理提示框，可以自动等待提示框出现 :param accept: True表示确认，False表示取消，其它值不会按按钮但依然返回文本值 :param send: 处理prompt提示框时可输入文本 @@ -858,10 +864,10 @@ class ChromiumBase(BasePage): return False res_text = self._alert.text - if self._alert.type == 'prompt': - self.driver.run('Page.handleJavaScriptDialog', accept=accept, promptText=send) - else: - self.driver.run('Page.handleJavaScriptDialog', accept=accept) + d = {'accept': accept, '_timeout': 0} + if self._alert.type == 'prompt' and send is not None: + d['promptText'] = send + self.driver.run('Page.handleJavaScriptDialog', **d) return res_text def _on_alert_open(self, **kwargs): @@ -875,9 +881,9 @@ class ChromiumBase(BasePage): self._has_alert = True if self._alert.auto is not None: - self.handle_alert(self._alert.auto) + self._handle_alert(self._alert.auto) elif self._alert.handle_next is not None: - self.handle_alert(self._alert.handle_next, self._alert.next_text) + self._handle_alert(self._alert.handle_next, self._alert.next_text) self._alert.handle_next = None def _on_alert_close(self, **kwargs): diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index db025c0..39a6ac5 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -227,6 +227,9 @@ class ChromiumBase(BasePage): def handle_alert(self, accept: bool = True, send: str = None, timeout: float = None, next_one: bool = False) -> Union[str, False]: ... + def _handle_alert(self, accept: bool = True, send: str = None, timeout: float = None, + next_one: bool = False) -> Union[str, False]: ... + def _on_alert_close(self, **kwargs): ... def _on_alert_open(self, **kwargs): ... diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index 27d6719..42f48fe 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -147,10 +147,10 @@ class Clicker(object): :return: None """ self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mousePressed', - x=client_x, y=client_y, button=button, clickCount=count) + x=client_x, y=client_y, button=button, clickCount=count, _timeout=0.3) # sleep(.05) self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mouseReleased', - x=client_x, y=client_y, button=button) + x=client_x, y=client_y, button=button, _timeout=0.2) # -------------即将废弃-------------- From 8e3e0750cedb707a2b6bec69e567c668adc57d58 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 18 Dec 2023 18:51:58 +0800 Subject: [PATCH 140/182] =?UTF-8?q?=E7=AD=89=E5=BE=85=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E4=BF=AE=E6=94=B9(+)=20ele=5Fdelete()=E6=94=B9=E4=B8=BAele=5Fd?= =?UTF-8?q?eleted()=E3=80=81ele=5Fload()=E6=94=B9=E4=B8=BAele=5Floaded()?= =?UTF-8?q?=E3=80=81delete()=E6=94=B9=E4=B8=BAdeleted()=E3=80=81disabled?= =?UTF-8?q?=5For=5Fdelete()=E6=94=B9=E4=B8=BAdisabled=5For=5Fdeleted()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_units/waiter.py | 10 +++++----- DrissionPage/_units/waiter.pyi | 11 +++++------ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index b27328d..e02ca44 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -19,7 +19,7 @@ class BaseWaiter(object): """ sleep(second) - def ele_delete(self, loc_or_ele, timeout=None, raise_err=None): + def ele_deleted(self, loc_or_ele, timeout=None, raise_err=None): """等待元素从DOM中删除 :param loc_or_ele: 要等待的元素，可以是已有元素、定位符 :param timeout: 超时时间，默认读取页面超时时间 @@ -27,7 +27,7 @@ class BaseWaiter(object): :return: 是否等待成功 """ ele = self._driver._ele(loc_or_ele, raise_err=False, timeout=0) - return ele.wait.delete(timeout, raise_err=raise_err) if ele else True + return ele.wait.deleted(timeout, raise_err=raise_err) if ele else True def ele_display(self, loc_or_ele, timeout=None, raise_err=None): """等待元素变成显示状态 @@ -49,7 +49,7 @@ class BaseWaiter(object): ele = self._driver._ele(loc_or_ele, raise_err=False, timeout=0) return ele.wait.hidden(timeout, raise_err=raise_err) - def ele_load(self, loc, timeout=None, raise_err=None): + def ele_loaded(self, loc, timeout=None, raise_err=None): """等待元素加载到DOM :param loc: 要等待的元素，输入定位符 :param timeout: 超时时间，默认读取页面超时时间 @@ -290,7 +290,7 @@ class ElementWaiter(object): """ sleep(second) - def delete(self, timeout=None, raise_err=None): + def deleted(self, timeout=None, raise_err=None): """等待元素从dom删除 :param timeout: 超时时间，为None使用元素所在页面timeout属性 :param raise_err: 等待失败时是否报错，为None时根据Settings设置 @@ -346,7 +346,7 @@ class ElementWaiter(object): """ return self._wait_state('is_enabled', False, timeout, raise_err) - def disabled_or_delete(self, timeout=None, raise_err=None): + def disabled_or_deleted(self, timeout=None, raise_err=None): """等待当前元素变成不可用或从DOM移除 :param timeout: 超时时间，为None使用元素所在页面timeout属性 :param raise_err: 等待失败时是否报错，为None时根据Settings设置 diff --git a/DrissionPage/_units/waiter.pyi b/DrissionPage/_units/waiter.pyi index f8634b1..f41d2ad 100644 --- a/DrissionPage/_units/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -3,10 +3,9 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union, List +from typing import Union from .downloader import DownloadMission -from .listener import DataPacket from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame @@ -19,7 +18,7 @@ class BaseWaiter(object): def __call__(self, second: float) -> None: ... - def ele_delete(self, loc_or_ele: Union[str, tuple, ChromiumElement], timeout: float = None, + def ele_deleted(self, loc_or_ele: Union[str, tuple, ChromiumElement], timeout: float = None, raise_err: bool = None) -> bool: ... def ele_display(self, loc_or_ele: Union[str, tuple, ChromiumElement], timeout: float = None, @@ -28,7 +27,7 @@ class BaseWaiter(object): def ele_hidden(self, loc_or_ele: Union[str, tuple, ChromiumElement], timeout: float = None, raise_err: bool = None) -> bool: ... - def ele_load(self, loc: Union[str, tuple], timeout: float = None, + def ele_loaded(self, loc: Union[str, tuple], timeout: float = None, raise_err: bool = None) -> Union[bool, ChromiumElement]: ... def _loading(self, timeout: float = None, start: bool = True, gap: float = .01, raise_err: bool = None) -> bool: ... @@ -73,7 +72,7 @@ class ElementWaiter(object): def __call__(self, second: float) -> None: ... - def delete(self, timeout: float = None, raise_err: bool = None) -> bool: ... + def deleted(self, timeout: float = None, raise_err: bool = None) -> bool: ... def display(self, timeout: float = None, raise_err: bool = None) -> bool: ... @@ -87,7 +86,7 @@ class ElementWaiter(object): def disabled(self, timeout: float = None, raise_err: bool = None) -> bool: ... - def disabled_or_delete(self, timeout: float = None, raise_err: bool = None) -> bool: ... + def disabled_or_deleted(self, timeout: float = None, raise_err: bool = None) -> bool: ... def stop_moving(self, gap: float = .1, timeout: float = None, raise_err: bool = None) -> bool: ... From 79c6eae2db6cefa80e125040cf10fe143dfb9e3f Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 18 Dec 2023 20:24:38 +0800 Subject: [PATCH 141/182] =?UTF-8?q?4.0.0b23=E4=BC=98=E5=8C=96=E5=A4=84?= =?UTF-8?q?=E7=90=86alert=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_base/driver.py | 15 +++++---- DrissionPage/_elements/chromium_element.py | 3 +- DrissionPage/_functions/keys.py | 10 +++--- DrissionPage/_functions/tools.py | 39 ++++++++++++---------- DrissionPage/_functions/tools.pyi | 2 +- DrissionPage/_pages/chromium_base.py | 3 +- DrissionPage/_units/actions.py | 13 +++++--- DrissionPage/_units/clicker.py | 10 +++--- setup.py | 2 +- 10 files changed, 57 insertions(+), 42 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index c4cb7e4..4ffc37d 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b22' +__version__ = '4.0.0b23' diff --git a/DrissionPage/_base/driver.py b/DrissionPage/_base/driver.py index 3b94f88..c9644ea 100644 --- a/DrissionPage/_base/driver.py +++ b/DrissionPage/_base/driver.py @@ -83,14 +83,15 @@ class Driver(object): return result except Empty: - if self.alert_flag: - self.alert_flag = False - self.method_results.pop(ws_id, None) - return {'result': {'message': 'alert exists.'}} + # if self.alert_flag: + # self.alert_flag = False + # self.method_results.pop(ws_id, None) + # return {'result': {'message': 'alert exists.'}} - elif timeout is not None and perf_counter() > end_time: + if timeout is not None and perf_counter() > end_time: self.method_results.pop(ws_id, None) - return {'error': {'message': 'timeout'}} + return {'error': {'message': 'alert exists.'}} \ + if self.alert_flag else {'error': {'message': 'timeout'}} continue @@ -157,7 +158,7 @@ class Driver(object): if self._stopped.is_set(): return {'error': 'tab closed', 'type': 'tab_closed'} - timeout = kwargs.pop('_timeout', 20) + timeout = kwargs.pop('_timeout', 10) result = self._send({'method': _method, 'params': kwargs}, timeout=timeout) if result is None: return {'error': 'tab closed', 'type': 'tab_closed'} diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index c12333c..33d2cc8 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -641,7 +641,8 @@ class ChromiumElement(DrissionElement): """ self.page.scroll.to_see(self) x, y = offset_scroll(self, offset_x, offset_y) - self.page.run_cdp('Input.dispatchMouseEvent', type='mouseMoved', x=x, y=y) + self.page.run_cdp('Input.dispatchMouseEvent', type='mouseMoved', x=x, y=y, + _ignore=AlertExistsError, _timeout=0.3) def drag(self, offset_x=0, offset_y=0, duration=.5): """拖拽当前元素到相对位置 diff --git a/DrissionPage/_functions/keys.py b/DrissionPage/_functions/keys.py index 626470b..88b5fd3 100644 --- a/DrissionPage/_functions/keys.py +++ b/DrissionPage/_functions/keys.py @@ -3,6 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ +from ..errors import AlertExistsError class Keys: @@ -403,7 +404,7 @@ def keyDescriptionForString(_modifiers, keyString): # noqa: C901 def send_key(page, modifier, key): """发送一个字，在键盘中的字符触发按键，其它直接发送文本""" if key not in keyDefinitions: - page.run_cdp('Input.insertText', text=key) + page.run_cdp('Input.insertText', text=key, _ignore=AlertExistsError, _timeout=0.3) else: description = keyDescriptionForString(modifier, key) @@ -417,7 +418,8 @@ def send_key(page, modifier, key): 'autoRepeat': False, 'unmodifiedText': text, 'location': description['location'], - 'isKeypad': description['location'] == 3} + 'isKeypad': description['location'] == 3, + '_ignore': AlertExistsError, '_timeout': 0.3} page.run_cdp('Input.dispatchKeyEvent', **data) data['type'] = 'keyUp' @@ -440,7 +442,7 @@ def input_text_or_keys(page, text_or_keys): return if text_or_keys.endswith(('\n', '\ue007')): - page.run_cdp('Input.insertText', text=text_or_keys[:-1]) + page.run_cdp('Input.insertText', text=text_or_keys[:-1], _ignore=AlertExistsError, _timeout=0.3) send_key(page, modifier, '\n') else: - page.run_cdp('Input.insertText', text=text_or_keys) + page.run_cdp('Input.insertText', text=text_or_keys, _ignore=AlertExistsError, _timeout=0.3) diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index ead86eb..f0cc963 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -254,35 +254,40 @@ def configs_to_here(save_name=None): om.save(save_name) -def raise_error(r): +def raise_error(result, ignore=None): """抛出error对应报错 - :param r: 包含error的dict + :param result: 包含error的dict + :param ignore: 要忽略的错误 :return: None """ - error = r['error'] + error = result['error'] if error in ('Cannot find context with specified id', 'Inspected target navigated or closed'): - raise ContextLostError + r = ContextLostError() elif error in ('Could not find node with given id', 'Could not find object with given id', 'No node with given id found', 'Node with given id does not belong to the document', 'No node found for given backend id'): - raise ElementLostError + r = ElementLostError() elif error == ('tab closed', 'No target with given id found'): - raise PageClosedError + r = PageClosedError() elif error == 'timeout': - return - # raise TimeoutError + r = TimeoutError(f'超时。\n错误：{result["error"]}\nmethod：{result["method"]}\nargs：{result["args"]}\n' + f'出现这个错误可能意味着程序有bug，请把错误信息和重现方法告知作者，谢谢。\n' + '报告网站：https://gitee.com/g1879/DrissionPage/issues') elif error == 'alert exists.': - raise AlertExistsError + r = AlertExistsError() elif error in ('Node does not have a layout object', 'Could not compute box model.'): - raise NoRectError + r = NoRectError() elif error == 'Cannot navigate to invalid URL': - raise WrongURLError(f'无效的url：{r["args"]["url"]}。也许要加上"http://"？') + r = WrongURLError(f'无效的url：{result["args"]["url"]}。也许要加上"http://"？') elif error == 'Frame corresponds to an opaque origin and its storage key cannot be serialized': - raise StorageError + r = StorageError() elif error == 'Sanitizing cookie failed': - raise CookieFormatError(f'cookie格式不正确：{r["args"]}') - elif r['type'] == 'call_method_error': - raise CDPError(f'\n错误：{r["error"]}\nmethod：{r["method"]}\nargs：{r["args"]}\n出现这个错误可能意味着程序有bug，' - '请把错误信息和重现方法告知作者，谢谢。\n报告网站：https://gitee.com/g1879/DrissionPage/issues') + r = CookieFormatError(f'cookie格式不正确：{result["args"]}') + elif result['type'] == 'call_method_error': + r = CDPError(f'\n错误：{result["error"]}\nmethod：{result["method"]}\nargs：{result["args"]}\n出现这个错误可能意味着程序有bug，' + '请把错误信息和重现方法告知作者，谢谢。\n报告网站：https://gitee.com/g1879/DrissionPage/issues') else: - raise RuntimeError(r) + r = RuntimeError(result) + + if not ignore or not isinstance(r, ignore): + raise r diff --git a/DrissionPage/_functions/tools.pyi b/DrissionPage/_functions/tools.pyi index 784265b..b516919 100644 --- a/DrissionPage/_functions/tools.pyi +++ b/DrissionPage/_functions/tools.pyi @@ -44,4 +44,4 @@ def stop_process_on_port(port: Union[int, str]) -> None: ... def configs_to_here(file_name: Union[Path, str] = None) -> None: ... -def raise_error(r: dict) -> None: ... +def raise_error(result: dict, ignore=None) -> None: ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 554ad32..86821db 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -449,8 +449,9 @@ class ChromiumBase(BasePage): :param cmd_args: 参数 :return: 执行的结果 """ + ignore = cmd_args.pop('_ignore', None) r = self.driver.run(cmd, **cmd_args) - return r if __ERROR__ not in r else raise_error(r) + return r if __ERROR__ not in r else raise_error(r, ignore) def run_cdp_loaded(self, cmd, **cmd_args): """执行Chrome DevTools Protocol语句，执行前等待页面加载完毕 diff --git a/DrissionPage/_units/actions.py b/DrissionPage/_units/actions.py index e933913..f8248f4 100644 --- a/DrissionPage/_units/actions.py +++ b/DrissionPage/_units/actions.py @@ -5,6 +5,7 @@ """ from time import sleep, perf_counter +from ..errors import AlertExistsError from .._functions.keys import modifierBit, keyDescriptionForString, input_text_or_keys, Keys from .._functions.web import location_in_viewport @@ -84,7 +85,7 @@ class Actions: self.curr_x = x self.curr_y = y self._dr.run('Input.dispatchMouseEvent', type='mouseMoved', x=self.curr_x, y=self.curr_y, - modifiers=self.modifier) + modifiers=self.modifier, _ignore=AlertExistsError, _timeout=0.3) ss = .02 - perf_counter() + t if ss > 0: sleep(ss) @@ -187,7 +188,7 @@ class Actions: if on_ele: self.move_to(on_ele, duration=0) self._dr.run('Input.dispatchMouseEvent', type='mousePressed', button=button, clickCount=count, - x=self.curr_x, y=self.curr_y, modifiers=self.modifier) + x=self.curr_x, y=self.curr_y, modifiers=self.modifier, _ignore=AlertExistsError, _timeout=0.3) return self def _release(self, button): @@ -196,7 +197,7 @@ class Actions: :return: self """ self._dr.run('Input.dispatchMouseEvent', type='mouseReleased', button=button, clickCount=1, - x=self.curr_x, y=self.curr_y, modifiers=self.modifier) + x=self.curr_x, y=self.curr_y, modifiers=self.modifier, _ignore=AlertExistsError, _timeout=0.3) return self def scroll(self, delta_x=0, delta_y=0, on_ele=None): @@ -209,7 +210,7 @@ class Actions: if on_ele: self.move_to(on_ele, duration=0) self._dr.run('Input.dispatchMouseEvent', type='mouseWheel', x=self.curr_x, y=self.curr_y, - deltaX=delta_x, deltaY=delta_y, modifiers=self.modifier) + deltaX=delta_x, deltaY=delta_y, modifiers=self.modifier, _ignore=AlertExistsError, _timeout=0.3) return self def up(self, pixel): @@ -251,6 +252,8 @@ class Actions: return self data = self._get_key_data(key, 'keyDown') + data['_timeout'] = .3 + data['_ignore'] = AlertExistsError self.page.run_cdp('Input.dispatchKeyEvent', **data) return self @@ -265,6 +268,8 @@ class Actions: return self data = self._get_key_data(key, 'keyUp') + data['_timeout'] = .3 + data['_ignore'] = AlertExistsError self.page.run_cdp('Input.dispatchKeyEvent', **data) return self diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index 42f48fe..a6c55a0 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -7,7 +7,7 @@ from time import perf_counter, sleep from .._functions.settings import Settings from .._functions.web import offset_scroll -from ..errors import CanNotClickError, CDPError, NoRectError +from ..errors import CanNotClickError, CDPError, NoRectError, AlertExistsError class Clicker(object): @@ -146,11 +146,11 @@ class Clicker(object): :param count: 点击次数 :return: None """ - self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mousePressed', - x=client_x, y=client_y, button=button, clickCount=count, _timeout=0.3) + self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mousePressed', x=client_x, + y=client_y, button=button, clickCount=count, _ignore=AlertExistsError, _timeout=0.3) # sleep(.05) - self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mouseReleased', - x=client_x, y=client_y, button=button, _timeout=0.2) + self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mouseReleased', x=client_x, + y=client_y, button=button, _ignore=AlertExistsError, _timeout=0.2) # -------------即将废弃-------------- diff --git a/setup.py b/setup.py index 9df76dc..5a0d122 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b22", + version="4.0.0b23", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From 774c6dd963141dbfff7fb6bec330b25ee099d509 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 19 Dec 2023 14:18:24 +0800 Subject: [PATCH 142/182] =?UTF-8?q?4.0.0b23=E5=BE=AE=E8=B0=83=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 2 +- DrissionPage/_functions/keys.py | 8 ++++---- DrissionPage/_units/actions.py | 12 ++++++------ DrissionPage/_units/clicker.py | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 33d2cc8..db64ee5 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -642,7 +642,7 @@ class ChromiumElement(DrissionElement): self.page.scroll.to_see(self) x, y = offset_scroll(self, offset_x, offset_y) self.page.run_cdp('Input.dispatchMouseEvent', type='mouseMoved', x=x, y=y, - _ignore=AlertExistsError, _timeout=0.3) + _ignore=AlertExistsError, _timeout=1) def drag(self, offset_x=0, offset_y=0, duration=.5): """拖拽当前元素到相对位置 diff --git a/DrissionPage/_functions/keys.py b/DrissionPage/_functions/keys.py index 88b5fd3..9ff2637 100644 --- a/DrissionPage/_functions/keys.py +++ b/DrissionPage/_functions/keys.py @@ -404,7 +404,7 @@ def keyDescriptionForString(_modifiers, keyString): # noqa: C901 def send_key(page, modifier, key): """发送一个字，在键盘中的字符触发按键，其它直接发送文本""" if key not in keyDefinitions: - page.run_cdp('Input.insertText', text=key, _ignore=AlertExistsError, _timeout=0.3) + page.run_cdp('Input.insertText', text=key, _ignore=AlertExistsError, _timeout=1) else: description = keyDescriptionForString(modifier, key) @@ -419,7 +419,7 @@ def send_key(page, modifier, key): 'unmodifiedText': text, 'location': description['location'], 'isKeypad': description['location'] == 3, - '_ignore': AlertExistsError, '_timeout': 0.3} + '_ignore': AlertExistsError, '_timeout': 1} page.run_cdp('Input.dispatchKeyEvent', **data) data['type'] = 'keyUp' @@ -442,7 +442,7 @@ def input_text_or_keys(page, text_or_keys): return if text_or_keys.endswith(('\n', '\ue007')): - page.run_cdp('Input.insertText', text=text_or_keys[:-1], _ignore=AlertExistsError, _timeout=0.3) + page.run_cdp('Input.insertText', text=text_or_keys[:-1], _ignore=AlertExistsError, _timeout=1) send_key(page, modifier, '\n') else: - page.run_cdp('Input.insertText', text=text_or_keys, _ignore=AlertExistsError, _timeout=0.3) + page.run_cdp('Input.insertText', text=text_or_keys, _ignore=AlertExistsError, _timeout=1) diff --git a/DrissionPage/_units/actions.py b/DrissionPage/_units/actions.py index f8248f4..f9da1bd 100644 --- a/DrissionPage/_units/actions.py +++ b/DrissionPage/_units/actions.py @@ -85,7 +85,7 @@ class Actions: self.curr_x = x self.curr_y = y self._dr.run('Input.dispatchMouseEvent', type='mouseMoved', x=self.curr_x, y=self.curr_y, - modifiers=self.modifier, _ignore=AlertExistsError, _timeout=0.3) + modifiers=self.modifier, _ignore=AlertExistsError, _timeout=1) ss = .02 - perf_counter() + t if ss > 0: sleep(ss) @@ -188,7 +188,7 @@ class Actions: if on_ele: self.move_to(on_ele, duration=0) self._dr.run('Input.dispatchMouseEvent', type='mousePressed', button=button, clickCount=count, - x=self.curr_x, y=self.curr_y, modifiers=self.modifier, _ignore=AlertExistsError, _timeout=0.3) + x=self.curr_x, y=self.curr_y, modifiers=self.modifier, _ignore=AlertExistsError, _timeout=1) return self def _release(self, button): @@ -197,7 +197,7 @@ class Actions: :return: self """ self._dr.run('Input.dispatchMouseEvent', type='mouseReleased', button=button, clickCount=1, - x=self.curr_x, y=self.curr_y, modifiers=self.modifier, _ignore=AlertExistsError, _timeout=0.3) + x=self.curr_x, y=self.curr_y, modifiers=self.modifier, _ignore=AlertExistsError, _timeout=1) return self def scroll(self, delta_x=0, delta_y=0, on_ele=None): @@ -210,7 +210,7 @@ class Actions: if on_ele: self.move_to(on_ele, duration=0) self._dr.run('Input.dispatchMouseEvent', type='mouseWheel', x=self.curr_x, y=self.curr_y, - deltaX=delta_x, deltaY=delta_y, modifiers=self.modifier, _ignore=AlertExistsError, _timeout=0.3) + deltaX=delta_x, deltaY=delta_y, modifiers=self.modifier, _ignore=AlertExistsError, _timeout=1) return self def up(self, pixel): @@ -252,7 +252,7 @@ class Actions: return self data = self._get_key_data(key, 'keyDown') - data['_timeout'] = .3 + data['_timeout'] = 1 data['_ignore'] = AlertExistsError self.page.run_cdp('Input.dispatchKeyEvent', **data) return self @@ -268,7 +268,7 @@ class Actions: return self data = self._get_key_data(key, 'keyUp') - data['_timeout'] = .3 + data['_timeout'] = 1 data['_ignore'] = AlertExistsError self.page.run_cdp('Input.dispatchKeyEvent', **data) return self diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index a6c55a0..6d98dd4 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -147,10 +147,10 @@ class Clicker(object): :return: None """ self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mousePressed', x=client_x, - y=client_y, button=button, clickCount=count, _ignore=AlertExistsError, _timeout=0.3) + y=client_y, button=button, clickCount=count, _ignore=AlertExistsError, _timeout=.5) # sleep(.05) self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mouseReleased', x=client_x, - y=client_y, button=button, _ignore=AlertExistsError, _timeout=0.2) + y=client_y, button=button, _ignore=AlertExistsError, _timeout=.5) # -------------即将废弃-------------- From b1f4c0767d177117255269382a5f8d03868a0a38 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 19 Dec 2023 21:29:04 +0800 Subject: [PATCH 143/182] =?UTF-8?q?4.0.0b24=E4=BF=AE=E5=A4=8D=E5=8A=A8?= =?UTF-8?q?=E4=BD=9C=E9=93=BE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_functions/tools.py | 7 ++++--- DrissionPage/_units/actions.py | 8 ++++---- setup.py | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 4ffc37d..e51d275 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b23' +__version__ = '4.0.0b24' diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index f0cc963..5665a69 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -211,7 +211,7 @@ def wait_until(page, condition, timeout=10, poll=0.1, raise_err=True): value = condition_method(page) if value: return value - except Exception as exc: + except Exception: pass sleep(poll) @@ -284,8 +284,9 @@ def raise_error(result, ignore=None): elif error == 'Sanitizing cookie failed': r = CookieFormatError(f'cookie格式不正确：{result["args"]}') elif result['type'] == 'call_method_error': - r = CDPError(f'\n错误：{result["error"]}\nmethod：{result["method"]}\nargs：{result["args"]}\n出现这个错误可能意味着程序有bug，' - '请把错误信息和重现方法告知作者，谢谢。\n报告网站：https://gitee.com/g1879/DrissionPage/issues') + r = CDPError(f'\n错误：{result["error"]}\nmethod：{result["method"]}\nargs：{result["args"]}\n' + f'出现这个错误可能意味着程序有bug，请把错误信息和重现方法告知作者，谢谢。' + f'\n报告网站：https://gitee.com/g1879/DrissionPage/issues') else: r = RuntimeError(result) diff --git a/DrissionPage/_units/actions.py b/DrissionPage/_units/actions.py index f9da1bd..c6a47ac 100644 --- a/DrissionPage/_units/actions.py +++ b/DrissionPage/_units/actions.py @@ -85,7 +85,7 @@ class Actions: self.curr_x = x self.curr_y = y self._dr.run('Input.dispatchMouseEvent', type='mouseMoved', x=self.curr_x, y=self.curr_y, - modifiers=self.modifier, _ignore=AlertExistsError, _timeout=1) + modifiers=self.modifier, _timeout=1) ss = .02 - perf_counter() + t if ss > 0: sleep(ss) @@ -188,7 +188,7 @@ class Actions: if on_ele: self.move_to(on_ele, duration=0) self._dr.run('Input.dispatchMouseEvent', type='mousePressed', button=button, clickCount=count, - x=self.curr_x, y=self.curr_y, modifiers=self.modifier, _ignore=AlertExistsError, _timeout=1) + x=self.curr_x, y=self.curr_y, modifiers=self.modifier, _timeout=1) return self def _release(self, button): @@ -197,7 +197,7 @@ class Actions: :return: self """ self._dr.run('Input.dispatchMouseEvent', type='mouseReleased', button=button, clickCount=1, - x=self.curr_x, y=self.curr_y, modifiers=self.modifier, _ignore=AlertExistsError, _timeout=1) + x=self.curr_x, y=self.curr_y, modifiers=self.modifier, _timeout=1) return self def scroll(self, delta_x=0, delta_y=0, on_ele=None): @@ -210,7 +210,7 @@ class Actions: if on_ele: self.move_to(on_ele, duration=0) self._dr.run('Input.dispatchMouseEvent', type='mouseWheel', x=self.curr_x, y=self.curr_y, - deltaX=delta_x, deltaY=delta_y, modifiers=self.modifier, _ignore=AlertExistsError, _timeout=1) + deltaX=delta_x, deltaY=delta_y, modifiers=self.modifier, _timeout=1) return self def up(self, pixel): diff --git a/setup.py b/setup.py index 5a0d122..99298bd 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b23", + version="4.0.0b24", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From e9f6322e5e550915c15afd118b014361c6e90105 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 20 Dec 2023 14:35:14 +0800 Subject: [PATCH 144/182] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=A4=84=E7=90=86ale?= =?UTF-8?q?rt=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/driver.py | 6 ++---- DrissionPage/_elements/chromium_element.py | 3 +-- DrissionPage/_functions/keys.py | 6 +++--- DrissionPage/_units/actions.py | 10 +++++----- DrissionPage/_units/clicker.py | 4 ++-- 5 files changed, 13 insertions(+), 16 deletions(-) diff --git a/DrissionPage/_base/driver.py b/DrissionPage/_base/driver.py index c9644ea..91a7465 100644 --- a/DrissionPage/_base/driver.py +++ b/DrissionPage/_base/driver.py @@ -83,10 +83,8 @@ class Driver(object): return result except Empty: - # if self.alert_flag: - # self.alert_flag = False - # self.method_results.pop(ws_id, None) - # return {'result': {'message': 'alert exists.'}} + if self.alert_flag and message['method'].startswith('Input.'): + return {'result': {'message': 'alert exists.'}} if timeout is not None and perf_counter() > end_time: self.method_results.pop(ws_id, None) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index db64ee5..bd98944 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -641,8 +641,7 @@ class ChromiumElement(DrissionElement): """ self.page.scroll.to_see(self) x, y = offset_scroll(self, offset_x, offset_y) - self.page.run_cdp('Input.dispatchMouseEvent', type='mouseMoved', x=x, y=y, - _ignore=AlertExistsError, _timeout=1) + self.page.run_cdp('Input.dispatchMouseEvent', type='mouseMoved', x=x, y=y, _ignore=AlertExistsError) def drag(self, offset_x=0, offset_y=0, duration=.5): """拖拽当前元素到相对位置 diff --git a/DrissionPage/_functions/keys.py b/DrissionPage/_functions/keys.py index 9ff2637..7e4a300 100644 --- a/DrissionPage/_functions/keys.py +++ b/DrissionPage/_functions/keys.py @@ -404,7 +404,7 @@ def keyDescriptionForString(_modifiers, keyString): # noqa: C901 def send_key(page, modifier, key): """发送一个字，在键盘中的字符触发按键，其它直接发送文本""" if key not in keyDefinitions: - page.run_cdp('Input.insertText', text=key, _ignore=AlertExistsError, _timeout=1) + page.run_cdp('Input.insertText', text=key, _ignore=AlertExistsError) else: description = keyDescriptionForString(modifier, key) @@ -442,7 +442,7 @@ def input_text_or_keys(page, text_or_keys): return if text_or_keys.endswith(('\n', '\ue007')): - page.run_cdp('Input.insertText', text=text_or_keys[:-1], _ignore=AlertExistsError, _timeout=1) + page.run_cdp('Input.insertText', text=text_or_keys[:-1], _ignore=AlertExistsError) send_key(page, modifier, '\n') else: - page.run_cdp('Input.insertText', text=text_or_keys, _ignore=AlertExistsError, _timeout=1) + page.run_cdp('Input.insertText', text=text_or_keys, _ignore=AlertExistsError) diff --git a/DrissionPage/_units/actions.py b/DrissionPage/_units/actions.py index c6a47ac..4f4b282 100644 --- a/DrissionPage/_units/actions.py +++ b/DrissionPage/_units/actions.py @@ -84,8 +84,8 @@ class Actions: t = perf_counter() self.curr_x = x self.curr_y = y - self._dr.run('Input.dispatchMouseEvent', type='mouseMoved', x=self.curr_x, y=self.curr_y, - modifiers=self.modifier, _timeout=1) + self._dr.run('Input.dispatchMouseEvent', type='mouseMoved', + x=self.curr_x, y=self.curr_y, modifiers=self.modifier) ss = .02 - perf_counter() + t if ss > 0: sleep(ss) @@ -188,7 +188,7 @@ class Actions: if on_ele: self.move_to(on_ele, duration=0) self._dr.run('Input.dispatchMouseEvent', type='mousePressed', button=button, clickCount=count, - x=self.curr_x, y=self.curr_y, modifiers=self.modifier, _timeout=1) + x=self.curr_x, y=self.curr_y, modifiers=self.modifier) return self def _release(self, button): @@ -197,7 +197,7 @@ class Actions: :return: self """ self._dr.run('Input.dispatchMouseEvent', type='mouseReleased', button=button, clickCount=1, - x=self.curr_x, y=self.curr_y, modifiers=self.modifier, _timeout=1) + x=self.curr_x, y=self.curr_y, modifiers=self.modifier) return self def scroll(self, delta_x=0, delta_y=0, on_ele=None): @@ -210,7 +210,7 @@ class Actions: if on_ele: self.move_to(on_ele, duration=0) self._dr.run('Input.dispatchMouseEvent', type='mouseWheel', x=self.curr_x, y=self.curr_y, - deltaX=delta_x, deltaY=delta_y, modifiers=self.modifier, _timeout=1) + deltaX=delta_x, deltaY=delta_y, modifiers=self.modifier) return self def up(self, pixel): diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index 6d98dd4..9a85215 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -147,10 +147,10 @@ class Clicker(object): :return: None """ self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mousePressed', x=client_x, - y=client_y, button=button, clickCount=count, _ignore=AlertExistsError, _timeout=.5) + y=client_y, button=button, clickCount=count, _ignore=AlertExistsError) # sleep(.05) self._ele.page.run_cdp('Input.dispatchMouseEvent', type='mouseReleased', x=client_x, - y=client_y, button=button, _ignore=AlertExistsError, _timeout=.5) + y=client_y, button=button, _ignore=AlertExistsError) # -------------即将废弃-------------- From 4adb8247fd654bab3c201fc54edb7e405d6524dc Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 20 Dec 2023 17:37:30 +0800 Subject: [PATCH 145/182] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B3=A8=E9=87=8A?= =?UTF-8?q?=EF=BC=8Cele()=E5=8F=AA=E8=BF=94=E5=9B=9E=E5=85=83=E7=B4=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 6 +- DrissionPage/_elements/chromium_element.pyi | 103 ++++++++++++-------- DrissionPage/_elements/session_element.pyi | 72 ++++++++------ DrissionPage/_pages/chromium_base.pyi | 10 +- DrissionPage/_pages/chromium_frame.pyi | 89 ++++++++++++----- DrissionPage/_pages/chromium_tab.pyi | 15 +-- DrissionPage/_pages/session_page.pyi | 26 +++-- DrissionPage/_pages/web_page.pyi | 36 ++++--- 8 files changed, 225 insertions(+), 132 deletions(-) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index bd98944..a99eaed 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -1149,7 +1149,7 @@ def find_by_xpath(ele, xpath, single, timeout, relative=True): return NoneElement(ele.page) if r['result']['subtype'] == 'null' \ else make_chromium_ele(ele.page, obj_id=r['result']['objectId']) - if r['result']['description'] == 'NodeList(0)': + if r['result']['description'] in ('NodeList(0)', 'Array(0)'): return [] else: r = ele.page.run_cdp_loaded('Runtime.getProperties', objectId=r['result']['objectId'], @@ -1176,7 +1176,7 @@ def find_by_css(ele, selector, single, timeout): end_time = perf_counter() + timeout while ('exceptionDetails' in r or r['result']['subtype'] == 'null' or - r['result']['description'] == 'NodeList(0)') and perf_counter() < end_time: + r['result']['description'] in ('NodeList(0)', 'Array(0)')) and perf_counter() < end_time: r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, returnByValue=False, awaitPromise=True, userGesture=True) @@ -1187,7 +1187,7 @@ def find_by_css(ele, selector, single, timeout): return NoneElement(ele.page) if r['result']['subtype'] == 'null' \ else make_chromium_ele(ele.page, obj_id=r['result']['objectId']) - if r['result']['description'] == 'NodeList(0)': + if r['result']['description'] in ('NodeList(0)', 'Array(0)'): return [] else: r = ele.page.run_cdp_loaded('Runtime.getProperties', objectId=r['result']['objectId'], diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index bb39b75..7dd3a52 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -45,7 +45,7 @@ class ChromiumElement(DrissionElement): def __repr__(self) -> str: ... def __call__(self, loc_or_str: Union[Tuple[str, str], str], - timeout: float = None) -> Union[ChromiumElement, str, None]: ... + timeout: float = None) -> Union[ChromiumElement, NoneElement]: ... @property def tag(self) -> str: ... @@ -91,50 +91,62 @@ class ChromiumElement(DrissionElement): @property def click(self) -> Clicker: ... - def parent(self, level_or_loc: Union[tuple, str, int] = 1, index: int = 1) -> Union[ChromiumElement, None]: ... + def parent(self, + level_or_loc: Union[tuple, str, int] = 1, + index: int = 1) -> Union[ChromiumElement, NoneElement]: ... - def child(self, filter_loc: Union[tuple, str, int] = '', + def child(self, + filter_loc: Union[tuple, str, int] = '', index: int = 1, timeout: float = 0, - ele_only: bool = True) -> Union[ChromiumElement, str, None]: ... + ele_only: bool = True) -> Union[ChromiumElement, str, NoneElement]: ... - def prev(self, filter_loc: Union[tuple, str, int] = '', + def prev(self, + filter_loc: Union[tuple, str, int] = '', index: int = 1, timeout: float = 0, - ele_only: bool = True) -> Union[ChromiumElement, str, None]: ... + ele_only: bool = True) -> Union[ChromiumElement, str, NoneElement]: ... - def next(self, filter_loc: Union[tuple, str, int] = '', + def next(self, + filter_loc: Union[tuple, str, int] = '', index: int = 1, timeout: float = 0, - ele_only: bool = True) -> Union[ChromiumElement, str, None]: ... + ele_only: bool = True) -> Union[ChromiumElement, str, NoneElement]: ... - def before(self, filter_loc: Union[tuple, str, int] = '', + def before(self, + filter_loc: Union[tuple, str, int] = '', index: int = 1, timeout: float = None, - ele_only: bool = True) -> Union[ChromiumElement, str, None]: ... + ele_only: bool = True) -> Union[ChromiumElement, str, NoneElement]: ... - def after(self, filter_loc: Union[tuple, str, int] = '', + def after(self, + filter_loc: Union[tuple, str, int] = '', index: int = 1, timeout: float = None, - ele_only: bool = True) -> Union[ChromiumElement, str, None]: ... + ele_only: bool = True) -> Union[ChromiumElement, str, NoneElement]: ... - def children(self, filter_loc: Union[tuple, str] = '', + def children(self, + filter_loc: Union[tuple, str] = '', timeout: float = 0, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def prevs(self, filter_loc: Union[tuple, str] = '', + def prevs(self, + filter_loc: Union[tuple, str] = '', timeout: float = 0, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def nexts(self, filter_loc: Union[tuple, str] = '', + def nexts(self, + filter_loc: Union[tuple, str] = '', timeout: float = 0, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def befores(self, filter_loc: Union[tuple, str] = '', + def befores(self, + filter_loc: Union[tuple, str] = '', timeout: float = None, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def afters(self, filter_loc: Union[tuple, str] = '', + def afters(self, + filter_loc: Union[tuple, str] = '', timeout: float = None, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... @@ -156,20 +168,25 @@ class ChromiumElement(DrissionElement): def run_async_js(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> None: ... - def ele(self, loc_or_str: Union[Tuple[str, str], str], - timeout: float = None) -> Union[ChromiumElement, str]: ... + def ele(self, + loc_or_str: Union[Tuple[str, str], str], + timeout: float = None) -> Union[ChromiumElement, NoneElement]: ... - def eles(self, loc_or_str: Union[Tuple[str, str], str], - timeout: float = None) -> List[Union[ChromiumElement, str]]: ... + def eles(self, + loc_or_str: Union[Tuple[str, str], str], + timeout: float = None) -> List[ChromiumElement]: ... - def s_ele(self, loc_or_str: Union[Tuple[str, str], str] = None) -> Union[SessionElement, str, NoneElement]: ... + def s_ele(self, loc_or_str: Union[Tuple[str, str], str] = None) -> Union[SessionElement, NoneElement]: ... - def s_eles(self, loc_or_str: Union[Tuple[str, str], str] = None) -> List[Union[SessionElement, str]]: ... + def s_eles(self, loc_or_str: Union[Tuple[str, str], str] = None) -> List[SessionElement]: ... - def _find_elements(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None, - single: bool = True, relative: bool = False, raise_err: bool = False) \ - -> Union[ChromiumElement, ChromiumFrame, str, NoneElement, - List[Union[ChromiumElement, ChromiumFrame, str]]]: ... + def _find_elements(self, + loc_or_str: Union[Tuple[str, str], str], + timeout: float = None, + single: bool = True, + relative: bool = False, + raise_err: bool = False) -> Union[ChromiumElement, ChromiumFrame, NoneElement, + List[Union[ChromiumElement, ChromiumFrame]]]: ... def style(self, style: str, pseudo_ele: str = '') -> str: ... @@ -177,8 +194,12 @@ class ChromiumElement(DrissionElement): def save(self, path: [str, bool] = None, name: str = None, timeout: float = None) -> str: ... - def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: PIC_TYPE = None, - as_base64: PIC_TYPE = None, scroll_to_center: bool = True) -> Union[str, bytes]: ... + def get_screenshot(self, + path: [str, Path] = None, + name: str = None, + as_bytes: PIC_TYPE = None, + as_base64: PIC_TYPE = None, + scroll_to_center: bool = True) -> Union[str, bytes]: ... def input(self, vals: Any, clear: bool = True, by_js: bool = False) -> None: ... @@ -239,34 +260,34 @@ class ShadowRoot(BaseElement): def parent(self, level_or_loc: Union[str, int] = 1, index: int = 1) -> ChromiumElement: ... def child(self, filter_loc: Union[tuple, str] = '', - index: int = 1) -> Union[ChromiumElement, str, None]: ... + index: int = 1) -> Union[ChromiumElement, NoneElement]: ... def next(self, filter_loc: Union[tuple, str] = '', - index: int = 1) -> Union[ChromiumElement, str, None]: ... + index: int = 1) -> Union[ChromiumElement, NoneElement]: ... def before(self, filter_loc: Union[tuple, str] = '', - index: int = 1) -> Union[ChromiumElement, str, None]: ... + index: int = 1) -> Union[ChromiumElement, NoneElement]: ... def after(self, filter_loc: Union[tuple, str] = '', - index: int = 1) -> Union[ChromiumElement, str, None]: ... + index: int = 1) -> Union[ChromiumElement, NoneElement]: ... - def children(self, filter_loc: Union[tuple, str] = '') -> List[Union[ChromiumElement, str]]: ... + def children(self, filter_loc: Union[tuple, str] = '') -> List[ChromiumElement]: ... - def nexts(self, filter_loc: Union[tuple, str] = '') -> List[Union[ChromiumElement, str]]: ... + def nexts(self, filter_loc: Union[tuple, str] = '') -> List[ChromiumElement]: ... - def befores(self, filter_loc: Union[tuple, str] = '') -> List[Union[ChromiumElement, str]]: ... + def befores(self, filter_loc: Union[tuple, str] = '') -> List[ChromiumElement]: ... - def afters(self, filter_loc: Union[tuple, str] = '') -> List[Union[ChromiumElement, str]]: ... + def afters(self, filter_loc: Union[tuple, str] = '') -> List[ChromiumElement]: ... def ele(self, loc_or_str: Union[Tuple[str, str], str], - timeout: float = None) -> Union[ChromiumElement]: ... + timeout: float = None) -> Union[ChromiumElement, NoneElement]: ... def eles(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> List[ChromiumElement]: ... - def s_ele(self, loc_or_str: Union[Tuple[str, str], str] = None) -> Union[SessionElement, str, NoneElement]: ... + def s_ele(self, loc_or_str: Union[Tuple[str, str], str] = None) -> Union[SessionElement, NoneElement]: ... - def s_eles(self, loc_or_str: Union[Tuple[str, str], str]) -> List[Union[SessionElement, str]]: ... + def s_eles(self, loc_or_str: Union[Tuple[str, str], str]) -> List[SessionElement]: ... def _find_elements(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None, single: bool = True, relative: bool = False, raise_err: bool = None) \ @@ -282,7 +303,7 @@ class ShadowRoot(BaseElement): def find_in_chromium_ele(ele: ChromiumElement, loc: Union[str, Tuple[str, str]], single: bool = True, timeout: float = None, relative: bool = True) \ - -> Union[ChromiumElement, str, NoneElement, List[Union[ChromiumElement, str]]]: ... + -> Union[ChromiumElement, NoneElement, List[ChromiumElement]]: ... def find_by_xpath(ele: ChromiumElement, xpath: str, single: bool, timeout: float, diff --git a/DrissionPage/_elements/session_element.pyi b/DrissionPage/_elements/session_element.pyi index 40a8909..190d06b 100644 --- a/DrissionPage/_elements/session_element.pyi +++ b/DrissionPage/_elements/session_element.pyi @@ -3,7 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ -from typing import Union, List, Tuple +from typing import Union, List, Tuple, Optional from lxml.html import HtmlElement @@ -28,7 +28,7 @@ class SessionElement(DrissionElement): def __call__(self, loc_or_str: Union[Tuple[str, str], str], - timeout: float = None) -> Union['SessionElement', str, None]: ... + timeout: float = None) -> Union[SessionElement, NoneElement]: ... @property def tag(self) -> str: ... @@ -48,68 +48,80 @@ class SessionElement(DrissionElement): @property def raw_text(self) -> str: ... - def parent(self, level_or_loc: Union[tuple, str, int] = 1, index: int = 1) -> Union['SessionElement', None]: ... + def parent(self, + level_or_loc: Union[tuple, str, int] = 1, + index: int = 1) -> Union[SessionElement, NoneElement]: ... - def child(self, filter_loc: Union[tuple, str, int] = '', + def child(self, + filter_loc: Union[tuple, str, int] = '', index: int = 1, timeout: float = None, - ele_only: bool = True) -> Union['SessionElement', str, None]: ... + ele_only: bool = True) -> Union[SessionElement, str, NoneElement]: ... - def prev(self, filter_loc: Union[tuple, str, int] = '', + def prev(self, + filter_loc: Union[tuple, str, int] = '', index: int = 1, timeout: float = None, - ele_only: bool = True) -> Union['SessionElement', str, None]: ... + ele_only: bool = True) -> Union[SessionElement, str, NoneElement]: ... - def next(self, filter_loc: Union[tuple, str, int] = '', + def next(self, + filter_loc: Union[tuple, str, int] = '', index: int = 1, timeout: float = None, - ele_only: bool = True) -> Union['SessionElement', str, None]: ... + ele_only: bool = True) -> Union[SessionElement, str, NoneElement]: ... - def before(self, filter_loc: Union[tuple, str, int] = '', + def before(self, + filter_loc: Union[tuple, str, int] = '', index: int = 1, timeout: float = None, - ele_only: bool = True) -> Union['SessionElement', str, None]: ... + ele_only: bool = True) -> Union[SessionElement, str, NoneElement]: ... - def after(self, filter_loc: Union[tuple, str, int] = '', + def after(self, + filter_loc: Union[tuple, str, int] = '', index: int = 1, timeout: float = None, - ele_only: bool = True) -> Union['SessionElement', str, None]: ... + ele_only: bool = True) -> Union[SessionElement, str, NoneElement]: ... - def children(self, filter_loc: Union[tuple, str] = '', + def children(self, + filter_loc: Union[tuple, str] = '', timeout: float = None, - ele_only: bool = True) -> List[Union['SessionElement', str]]: ... + ele_only: bool = True) -> List[Union[SessionElement, str]]: ... - def prevs(self, filter_loc: Union[tuple, str] = '', + def prevs(self, + filter_loc: Union[tuple, str] = '', timeout: float = None, - ele_only: bool = True) -> List[Union['SessionElement', str]]: ... + ele_only: bool = True) -> List[Union[SessionElement, str]]: ... - def nexts(self, filter_loc: Union[tuple, str] = '', + def nexts(self, + filter_loc: Union[tuple, str] = '', timeout: float = None, - ele_only: bool = True) -> List[Union['SessionElement', str]]: ... + ele_only: bool = True) -> List[Union[SessionElement, str]]: ... - def befores(self, filter_loc: Union[tuple, str] = '', + def befores(self, + filter_loc: Union[tuple, str] = '', timeout: float = None, - ele_only: bool = True) -> List[Union['SessionElement', str]]: ... + ele_only: bool = True) -> List[Union[SessionElement, str]]: ... - def afters(self, filter_loc: Union[tuple, str] = '', + def afters(self, + filter_loc: Union[tuple, str] = '', timeout: float = None, - ele_only: bool = True) -> List[Union['SessionElement', str]]: ... + ele_only: bool = True) -> List[Union[SessionElement, str]]: ... - def attr(self, attr: str) -> Union[str, None]: ... + def attr(self, attr: str) -> Optional[str]: ... def ele(self, loc_or_str: Union[Tuple[str, str], str], - timeout: float = None) -> Union['SessionElement', str, NoneElement]: ... + timeout: float = None) -> Union[SessionElement, NoneElement]: ... def eles(self, loc_or_str: Union[Tuple[str, str], str], - timeout: float = None) -> List[Union['SessionElement', str]]: ... + timeout: float = None) -> List[SessionElement]: ... def s_ele(self, - loc_or_str: Union[Tuple[str, str], str] = None) -> Union['SessionElement', str, NoneElement]: ... + loc_or_str: Union[Tuple[str, str], str] = None) -> Union[SessionElement, NoneElement]: ... def s_eles(self, - loc_or_str: Union[Tuple[str, str], str]) -> List[Union['SessionElement', str]]: ... + loc_or_str: Union[Tuple[str, str], str]) -> List[SessionElement]: ... def _find_elements(self, loc_or_str: Union[Tuple[str, str], str], @@ -117,7 +129,7 @@ class SessionElement(DrissionElement): single: bool = True, relative: bool = False, raise_err: bool = None) \ - -> Union['SessionElement', str, NoneElement, List[Union['SessionElement', str]]]: ... + -> Union[SessionElement, NoneElement, List[SessionElement]]: ... def _get_ele_path(self, mode: str) -> str: ... @@ -126,4 +138,4 @@ def make_session_ele(html_or_ele: Union[str, SessionElement, SessionPage, Chromi ChromiumBase], loc: Union[str, Tuple[str, str]] = None, single: bool = True) -> Union[ - SessionElement, str, NoneElement, List[Union[SessionElement, str]]]: ... + SessionElement, NoneElement, List[SessionElement]]: ... diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 39a6ac5..98c98fb 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -91,7 +91,7 @@ class ChromiumBase(BasePage): def _d_set_runtime_settings(self) -> None: ... def __call__(self, loc_or_str: Union[Tuple[str, str], str, ChromiumElement], - timeout: float = None) -> ChromiumElement: ... + timeout: float = None) -> Union[ChromiumElement, NoneElement]: ... @property def _js_ready_state(self) -> str: ... @@ -175,15 +175,15 @@ class ChromiumBase(BasePage): all_info: bool = False) -> Union[list, dict]: ... def ele(self, loc_or_ele: Union[Tuple[str, str], str, ChromiumElement, ChromiumFrame], - timeout: float = None) -> Union[ChromiumElement, str]: ... + timeout: float = None) -> Union[ChromiumElement, NoneElement]: ... def eles(self, loc_or_str: Union[Tuple[str, str], str], - timeout: float = None) -> List[Union[ChromiumElement, str]]: ... + timeout: float = None) -> List[ChromiumElement]: ... def s_ele(self, loc_or_ele: Union[Tuple[str, str], str] = None) \ - -> Union[SessionElement, str, NoneElement]: ... + -> Union[SessionElement, NoneElement]: ... - def s_eles(self, loc_or_str: Union[Tuple[str, str], str]) -> List[Union[SessionElement, str]]: ... + def s_eles(self, loc_or_str: Union[Tuple[str, str], str]) -> List[SessionElement]: ... def _find_elements(self, loc_or_ele: Union[Tuple[str, str], str, ChromiumElement, ChromiumFrame], timeout: float = None, single: bool = True, relative: bool = False, raise_err: bool = None) \ diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 37df5a6..8bb10ba 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -11,6 +11,7 @@ from .chromium_page import ChromiumPage from .chromium_tab import ChromiumTab from .web_page import WebPage from .._elements.chromium_element import ChromiumElement +from .._elements.none_element import NoneElement from .._units.listener import FrameListener from .._units.rect import FrameRect from .._units.scroller import FrameScroller @@ -21,7 +22,9 @@ from .._units.waiter import FrameWaiter class ChromiumFrame(ChromiumBase): - def __init__(self, page: Union[ChromiumPage, WebPage, ChromiumTab, ChromiumFrame], ele: ChromiumElement): + def __init__(self, + page: Union[ChromiumPage, WebPage, ChromiumTab, ChromiumFrame], + ele: ChromiumElement): self._page: ChromiumPage = ... self._target_page: ChromiumBase = ... self.tab: ChromiumTab = ... @@ -36,8 +39,9 @@ class ChromiumFrame(ChromiumBase): self._rect: FrameRect = ... self._listener: FrameListener = ... - def __call__(self, loc_or_str: Union[Tuple[str, str], str], - timeout: float = None) -> Union[ChromiumElement, str]: ... + def __call__(self, + loc_or_str: Union[Tuple[str, str], str], + timeout: float = None) -> Union[ChromiumElement, NoneElement]: ... def _check_alive(self) -> None: ... @@ -127,43 +131,82 @@ class ChromiumFrame(ChromiumBase): def remove_attr(self, attr: str) -> None: ... - def run_js(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> Any: ... + def run_js(self, + script: str, + *args, + as_expr: bool = False, + timeout: float = None) -> Any: ... - def parent(self, level_or_loc: Union[tuple, str, int] = 1, index: int = 1) -> Union[ChromiumElement, None]: ... + def parent(self, + level_or_loc: Union[tuple, str, int] = 1, + index: int = 1) -> Union[ChromiumElement, NoneElement]: ... - def prev(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, - timeout: float = 0, ele_only: bool = True) -> Union[ChromiumElement, str]: ... + def prev(self, + filter_loc: Union[tuple, str, int] = '', + index: int = 1, + timeout: float = 0, + ele_only: bool = True) -> Union[ChromiumElement, NoneElement, str]: ... - def next(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, - timeout: float = 0, ele_only: bool = True) -> Union[ChromiumElement, str]: ... + def next(self, + filter_loc: Union[tuple, str, int] = '', + index: int = 1, + timeout: float = 0, + ele_only: bool = True) -> Union[ChromiumElement, NoneElement, str]: ... - def before(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, - timeout: float = None, ele_only: bool = True) -> Union[ChromiumElement, str]: ... + def before(self, + filter_loc: Union[tuple, str, int] = '', + index: int = 1, + timeout: float = None, + ele_only: bool = True) -> Union[ChromiumElement, NoneElement, str]: ... - def after(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, - timeout: float = None, ele_only: bool = True) -> Union[ChromiumElement, str]: ... + def after(self, + filter_loc: Union[tuple, str, int] = '', + index: int = 1, + timeout: float = None, + ele_only: bool = True) -> Union[ChromiumElement, NoneElement, str]: ... - def prevs(self, filter_loc: Union[tuple, str] = '', timeout: float = 0, + def prevs(self, + filter_loc: Union[tuple, str] = '', + timeout: float = 0, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def nexts(self, filter_loc: Union[tuple, str] = '', timeout: float = 0, + def nexts(self, + filter_loc: Union[tuple, str] = '', + timeout: float = 0, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def befores(self, filter_loc: Union[tuple, str] = '', timeout: float = None, + def befores(self, + filter_loc: Union[tuple, str] = '', + timeout: float = None, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def afters(self, filter_loc: Union[tuple, str] = '', timeout: float = None, + def afters(self, + filter_loc: Union[tuple, str] = '', + timeout: float = None, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... - def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, + def get_screenshot(self, + path: [str, Path] = None, + name: str = None, + as_bytes: [bool, str] = None, as_base64: [bool, str] = None) -> Union[str, bytes]: ... - def _get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: [bool, str] = None, - as_base64: [bool, str] = None, full_page: bool = False, left_top: Tuple[int, int] = None, - right_bottom: Tuple[int, int] = None, ele: ChromiumElement = None) -> Union[str, bytes]: ... + def _get_screenshot(self, + path: [str, Path] = None, + name: str = None, + as_bytes: [bool, str] = None, + as_base64: [bool, str] = None, + full_page: bool = False, + left_top: Tuple[int, int] = None, + right_bottom: Tuple[int, int] = None, + ele: ChromiumElement = None) -> Union[str, bytes]: ... - def _find_elements(self, loc_or_ele: Union[Tuple[str, str], str, ChromiumElement, ChromiumFrame], - timeout: float = None, single: bool = True, relative: bool = False, raise_err: bool = None) \ + def _find_elements(self, + loc_or_ele: Union[Tuple[str, str], str, ChromiumElement, ChromiumFrame], + timeout: float = None, + single: bool = True, + relative: bool = False, + raise_err: bool = None) \ -> Union[ChromiumElement, ChromiumFrame, None, List[Union[ChromiumElement, ChromiumFrame]]]: ... def _is_inner_frame(self) -> bool: ... diff --git a/DrissionPage/_pages/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi index 29c5c7b..327b037 100644 --- a/DrissionPage/_pages/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -14,6 +14,7 @@ from .session_page import SessionPage from .web_page import WebPage from .._base.browser import Browser from .._elements.chromium_element import ChromiumElement +from .._elements.none_element import NoneElement from .._elements.session_element import SessionElement from .._units.rect import TabRect from .._units.setter import TabSetter, WebPageTabSetter @@ -51,7 +52,7 @@ class WebPageTab(SessionPage, ChromiumTab): def __call__(self, loc_or_str: Union[Tuple[str, str], str, ChromiumElement, SessionElement], - timeout: float = None) -> Union[ChromiumElement, SessionElement]: ... + timeout: float = None) -> Union[ChromiumElement, SessionElement, NoneElement]: ... @property def page(self) -> WebPage: ... @@ -120,16 +121,16 @@ class WebPageTab(SessionPage, ChromiumTab): def ele(self, loc_or_ele: Union[Tuple[str, str], str, ChromiumElement, SessionElement], - timeout: float = None) -> Union[ChromiumElement, SessionElement, str]: ... + timeout: float = None) -> Union[ChromiumElement, SessionElement, NoneElement]: ... def eles(self, loc_or_str: Union[Tuple[str, str], str], - timeout: float = None) -> List[Union[ChromiumElement, SessionElement, str]]: ... + timeout: float = None) -> List[Union[ChromiumElement, SessionElement]]: ... def s_ele(self, loc_or_ele: Union[Tuple[str, str], str] = None) \ - -> Union[SessionElement, str, None]: ... + -> Union[SessionElement, NoneElement]: ... - def s_eles(self, loc_or_str: Union[Tuple[str, str], str]) -> List[Union[SessionElement, str]]: ... + def s_eles(self, loc_or_str: Union[Tuple[str, str], str]) -> List[SessionElement]: ... def change_mode(self, mode: str = None, go: bool = True, copy_cookies: bool = True) -> None: ... @@ -168,5 +169,5 @@ class WebPageTab(SessionPage, ChromiumTab): def _find_elements(self, loc_or_ele: Union[Tuple[str, str], str, ChromiumElement, SessionElement, ChromiumFrame], timeout: float = None, single: bool = True, relative: bool = False, raise_err: bool = None) \ - -> Union[ChromiumElement, SessionElement, ChromiumFrame, str, None, List[Union[SessionElement, str]], List[ - Union[ChromiumElement, str, ChromiumFrame]]]: ... + -> Union[ChromiumElement, SessionElement, ChromiumFrame, NoneElement, List[SessionElement], List[ + Union[ChromiumElement, ChromiumFrame]]]: ... diff --git a/DrissionPage/_pages/session_page.pyi b/DrissionPage/_pages/session_page.pyi index 017bc13..b3116b2 100644 --- a/DrissionPage/_pages/session_page.pyi +++ b/DrissionPage/_pages/session_page.pyi @@ -40,7 +40,7 @@ class SessionPage(BasePage): def __call__(self, loc_or_str: Union[Tuple[str, str], str, SessionElement], - timeout: float = None) -> Union[SessionElement, str, NoneElement]: ... + timeout: float = None) -> Union[SessionElement, NoneElement]: ... # -----------------共有属性和方法------------------- @property @@ -89,23 +89,28 @@ class SessionPage(BasePage): def ele(self, loc_or_ele: Union[Tuple[str, str], str, SessionElement], - timeout: float = None) -> Union[SessionElement, str, NoneElement]: ... + timeout: float = None) -> Union[SessionElement, NoneElement]: ... def eles(self, loc_or_str: Union[Tuple[str, str], str], - timeout: float = None) -> List[Union[SessionElement, str]]: ... + timeout: float = None) -> List[SessionElement]: ... def s_ele(self, loc_or_ele: Union[Tuple[str, str], str, SessionElement] = None) \ - -> Union[SessionElement, str, NoneElement]: ... + -> Union[SessionElement, NoneElement]: ... - def s_eles(self, loc_or_str: Union[Tuple[str, str], str]) -> List[Union[SessionElement, str]]: ... + def s_eles(self, loc_or_str: Union[Tuple[str, str], str]) -> List[SessionElement]: ... - def _find_elements(self, loc_or_ele: Union[Tuple[str, str], str, SessionElement], - timeout: float = None, single: bool = True, raise_err: bool = None) \ - -> Union[SessionElement, str, NoneElement, List[Union[SessionElement, str]]]: ... + def _find_elements(self, + loc_or_ele: Union[Tuple[str, str], str, SessionElement], + timeout: float = None, + single: bool = True, + raise_err: bool = None) \ + -> Union[SessionElement, NoneElement, List[SessionElement]]: ... - def get_cookies(self, as_dict: bool = False, all_domains: bool = False, + def get_cookies(self, + as_dict: bool = False, + all_domains: bool = False, all_info: bool = False) -> Union[dict, list]: ... # ----------------session独有属性和方法----------------------- @@ -164,7 +169,8 @@ class SessionPage(BasePage): **kwargs) -> tuple: ... -def check_headers(kwargs: Union[dict, CaseInsensitiveDict], headers: Union[dict, CaseInsensitiveDict], +def check_headers(kwargs: Union[dict, CaseInsensitiveDict], + headers: Union[dict, CaseInsensitiveDict], arg: str) -> bool: ... diff --git a/DrissionPage/_pages/web_page.pyi b/DrissionPage/_pages/web_page.pyi index a0b78d5..9c80c71 100644 --- a/DrissionPage/_pages/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -16,6 +16,7 @@ from .._base.driver import Driver from .._configs.chromium_options import ChromiumOptions from .._configs.session_options import SessionOptions from .._elements.chromium_element import ChromiumElement +from .._elements.none_element import NoneElement from .._elements.session_element import SessionElement from .._units.setter import WebPageSetter @@ -35,7 +36,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def __call__(self, loc_or_str: Union[Tuple[str, str], str, ChromiumElement, SessionElement], - timeout: float = None) -> Union[ChromiumElement, SessionElement]: ... + timeout: float = None) -> Union[ChromiumElement, SessionElement, NoneElement]: ... # -----------------共有属性和方法------------------- @property @@ -102,16 +103,15 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def ele(self, loc_or_ele: Union[Tuple[str, str], str, ChromiumElement, SessionElement], - timeout: float = None) -> Union[ChromiumElement, SessionElement, str]: ... + timeout: float = None) -> Union[ChromiumElement, SessionElement, NoneElement]: ... def eles(self, loc_or_str: Union[Tuple[str, str], str], - timeout: float = None) -> List[Union[ChromiumElement, SessionElement, str]]: ... + timeout: float = None) -> List[Union[ChromiumElement, SessionElement]]: ... - def s_ele(self, loc_or_ele: Union[Tuple[str, str], str] = None) \ - -> Union[SessionElement, str, None]: ... + def s_ele(self, loc_or_ele: Union[Tuple[str, str], str] = None) -> Union[SessionElement, NoneElement]: ... - def s_eles(self, loc_or_str: Union[Tuple[str, str], str]) -> List[Union[SessionElement, str]]: ... + def s_eles(self, loc_or_str: Union[Tuple[str, str], str]) -> List[SessionElement]: ... def change_mode(self, mode: str = None, go: bool = True, copy_cookies: bool = True) -> None: ... @@ -119,12 +119,17 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def cookies_to_browser(self) -> None: ... - def get_cookies(self, as_dict: bool = False, all_domains: bool = False, + def get_cookies(self, + as_dict: bool = False, + all_domains: bool = False, all_info: bool = False) -> Union[dict, list]: ... def get_tab(self, id_or_num: Union[str, WebPageTab, int] = None) -> WebPageTab: ... - def new_tab(self, url: str = None, new_window: bool = False, background: bool = False, + def new_tab(self, + url: str = None, + new_window: bool = False, + background: bool = False, new_context: bool = False) -> WebPageTab: ... def close_driver(self) -> None: ... @@ -157,12 +162,17 @@ class WebPage(SessionPage, ChromiumPage, BasePage): @property def set(self) -> WebPageSetter: ... - def _find_elements(self, loc_or_ele: Union[Tuple[str, str], str, ChromiumElement, SessionElement, ChromiumFrame], - timeout: float = None, single: bool = True, relative: bool = False, raise_err: bool = None) \ - -> Union[ChromiumElement, SessionElement, ChromiumFrame, str, None, List[Union[SessionElement, str]], List[ - Union[ChromiumElement, str, ChromiumFrame]]]: ... + def _find_elements(self, + loc_or_ele: Union[Tuple[str, str], str, ChromiumElement, SessionElement, ChromiumFrame], + timeout: float = None, + single: bool = True, + relative: bool = False, + raise_err: bool = None) \ + -> Union[ChromiumElement, SessionElement, ChromiumFrame, NoneElement, List[SessionElement], + List[Union[ChromiumElement, ChromiumFrame]]]: ... - def _set_start_options(self, dr_opt: Union[Driver, bool, None], + def _set_start_options(self, + dr_opt: Union[Driver, bool, None], se_opt: Union[Session, SessionOptions, bool, None]) -> None: ... def quit(self, timeout: float = 5, force: bool = True) -> None: ... From 71dfe3037c873887e4909dcb620a0e367d29d1c6 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 21 Dec 2023 10:48:48 +0800 Subject: [PATCH 146/182] =?UTF-8?q?wait.ele=5Fdisplay()=E5=92=8Cwait.ele?= =?UTF-8?q?=5Fhidden()=E4=BC=9A=E7=AD=89=E5=BE=85=E5=85=83=E7=B4=A0?= =?UTF-8?q?=E5=8A=A0=E8=BD=BD=EF=BC=9B=20=E4=BF=AE=E5=A4=8DPage=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E5=85=83=E7=B4=A0=E6=9C=89=E6=97=B6=E4=BC=9A=E8=BF=94?= =?UTF-8?q?=E5=9B=9Estr=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_base/base.py | 13 ++--- DrissionPage/_elements/chromium_element.py | 61 ++++++++++++++++++++- DrissionPage/_elements/chromium_element.pyi | 28 ++++++---- DrissionPage/_pages/chromium_base.py | 34 +++++------- DrissionPage/_pages/chromium_frame.py | 2 - DrissionPage/_pages/chromium_frame.pyi | 2 +- DrissionPage/_units/waiter.py | 50 ++++++++++++----- DrissionPage/_units/waiter.pyi | 27 ++++++--- 9 files changed, 153 insertions(+), 66 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index e51d275..2b4c57c 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b24' +__version__ = '4.0.0b25' diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index b71730a..c3dbec1 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -447,14 +447,13 @@ class BasePage(BaseParser): r = self._find_elements(loc_or_ele, timeout=timeout, single=single, raise_err=raise_err) - if not single: + if r or isinstance(r, list): return r - if isinstance(r, NoneElement): - if Settings.raise_when_ele_not_found or raise_err is True: - raise ElementNotFoundError(None, method, {'loc_or_str': loc_or_ele}) - else: - r.method = method - r.args = {'loc_or_str': loc_or_ele} + if Settings.raise_when_ele_not_found or raise_err is True: + raise ElementNotFoundError(None, method, {'loc_or_str': loc_or_ele}) + + r.method = method + r.args = {'loc_or_str': loc_or_ele} return r @abstractmethod diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index a99eaed..5528b28 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -1040,7 +1040,7 @@ class ShadowRoot(BaseElement): else: nod_ids = self.page.run_cdp('DOM.querySelectorAll', nodeId=self._node_id, selector=loc[1])['nodeId'] - result = [make_chromium_ele(self.page, node_id=n) for n in nod_ids] + result = make_chromium_eles(self.page, node_ids=nod_ids, single=False) else: eles = make_session_ele(self.html).eles(loc) @@ -1205,6 +1205,7 @@ def make_chromium_ele(page, node_id=None, obj_id=None): if node_id: node = page.run_cdp('DOM.describeNode', nodeId=node_id) if node['node']['nodeName'] in ('#text', '#comment'): + # todo: Node() return node['node']['nodeValue'] backend_id = node['node']['backendNodeId'] obj_id = page.run_cdp('DOM.resolveNode', nodeId=node_id)['object']['objectId'] @@ -1212,6 +1213,7 @@ def make_chromium_ele(page, node_id=None, obj_id=None): elif obj_id: node = page.run_cdp('DOM.describeNode', objectId=obj_id) if node['node']['nodeName'] in ('#text', '#comment'): + # todo: Node() return node['node']['nodeValue'] backend_id = node['node']['backendNodeId'] node_id = node['node']['nodeId'] @@ -1227,6 +1229,61 @@ def make_chromium_ele(page, node_id=None, obj_id=None): return ele +def make_chromium_eles(page, node_ids=None, obj_ids=None, single=True, ele_only=True): + """根据node id或object id生成相应元素对象 + :param page: ChromiumPage对象 + :param node_ids: 元素的node id + :param obj_ids: 元素的object id + :param single: 是否获取但个元素 + :param ele_only: 是否只要ele + :return: ChromiumElement对象或ChromiumFrame对象 + """ + nodes = [] + if node_ids: + for node_id in node_ids: + if not node_id: + return False + node = page.run_cdp('DOM.describeNode', nodeId=node_id) + if node['node']['nodeName'] in ('#text', '#comment'): + if ele_only: + continue + else: + # todo: Node() + pass + + obj_id = page.run_cdp('DOM.resolveNode', nodeId=node_id)['object']['objectId'] + ele = ChromiumElement(page, obj_id=obj_id, node_id=node_id, backend_id=node['node']['backendNodeId']) + if ele.tag in __FRAME_ELEMENT__: + from .._pages.chromium_frame import ChromiumFrame + ele = ChromiumFrame(page, ele) + if single: + return ele + nodes.append(ele) + + if obj_ids: + for obj_id in obj_ids: + if not obj_id: + return False + node = page.run_cdp('DOM.describeNode', objectId=obj_id) + if node['node']['nodeName'] in ('#text', '#comment'): + if ele_only: + continue + else: + # todo: Node + pass + + ele = ChromiumElement(page, obj_id=obj_id, node_id=node['node']['nodeId'], + backend_id=node['node']['backendNodeId']) + if ele.tag in __FRAME_ELEMENT__: + from .._pages.chromium_frame import ChromiumFrame + ele = ChromiumFrame(page, ele) + if single: + return ele + nodes.append(ele) + + return NoneElement(page) if single and not nodes else nodes + + def make_js_for_find_ele_by_xpath(xpath, type_txt, node_txt): """生成用xpath在元素中查找元素的js文本 :param xpath: xpath文本 @@ -1347,7 +1404,7 @@ def parse_js_result(page, ele, result): elif class_name == 'HTMLDocument': return result else: - return make_chromium_ele(page, obj_id=result['objectId']) + return make_chromium_eles(page, obj_ids=(result['objectId'],)) elif sub_type == 'array': r = page.run_cdp('Runtime.getProperties', objectId=result['objectId'], diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 7dd3a52..f006917 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -26,10 +26,10 @@ PIC_TYPE = Literal['jpg', 'jpeg', 'png', 'webp', True] class ChromiumElement(DrissionElement): - def __init__(self, page: ChromiumBase, node_id: str = None, obj_id: str = None, backend_id: int = None): + def __init__(self, page: ChromiumBase, node_id: int = None, obj_id: str = None, backend_id: int = None): self._tag: str = ... self.page: Union[ChromiumPage, WebPage] = ... - self._node_id: str = ... + self._node_id: int = ... self._obj_id: str = ... self._backend_id: int = ... self._doc_id: str = ... @@ -217,11 +217,11 @@ class ChromiumElement(DrissionElement): def drag_to(self, ele_or_loc: Union[tuple, ChromiumElement], duration: float = 0.5) -> None: ... - def _get_obj_id(self, node_id: str = None, backend_id: int = None) -> str: ... + def _get_obj_id(self, node_id: int = None, backend_id: int = None) -> str: ... - def _get_node_id(self, obj_id: str = None, backend_id: int = None) -> str: ... + def _get_node_id(self, obj_id: str = None, backend_id: int = None) -> int: ... - def _get_backend_id(self, node_id: str) -> str: ... + def _get_backend_id(self, node_id: int) -> int: ... def _get_ele_path(self, mode: str) -> str: ... @@ -230,7 +230,7 @@ class ShadowRoot(BaseElement): def __init__(self, parent_ele: ChromiumElement, obj_id: str = None, backend_id: int = None): self._obj_id: str = ... - self._node_id: str = ... + self._node_id: int = ... self._backend_id: int = ... self.page: ChromiumPage = ... self.parent_ele: ChromiumElement = ... @@ -294,11 +294,11 @@ class ShadowRoot(BaseElement): -> Union[ChromiumElement, ChromiumFrame, NoneElement, str, List[Union[ChromiumElement, ChromiumFrame, str]]]: ... - def _get_node_id(self, obj_id: str) -> str: ... + def _get_node_id(self, obj_id: str) -> int: ... - def _get_obj_id(self, back_id: str) -> str: ... + def _get_obj_id(self, back_id: int) -> str: ... - def _get_backend_id(self, node_id: str) -> int: ... + def _get_backend_id(self, node_id: int) -> int: ... def find_in_chromium_ele(ele: ChromiumElement, loc: Union[str, Tuple[str, str]], @@ -314,10 +314,18 @@ def find_by_css(ele: ChromiumElement, selector: str, single: bool, timeout: float) -> Union[ChromiumElement, List[ChromiumElement], NoneElement]: ... -def make_chromium_ele(page: ChromiumBase, node_id: str = ..., obj_id: str = ...) \ +def make_chromium_ele(page: ChromiumBase, node_id: int = ..., obj_id: str = ...) \ -> Union[ChromiumElement, ChromiumFrame, str]: ... +def make_chromium_eles(page: ChromiumBase, + node_ids: Union[tuple, list] = None, + obj_ids: Union[tuple, list] = None, + single: bool = True, + ele_only: bool = True) -> Union[ChromiumElement, ChromiumFrame, NoneElement, +List[Union[ChromiumElement, ChromiumFrame]]]: ... + + def make_js_for_find_ele_by_xpath(xpath: str, type_txt: str, node_txt: str) -> str: ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 86821db..bfa068a 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -11,13 +11,13 @@ from time import perf_counter, sleep from urllib.parse import quote from .._base.base import BasePage +from .._elements.chromium_element import ChromiumElement, run_js, make_chromium_eles +from .._elements.none_element import NoneElement +from .._elements.session_element import make_session_ele from .._functions.locator import get_loc, is_loc from .._functions.settings import Settings from .._functions.tools import get_usable_path, raise_error from .._functions.web import location_in_viewport -from .._elements.chromium_element import ChromiumElement, run_js, make_chromium_ele -from .._elements.none_element import NoneElement -from .._elements.session_element import make_session_ele from .._units.actions import Actions from .._units.listener import Listener from .._units.rect import TabRect @@ -26,7 +26,7 @@ from .._units.scroller import PageScroller from .._units.setter import ChromiumBaseSetter from .._units.states import PageStates from .._units.waiter import BaseWaiter -from ..errors import ContextLostError, ElementLostError, CDPError, PageClosedError, ElementNotFoundError +from ..errors import ContextLostError, CDPError, PageClosedError, ElementNotFoundError __ERROR__ = 'error' @@ -586,9 +586,11 @@ class ChromiumBase(BasePage): timeout = timeout if timeout is not None else self.timeout end_time = perf_counter() + timeout + search_ids = [] try: search_result = self.run_cdp_loaded('DOM.performSearch', query=loc, includeUserAgentShadowDOM=True) count = search_result['resultCount'] + search_ids.append(search_result['searchId']) except ContextLostError: search_result = None count = 0 @@ -606,33 +608,27 @@ class ChromiumBase(BasePage): pass if ok: - try: - if single: - r = make_chromium_ele(self, node_id=nodeIds['nodeIds'][0]) - break - - else: - r = [make_chromium_ele(self, node_id=i) for i in nodeIds['nodeIds']] - break - - except ElementLostError: + r = make_chromium_eles(self, node_ids=nodeIds['nodeIds'], single=single) + if r is not False: + break + else: ok = False try: search_result = self.run_cdp_loaded('DOM.performSearch', query=loc, includeUserAgentShadowDOM=True) count = search_result['resultCount'] + search_ids.append(search_result['searchId']) except ContextLostError: pass if perf_counter() >= end_time: - return NoneElement(self) if single else [] + return None if single else [] sleep(.1) - try: - self.run_cdp('DOM.discardSearchResults', searchId=search_result['searchId']) - except: - pass + for _id in search_ids: + self._driver.run('DOM.discardSearchResults', searchId=_id) + return r def refresh(self, ignore_cache=False): diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 228d96f..873f331 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -586,9 +586,7 @@ class ChromiumFrame(ChromiumBase): """ if isinstance(loc_or_ele, ChromiumElement): return loc_or_ele - self.wait.load_complete() - return self.doc_ele._ele(loc_or_ele, timeout, raise_err=raise_err) if single else self.doc_ele.eles(loc_or_ele, timeout) diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 8bb10ba..8dc1380 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -96,7 +96,7 @@ class ChromiumFrame(ChromiumBase): def _obj_id(self) -> str: ... @property - def _node_id(self) -> str: ... + def _node_id(self) -> int: ... @property def active_ele(self) -> ChromiumElement: ... diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index e02ca44..042a0e6 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -36,7 +36,16 @@ class BaseWaiter(object): :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ - ele = self._driver._ele(loc_or_ele, raise_err=False, timeout=0) + if timeout is None: + timeout = self._driver.timeout + end_time = perf_counter() + timeout + ele = self._driver._ele(loc_or_ele, raise_err=False, timeout=timeout) + timeout = end_time - perf_counter() + if timeout <= 0: + if raise_err is True or Settings.raise_when_wait_failed is True: + raise WaitTimeoutError('等待元素显示失败。') + else: + return False return ele.wait.display(timeout, raise_err=raise_err) def ele_hidden(self, loc_or_ele, timeout=None, raise_err=None): @@ -46,7 +55,16 @@ class BaseWaiter(object): :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ - ele = self._driver._ele(loc_or_ele, raise_err=False, timeout=0) + if timeout is None: + timeout = self._driver.timeout + end_time = perf_counter() + timeout + ele = self._driver._ele(loc_or_ele, raise_err=False, timeout=timeout) + timeout = end_time - perf_counter() + if timeout <= 0: + if raise_err is True or Settings.raise_when_wait_failed is True: + raise WaitTimeoutError('等待元素显示失败。') + else: + return False return ele.wait.hidden(timeout, raise_err=raise_err) def ele_loaded(self, loc, timeout=None, raise_err=None): @@ -296,7 +314,7 @@ class ElementWaiter(object): :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ - return self._wait_state('is_alive', False, timeout, raise_err) + return self._wait_state('is_alive', False, timeout, raise_err, err_text='等待元素被删除失败。') def display(self, timeout=None, raise_err=None): """等待元素从dom显示 @@ -304,7 +322,7 @@ class ElementWaiter(object): :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ - return self._wait_state('is_displayed', True, timeout, raise_err) + return self._wait_state('is_displayed', True, timeout, raise_err, err_text='等待元素显示失败。') def hidden(self, timeout=None, raise_err=None): """等待元素从dom隐藏 @@ -312,7 +330,7 @@ class ElementWaiter(object): :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ - return self._wait_state('is_displayed', False, timeout, raise_err) + return self._wait_state('is_displayed', False, timeout, raise_err, err_text='等待元素隐藏失败。') def covered(self, timeout=None, raise_err=None): """等待当前元素被遮盖 @@ -320,15 +338,15 @@ class ElementWaiter(object): :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ - return self._wait_state('is_covered', True, timeout, raise_err) + return self._wait_state('is_covered', True, timeout, raise_err, err_text='等待元素被覆盖失败。') def not_covered(self, timeout=None, raise_err=None): - """等待当前元素被遮盖 + """等待当前元素不被遮盖 :param timeout: 超时时间，为None使用元素所在页面timeout属性 :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ - return self._wait_state('is_covered', False, timeout, raise_err) + return self._wait_state('is_covered', False, timeout, raise_err, err_text='等待元素不被覆盖失败。') def enabled(self, timeout=None, raise_err=None): """等待当前元素变成可用 @@ -336,15 +354,15 @@ class ElementWaiter(object): :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ - return self._wait_state('is_enabled', True, timeout, raise_err) + return self._wait_state('is_enabled', True, timeout, raise_err, err_text='等待元素变成可用失败。') def disabled(self, timeout=None, raise_err=None): - """等待当前元素变成可用 + """等待当前元素变成不可用 :param timeout: 超时时间，为None使用元素所在页面timeout属性 :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ - return self._wait_state('is_enabled', False, timeout, raise_err) + return self._wait_state('is_enabled', False, timeout, raise_err, err_text='等待元素变成不可用失败。') def disabled_or_deleted(self, timeout=None, raise_err=None): """等待当前元素变成不可用或从DOM移除 @@ -361,7 +379,7 @@ class ElementWaiter(object): sleep(.05) if raise_err is True or Settings.raise_when_wait_failed is True: - raise WaitTimeoutError('等待元素隐藏或删除失败。') + raise WaitTimeoutError('等待元素隐藏或被删除失败。') else: return False @@ -397,14 +415,16 @@ class ElementWaiter(object): else: return False - def _wait_state(self, attr, mode=False, timeout=None, raise_err=None): - """等待元素某个bool状态到达指定状态 + def _wait_state(self, attr, mode=False, timeout=None, raise_err=None, err_text=None): + """等待元素某个元素状态到达指定状态 :param attr: 状态名称 :param mode: True或False :param timeout: 超时时间，为None使用元素所在页面timeout属性 :param raise_err: 等待失败时是否报错，为None时根据Settings设置 + :param err_text: 抛出错误时显示的信息 :return: 是否等待成功 """ + err_text = err_text or '等待元素状态改变失败。' if timeout is None: timeout = self._page.timeout end_time = perf_counter() + timeout @@ -414,7 +434,7 @@ class ElementWaiter(object): sleep(.05) if raise_err is True or Settings.raise_when_wait_failed is True: - raise WaitTimeoutError('等待元素状态改变失败。') + raise WaitTimeoutError(err_text) else: return False diff --git a/DrissionPage/_units/waiter.pyi b/DrissionPage/_units/waiter.pyi index f41d2ad..1782f86 100644 --- a/DrissionPage/_units/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -18,17 +18,23 @@ class BaseWaiter(object): def __call__(self, second: float) -> None: ... - def ele_deleted(self, loc_or_ele: Union[str, tuple, ChromiumElement], timeout: float = None, - raise_err: bool = None) -> bool: ... + def ele_deleted(self, + loc_or_ele: Union[str, tuple, ChromiumElement], + timeout: float = None, + raise_err: bool = None) -> bool: ... - def ele_display(self, loc_or_ele: Union[str, tuple, ChromiumElement], timeout: float = None, + def ele_display(self, + loc_or_ele: Union[str, tuple, ChromiumElement], + timeout: float = None, raise_err: bool = None) -> bool: ... def ele_hidden(self, loc_or_ele: Union[str, tuple, ChromiumElement], timeout: float = None, raise_err: bool = None) -> bool: ... - def ele_loaded(self, loc: Union[str, tuple], timeout: float = None, - raise_err: bool = None) -> Union[bool, ChromiumElement]: ... + def ele_loaded(self, + loc: Union[str, tuple], + timeout: float = None, + raise_err: bool = None) -> Union[bool, ChromiumElement]: ... def _loading(self, timeout: float = None, start: bool = True, gap: float = .01, raise_err: bool = None) -> bool: ... @@ -64,9 +70,7 @@ class PageWaiter(TabWaiter): class ElementWaiter(object): - def __init__(self, - page: ChromiumBase, - ele: ChromiumElement): + def __init__(self, page: ChromiumBase, ele: ChromiumElement): self._ele: ChromiumElement = ... self._page: ChromiumBase = ... @@ -90,7 +94,12 @@ class ElementWaiter(object): def stop_moving(self, gap: float = .1, timeout: float = None, raise_err: bool = None) -> bool: ... - def _wait_state(self, attr: str, mode: bool = False, timeout: float = None, raise_err: bool = None) -> bool: ... + def _wait_state(self, + attr: str, + mode: bool = False, + timeout: float = None, + raise_err: bool = None, + err_text: str = None) -> bool: ... class FrameWaiter(BaseWaiter, ElementWaiter): From 4de95e6354c73a9a7c07100cb987c15d3307bcb4 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 21 Dec 2023 16:28:41 +0800 Subject: [PATCH 147/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B0=8F=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/base.py | 13 ++++++------- DrissionPage/_functions/locator.py | 7 ++----- DrissionPage/_functions/tools.py | 2 +- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index c3dbec1..8549d78 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -68,14 +68,13 @@ class BaseElement(BaseParser): def _ele(self, loc_or_str, timeout=None, single=True, relative=False, raise_err=None, method=None): r = self._find_elements(loc_or_str, timeout=timeout, single=single, relative=relative, raise_err=raise_err) - if not single: + if r or isinstance(r, list): return r - if isinstance(r, NoneElement): - if Settings.raise_when_ele_not_found or raise_err is True: - raise ElementNotFoundError(None, method, {'loc_or_str': loc_or_str}) - else: - r.method = method - r.args = {'loc_or_str': loc_or_str} + if Settings.raise_when_ele_not_found or raise_err is True: + raise ElementNotFoundError(None, method, {'loc_or_str': loc_or_str}) + + r.method = method + r.args = {'loc_or_str': loc_or_str} return r @abstractmethod diff --git a/DrissionPage/_functions/locator.py b/DrissionPage/_functions/locator.py index 58d6222..13ee7d1 100644 --- a/DrissionPage/_functions/locator.py +++ b/DrissionPage/_functions/locator.py @@ -404,7 +404,7 @@ def translate_loc(loc): elif loc_0 == By.CLASS_NAME: loc_str = f'//*[@class="{loc[1]}"]' - elif loc_0 == By.PARTIAL_LINK_TEXT: + elif loc_0 == By.LINK_TEXT: loc_str = f'//a[text()="{loc[1]}"]' elif loc_0 == By.NAME: @@ -446,7 +446,7 @@ def translate_css_loc(loc): elif loc_0 == By.CLASS_NAME: loc_str = f'.{css_trans(loc[1])}' - elif loc_0 == By.PARTIAL_LINK_TEXT: + elif loc_0 == By.LINK_TEXT: loc_by = By.XPATH loc_str = f'//a[text()="{css_trans(loc[1])}"]' @@ -463,9 +463,6 @@ def translate_css_loc(loc): else: raise ValueError('无法识别的定位符。') - if loc_by == By.CSS_SELECTOR: - pass - return loc_by, loc_str diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index 5665a69..6ab1229 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -267,7 +267,7 @@ def raise_error(result, ignore=None): 'No node with given id found', 'Node with given id does not belong to the document', 'No node found for given backend id'): r = ElementLostError() - elif error == ('tab closed', 'No target with given id found'): + elif error in ('tab closed', 'No target with given id found'): r = PageClosedError() elif error == 'timeout': r = TimeoutError(f'超时。\n错误：{result["error"]}\nmethod：{result["method"]}\nargs：{result["args"]}\n' From 17728bdfa23a3cd9d5576e5ed4bd281c63476959 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 21 Dec 2023 17:08:30 +0800 Subject: [PATCH 148/182] =?UTF-8?q?4.0.0b25wait.ele=5Fdisplay()=E6=94=B9?= =?UTF-8?q?=E6=88=90wait.ele=5Fdisplayed()=EF=BC=9Bwait.display()=E6=94=B9?= =?UTF-8?q?=E6=88=90wait.displayed()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_units/waiter.py | 6 +++--- DrissionPage/_units/waiter.pyi | 4 ++-- README.md | 10 ++-------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 042a0e6..9393e6a 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -29,7 +29,7 @@ class BaseWaiter(object): ele = self._driver._ele(loc_or_ele, raise_err=False, timeout=0) return ele.wait.deleted(timeout, raise_err=raise_err) if ele else True - def ele_display(self, loc_or_ele, timeout=None, raise_err=None): + def ele_displayed(self, loc_or_ele, timeout=None, raise_err=None): """等待元素变成显示状态 :param loc_or_ele: 要等待的元素，可以是已有元素、定位符 :param timeout: 超时时间，默认读取页面超时时间 @@ -46,7 +46,7 @@ class BaseWaiter(object): raise WaitTimeoutError('等待元素显示失败。') else: return False - return ele.wait.display(timeout, raise_err=raise_err) + return ele.wait.displayed(timeout, raise_err=raise_err) def ele_hidden(self, loc_or_ele, timeout=None, raise_err=None): """等待元素变成隐藏状态 @@ -316,7 +316,7 @@ class ElementWaiter(object): """ return self._wait_state('is_alive', False, timeout, raise_err, err_text='等待元素被删除失败。') - def display(self, timeout=None, raise_err=None): + def displayed(self, timeout=None, raise_err=None): """等待元素从dom显示 :param timeout: 超时时间，为None使用元素所在页面timeout属性 :param raise_err: 等待失败时是否报错，为None时根据Settings设置 diff --git a/DrissionPage/_units/waiter.pyi b/DrissionPage/_units/waiter.pyi index 1782f86..94803ef 100644 --- a/DrissionPage/_units/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -23,7 +23,7 @@ class BaseWaiter(object): timeout: float = None, raise_err: bool = None) -> bool: ... - def ele_display(self, + def ele_displayed(self, loc_or_ele: Union[str, tuple, ChromiumElement], timeout: float = None, raise_err: bool = None) -> bool: ... @@ -78,7 +78,7 @@ class ElementWaiter(object): def deleted(self, timeout: float = None, raise_err: bool = None) -> bool: ... - def display(self, timeout: float = None, raise_err: bool = None) -> bool: ... + def displayed(self, timeout: float = None, raise_err: bool = None) -> bool: ... def hidden(self, timeout: float = None, raise_err: bool = None) -> bool: ... diff --git a/README.md b/README.md index 49e9d7f..d775971 100644 --- a/README.md +++ b/README.md @@ -32,13 +32,7 @@ python 版本：3.6 及以上 **📖 使用文档：** [点击查看](https://g1879.gitee.io/drissionpagedocs) -**交流 QQ 群：** 897838127[已满]、558778073[已满]、636361957 - ---- - -# 🔥 新版预告 - -查看下一步开发计划：[新版预告](https://g1879.gitee.io/drissionpagedocs/whatsnew/3_3/) +**交流 QQ 群：** 636361957 --- @@ -56,7 +50,7 @@ python 版本：3.6 及以上 # 💡 理念 -简洁！易用 ！方便！ +简洁而强大！ --- From 98b992ac90dbdd5cf4631c1c34bb769724c0a144 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 21 Dec 2023 21:31:05 +0800 Subject: [PATCH 149/182] =?UTF-8?q?4.0.0b26=E4=BF=AE=E5=A4=8D=E5=B0=8F?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_pages/chromium_base.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 2b4c57c..f631dfe 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b25' +__version__ = '4.0.0b26' diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index bfa068a..e9ba765 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -622,7 +622,7 @@ class ChromiumBase(BasePage): pass if perf_counter() >= end_time: - return None if single else [] + return NoneElement(self) if single else [] sleep(.1) diff --git a/setup.py b/setup.py index 99298bd..82bafe4 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b24", + version="4.0.0b26", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From da5a8a9e425fc72dd99ae31a1c5a2c593de43116 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 22 Dec 2023 17:49:49 +0800 Subject: [PATCH 150/182] =?UTF-8?q?=E5=A2=9E=E5=8A=A0wait.alert=5Fclosed()?= =?UTF-8?q?=E3=80=81wait.has=5Frect()=EF=BC=9Brun=5Fjs()=E6=97=A0=E8=A7=86?= =?UTF-8?q?=E5=85=B6=E4=B8=AD=E4=BA=A7=E7=94=9F=E7=9A=84=E5=BC=B9=E7=AA=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 22 +++++++++++++-------- DrissionPage/_base/driver.py | 6 +++--- DrissionPage/_elements/chromium_element.py | 23 +++++++++++++--------- DrissionPage/_functions/browser.py | 2 +- DrissionPage/_pages/chromium_base.py | 12 ++++++++--- DrissionPage/_units/waiter.py | 15 ++++++++++++++ DrissionPage/_units/waiter.pyi | 10 +++++++--- 7 files changed, 63 insertions(+), 27 deletions(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 4f12011..bf6c54b 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -5,6 +5,7 @@ """ from time import sleep, perf_counter +from errors import PageClosedError from .driver import BrowserDriver, Driver from .._functions.tools import stop_process_on_port, raise_error from .._units.downloader import DownloadManager @@ -65,9 +66,10 @@ class Browser(object): def _onTargetCreated(self, **kwargs): """标签页创建时执行""" - if kwargs['targetInfo']['type'] == 'page' and not kwargs['targetInfo']['url'].startswith('devtools://'): - self._drivers[kwargs['targetInfo']['targetId']] = Driver(kwargs['targetInfo']['targetId'], 'page', - self.address) + if (kwargs['targetInfo']['type'] in ('page', 'webview') + and not kwargs['targetInfo']['url'].startswith('devtools://')): + self._drivers[kwargs['targetInfo']['targetId']] = Driver(kwargs['targetInfo']['targetId'], + 'page', self.address) def _onTargetDestroyed(self, **kwargs): """标签页关闭时执行""" @@ -101,13 +103,13 @@ class Browser(object): def tabs_count(self): """返回标签页数量""" j = self.run_cdp('Target.getTargets')['targetInfos'] # 不要改用get，避免卡死 - return len([i for i in j if i['type'] == 'page' and not i['url'].startswith('devtools://')]) + return len([i for i in j if i['type'] in ('page', 'webview') and not i['url'].startswith('devtools://')]) @property def tabs(self): """返回所有标签页id组成的列表""" j = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp，因为顺序不对 - return [i['id'] for i in j if i['type'] == 'page' and not i['url'].startswith('devtools://')] + return [i['id'] for i in j if i['type'] in ('page', 'webview') and not i['url'].startswith('devtools://')] @property def process_id(self): @@ -140,7 +142,7 @@ class Browser(object): :param tab_id: 标签页id :return: None """ - self.run_cdp('Target.closeTarget', targetId=tab_id) + self.run_cdp('Target.closeTarget', targetId=tab_id, _ignore=PageClosedError) def activate_tab(self, tab_id): """使标签页变为活动状态 @@ -162,8 +164,12 @@ class Browser(object): :param force: 是否立刻强制终止进程 :return: None """ - self.run_cdp('Browser.close') - self.driver.stop() + try: + self.run_cdp('Browser.close') + self.driver.stop() + except PageClosedError: + self.driver.stop() + return if force: ip, port = self.address.split(':') diff --git a/DrissionPage/_base/driver.py b/DrissionPage/_base/driver.py index 91a7465..caf2404 100644 --- a/DrissionPage/_base/driver.py +++ b/DrissionPage/_base/driver.py @@ -83,8 +83,8 @@ class Driver(object): return result except Empty: - if self.alert_flag and message['method'].startswith('Input.'): - return {'result': {'message': 'alert exists.'}} + if self.alert_flag and message['method'].startswith(('Input.', 'Runtime.')): + return {'error': {'message': 'alert exists.'}} if timeout is not None and perf_counter() > end_time: self.method_results.pop(ws_id, None) @@ -156,7 +156,7 @@ class Driver(object): if self._stopped.is_set(): return {'error': 'tab closed', 'type': 'tab_closed'} - timeout = kwargs.pop('_timeout', 10) + timeout = kwargs.pop('_timeout', 15) result = self._send({'method': _method, 'params': kwargs}, timeout=timeout) if result is None: return {'error': 'tab closed', 'type': 'tab_closed'} diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 5528b28..63f0ff0 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -603,6 +603,7 @@ class ChromiumElement(DrissionElement): if clear and vals not in ('\n', '\ue007'): self.clear(by_js=False) else: + self.wait.has_rect() self._input_focus() input_text_or_keys(self.page, vals) @@ -614,10 +615,10 @@ class ChromiumElement(DrissionElement): """ if by_js: self.run_js("this.value='';") + return - else: - self._input_focus() - self.input(('\ue009', 'a', '\ue017'), clear=False) + self._input_focus() + self.input(('\ue009', 'a', '\ue017'), clear=False) def _input_focus(self): """输入前使元素获取焦点""" @@ -1248,8 +1249,10 @@ def make_chromium_eles(page, node_ids=None, obj_ids=None, single=True, ele_only= if ele_only: continue else: - # todo: Node() - pass + if single: + return node['node']['nodeValue'] + else: + nodes.append(node['node']['nodeValue']) obj_id = page.run_cdp('DOM.resolveNode', nodeId=node_id)['object']['objectId'] ele = ChromiumElement(page, obj_id=obj_id, node_id=node_id, backend_id=node['node']['backendNodeId']) @@ -1269,8 +1272,10 @@ def make_chromium_eles(page, node_ids=None, obj_ids=None, single=True, ele_only= if ele_only: continue else: - # todo: Node - pass + if single: + return node['node']['nodeValue'] + else: + nodes.append(node['node']['nodeValue']) ele = ChromiumElement(page, obj_id=obj_id, node_id=node['node']['nodeId'], backend_id=node['node']['backendNodeId']) @@ -1356,7 +1361,7 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): try: if as_expr: res = page.run_cdp('Runtime.evaluate', expression=script, returnByValue=False, - awaitPromise=True, userGesture=True, _timeout=timeout) + awaitPromise=True, userGesture=True, _timeout=timeout, _ignore=AlertExistsError) else: args = args or () @@ -1364,7 +1369,7 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): script = f'function(){{{script}}}' res = page.run_cdp('Runtime.callFunctionOn', functionDeclaration=script, objectId=obj_id, arguments=[convert_argument(arg) for arg in args], returnByValue=False, - awaitPromise=True, userGesture=True, _timeout=timeout) + awaitPromise=True, userGesture=True, _timeout=timeout, _ignore=AlertExistsError) except ContextLostError: if is_page: diff --git a/DrissionPage/_functions/browser.py b/DrissionPage/_functions/browser.py index 4e8237c..f5f4cca 100644 --- a/DrissionPage/_functions/browser.py +++ b/DrissionPage/_functions/browser.py @@ -203,7 +203,7 @@ def test_connect(ip, port, timeout=30): tabs = requests_get(f'http://{ip}:{port}/json', timeout=10, headers={'Connection': 'close'}, proxies={'http': None, 'https': None}).json() for tab in tabs: - if tab['type'] == 'page': + if tab['type'] in ('page', 'webview'): return except Exception: sleep(.2) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index e9ba765..12eb386 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -89,7 +89,8 @@ class ChromiumBase(BasePage): if not tab_id: tabs = self.browser.driver.get(f'http://{self.address}/json').json() - tabs = [(i['id'], i['url']) for i in tabs if i['type'] == 'page' and not i['url'].startswith('devtools://')] + tabs = [(i['id'], i['url']) for i in tabs + if i['type'] in ('page', 'webview') and not i['url'].startswith('devtools://')] dialog = None if len(tabs) > 1: for k, t in enumerate(tabs): @@ -588,7 +589,8 @@ class ChromiumBase(BasePage): search_ids = [] try: - search_result = self.run_cdp_loaded('DOM.performSearch', query=loc, includeUserAgentShadowDOM=True) + search_result = self.run_cdp_loaded('DOM.performSearch', query=loc, _timeout=timeout, + includeUserAgentShadowDOM=True) count = search_result['resultCount'] search_ids.append(search_result['searchId']) except ContextLostError: @@ -615,7 +617,11 @@ class ChromiumBase(BasePage): ok = False try: - search_result = self.run_cdp_loaded('DOM.performSearch', query=loc, includeUserAgentShadowDOM=True) + timeout = end_time - perf_counter() + if timeout <= 0: + timeout = .5 + search_result = self.run_cdp_loaded('DOM.performSearch', query=loc, _timeout=timeout, + includeUserAgentShadowDOM=True) count = search_result['resultCount'] search_ids.append(search_result['searchId']) except ContextLostError: diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 9393e6a..7a45607 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -238,6 +238,13 @@ class TabWaiter(BaseWaiter): else: return True + def alert_closed(self): + """等待弹出框关闭""" + while not self._driver.states.has_alert: + sleep(.2) + while self._driver.states.has_alert: + sleep(.2) + class PageWaiter(TabWaiter): def __init__(self, page): @@ -415,6 +422,14 @@ class ElementWaiter(object): else: return False + def has_rect(self, timeout=None, raise_err=None): + """等待当前元素有大小及位置属性 + :param timeout: 超时时间，为None使用元素所在页面timeout属性 + :param raise_err: 等待失败时是否报错，为None时根据Settings设置 + :return: 是否等待成功 + """ + return self._wait_state('has_rect', True, timeout, raise_err, err_text='等待元素拥有大小及位置属性失败。') + def _wait_state(self, attr, mode=False, timeout=None, raise_err=None, err_text=None): """等待元素某个元素状态到达指定状态 :param attr: 状态名称 diff --git a/DrissionPage/_units/waiter.pyi b/DrissionPage/_units/waiter.pyi index 94803ef..9e0ada1 100644 --- a/DrissionPage/_units/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -24,9 +24,9 @@ class BaseWaiter(object): raise_err: bool = None) -> bool: ... def ele_displayed(self, - loc_or_ele: Union[str, tuple, ChromiumElement], - timeout: float = None, - raise_err: bool = None) -> bool: ... + loc_or_ele: Union[str, tuple, ChromiumElement], + timeout: float = None, + raise_err: bool = None) -> bool: ... def ele_hidden(self, loc_or_ele: Union[str, tuple, ChromiumElement], timeout: float = None, raise_err: bool = None) -> bool: ... @@ -60,6 +60,8 @@ class TabWaiter(BaseWaiter): def downloads_done(self, timeout: float = None, cancel_if_timeout: bool = True) -> bool: ... + def alert_closed(self) -> None: ... + class PageWaiter(TabWaiter): _driver: ChromiumPage = ... @@ -90,6 +92,8 @@ class ElementWaiter(object): def disabled(self, timeout: float = None, raise_err: bool = None) -> bool: ... + def has_rect(self, timeout: float = None, raise_err: bool = None) -> bool: ... + def disabled_or_deleted(self, timeout: float = None, raise_err: bool = None) -> bool: ... def stop_moving(self, gap: float = .1, timeout: float = None, raise_err: bool = None) -> bool: ... From 364bb3f9e05cd7e3f1a5d93db1b636f9aa7fab2a Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 22 Dec 2023 22:01:30 +0800 Subject: [PATCH 151/182] 4.0.0b27 --- DrissionPage/__init__.py | 2 +- DrissionPage/_elements/chromium_element.pyi | 8 +++++--- DrissionPage/_units/listener.pyi | 2 +- setup.py | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index f631dfe..ee8436a 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b26' +__version__ = '4.0.0b27' diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index f006917..b381132 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -12,6 +12,7 @@ from .._elements.session_element import SessionElement from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame from .._pages.chromium_page import ChromiumPage +from .._pages.chromium_tab import ChromiumTab from .._pages.web_page import WebPage from .._units.clicker import Clicker from .._units.rect import ElementRect @@ -314,11 +315,12 @@ def find_by_css(ele: ChromiumElement, selector: str, single: bool, timeout: float) -> Union[ChromiumElement, List[ChromiumElement], NoneElement]: ... -def make_chromium_ele(page: ChromiumBase, node_id: int = ..., obj_id: str = ...) \ - -> Union[ChromiumElement, ChromiumFrame, str]: ... +def make_chromium_ele(page: Union[ChromiumPage, WebPage, ChromiumTab, ChromiumFrame], + node_id: int = ..., + obj_id: str = ...) -> Union[ChromiumElement, ChromiumFrame, str]: ... -def make_chromium_eles(page: ChromiumBase, +def make_chromium_eles(page: Union[ChromiumPage, WebPage, ChromiumTab, ChromiumFrame], node_ids: Union[tuple, list] = None, obj_ids: Union[tuple, list] = None, single: bool = True, diff --git a/DrissionPage/_units/listener.pyi b/DrissionPage/_units/listener.pyi index c281c68..dccb3ec 100644 --- a/DrissionPage/_units/listener.pyi +++ b/DrissionPage/_units/listener.pyi @@ -199,7 +199,7 @@ class Response(object): def raw_body(self) -> str: ... @property - def body(self) -> Union[str, dict, bool]: ... + def body(self) -> Union[str, dict]: ... class ExtraInfo(object): diff --git a/setup.py b/setup.py index 82bafe4..a21ad1c 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b26", + version="4.0.0b27", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From d8a85d361e3da541baf6e0d2d7d80d945e6b8818 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 22 Dec 2023 22:10:00 +0800 Subject: [PATCH 152/182] 4.0.0b28 --- DrissionPage/__init__.py | 2 +- DrissionPage/_base/browser.py | 2 +- DrissionPage/_units/cookies_setter.py | 4 ++++ DrissionPage/_units/scroller.py | 4 ++++ setup.py | 2 +- 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index ee8436a..5ee4cc9 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b27' +__version__ = '4.0.0b28' diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index bf6c54b..881b794 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -5,10 +5,10 @@ """ from time import sleep, perf_counter -from errors import PageClosedError from .driver import BrowserDriver, Driver from .._functions.tools import stop_process_on_port, raise_error from .._units.downloader import DownloadManager +from ..errors import PageClosedError __ERROR__ = 'error' diff --git a/DrissionPage/_units/cookies_setter.py b/DrissionPage/_units/cookies_setter.py index 160077e..e22878a 100644 --- a/DrissionPage/_units/cookies_setter.py +++ b/DrissionPage/_units/cookies_setter.py @@ -1,4 +1,8 @@ # -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" from http.cookiejar import Cookie from .._functions.web import set_browser_cookies, set_session_cookies diff --git a/DrissionPage/_units/scroller.py b/DrissionPage/_units/scroller.py index 1a67c4c..6e3606e 100644 --- a/DrissionPage/_units/scroller.py +++ b/DrissionPage/_units/scroller.py @@ -1,4 +1,8 @@ # -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +""" from time import sleep, perf_counter diff --git a/setup.py b/setup.py index a21ad1c..0a417d0 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b27", + version="4.0.0b28", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From e86ac7517d853d9745cd54c7dda721e3ea4ab931 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 25 Dec 2023 18:00:13 +0800 Subject: [PATCH 153/182] =?UTF-8?q?4.0.0b29(+)=20=E5=A2=9E=E5=8A=A0disconn?= =?UTF-8?q?ect()=E6=96=B9=E6=B3=95=EF=BC=9B=20=E4=BC=98=E5=8C=96=E6=9F=A5?= =?UTF-8?q?=E6=89=BE=E5=85=83=E7=B4=A0=E9=80=BB=E8=BE=91=EF=BC=9B=20?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8A=A8=E4=BD=9C=E9=93=BE=E6=8C=89=E9=94=AE?= =?UTF-8?q?=E8=B6=85=E6=97=B6=E9=97=AE=E9=A2=98=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_base/base.py | 22 +- DrissionPage/_base/browser.py | 2 +- DrissionPage/_base/driver.py | 24 +- DrissionPage/_elements/chromium_element.py | 270 +++++++++++--------- DrissionPage/_elements/chromium_element.pyi | 16 +- DrissionPage/_functions/browser.py | 2 +- DrissionPage/_functions/keys.py | 2 +- DrissionPage/_functions/tools.py | 4 +- DrissionPage/_pages/chromium_base.py | 100 ++++---- DrissionPage/_pages/chromium_base.pyi | 2 + DrissionPage/_pages/chromium_frame.py | 23 +- DrissionPage/_pages/chromium_frame.pyi | 3 +- DrissionPage/_pages/chromium_page.py | 4 +- DrissionPage/_pages/chromium_tab.py | 8 +- DrissionPage/_pages/session_page.py | 2 +- DrissionPage/_pages/web_page.py | 8 +- DrissionPage/_units/actions.py | 2 - DrissionPage/_units/clicker.py | 4 +- 19 files changed, 263 insertions(+), 237 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 5ee4cc9..098d123 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b28' +__version__ = '4.0.0b29' diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index 8549d78..8f2598d 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -144,7 +144,7 @@ class DrissionElement(BaseElement): """返回直接子元素元素或节点组成的列表，可用查询语法筛选 :param filter_loc: 用于筛选的查询语法 :param index: 第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 直接子元素或节点文本组成的列表 """ @@ -174,7 +174,7 @@ class DrissionElement(BaseElement): """返回前面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 :param filter_loc: 用于筛选的查询语法 :param index: 前面第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 兄弟元素 """ @@ -194,7 +194,7 @@ class DrissionElement(BaseElement): """返回后面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 :param filter_loc: 用于筛选的查询语法 :param index: 后面第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 兄弟元素 """ @@ -214,7 +214,7 @@ class DrissionElement(BaseElement): """返回前面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 :param filter_loc: 用于筛选的查询语法 :param index: 前面第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素前面的某个元素或节点 """ @@ -234,7 +234,7 @@ class DrissionElement(BaseElement): """返回后面的一个兄弟元素，可用查询语法筛选，可指定返回筛选结果的第几个 :param filter_loc: 用于筛选的查询语法 :param index: 后面第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素后面的某个元素或节点 """ @@ -253,7 +253,7 @@ class DrissionElement(BaseElement): def children(self, filter_loc='', timeout=None, ele_only=True): """返回直接子元素元素或节点组成的列表，可用查询语法筛选 :param filter_loc: 用于筛选的查询语法 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 直接子元素或节点文本组成的列表 """ @@ -272,7 +272,7 @@ class DrissionElement(BaseElement): def prevs(self, filter_loc='', timeout=None, ele_only=True): """返回前面全部兄弟元素或节点组成的列表，可用查询语法筛选 :param filter_loc: 用于筛选的查询语法 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 兄弟元素或节点文本组成的列表 """ @@ -281,7 +281,7 @@ class DrissionElement(BaseElement): def nexts(self, filter_loc='', timeout=None, ele_only=True): """返回后面全部兄弟元素或节点组成的列表，可用查询语法筛选 :param filter_loc: 用于筛选的查询语法 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 兄弟元素或节点文本组成的列表 """ @@ -290,7 +290,7 @@ class DrissionElement(BaseElement): def befores(self, filter_loc='', timeout=None, ele_only=True): """返回后面全部兄弟元素或节点组成的列表，可用查询语法筛选 :param filter_loc: 用于筛选的查询语法 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素前面的元素或节点组成的列表 """ @@ -300,7 +300,7 @@ class DrissionElement(BaseElement): def afters(self, filter_loc='', timeout=None, ele_only=True): """返回前面全部兄弟元素或节点组成的列表，可用查询语法筛选 :param filter_loc: 用于筛选的查询语法 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素后面的元素或节点组成的列表 """ @@ -314,7 +314,7 @@ class DrissionElement(BaseElement): :param filter_loc: 用于筛选的查询语法 :param direction: 'following' 或 'preceding'，查找的方向 :param brother: 查找范围，在同级查找还是整个dom前后查找 - :param timeout: 查找等待时间 + :param timeout: 查找等待时间（秒） :return: 元素对象或字符串 """ if index is not None and index < 1: diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 881b794..2d8b891 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -160,7 +160,7 @@ class Browser(object): def quit(self, timeout=5, force=False): """关闭浏览器 - :param timeout: 等待浏览器关闭超时时间 + :param timeout: 等待浏览器关闭超时时间（秒） :param force: 是否立刻强制终止进程 :return: None """ diff --git a/DrissionPage/_base/driver.py b/DrissionPage/_base/driver.py index caf2404..46a52fb 100644 --- a/DrissionPage/_base/driver.py +++ b/DrissionPage/_base/driver.py @@ -3,13 +3,14 @@ @Author : g1879 @Contact : g1879@qq.com """ -from json import dumps, loads +from json import dumps, loads, JSONDecodeError from queue import Queue, Empty from threading import Thread, Event from time import perf_counter from requests import get -from websocket import WebSocketTimeoutException, WebSocketConnectionClosedException, create_connection +from websocket import (WebSocketTimeoutException, WebSocketConnectionClosedException, create_connection, + WebSocketException) class Driver(object): @@ -74,7 +75,7 @@ class Driver(object): except (OSError, WebSocketConnectionClosedException): self.method_results.pop(ws_id, None) - return None + return {'error': {'message': 'page closed'}} while not self._stopped.is_set(): try: @@ -102,7 +103,7 @@ class Driver(object): msg = loads(msg_json) except WebSocketTimeoutException: continue - except: + except (WebSocketException, OSError, WebSocketConnectionClosedException, JSONDecodeError): self.stop() return @@ -119,10 +120,10 @@ class Driver(object): if 'method' in msg: if msg['method'].startswith('Page.javascriptDialog'): self.alert_flag = msg['method'].endswith('Opening') - if msg['method'] in self.immediate_event_handlers: - function = self.immediate_event_handlers.get(msg['method']) - if function: - function(**msg['params']) + function = self.immediate_event_handlers.get(msg['method']) + if function: + Thread(target=function, kwargs=msg['params']).start() + # function(**msg['params']) else: self.event_queue.put(msg) @@ -154,12 +155,10 @@ class Driver(object): :return: 执行结果 """ if self._stopped.is_set(): - return {'error': 'tab closed', 'type': 'tab_closed'} + return {'error': 'page closed', 'type': 'page_closed'} timeout = kwargs.pop('_timeout', 15) result = self._send({'method': _method, 'params': kwargs}, timeout=timeout) - if result is None: - return {'error': 'tab closed', 'type': 'tab_closed'} if 'result' not in result and 'error' in result: return {'error': result['error']['message'], 'type': result.get('type', 'call_method_error'), 'method': _method, 'args': kwargs} @@ -169,8 +168,7 @@ class Driver(object): def start(self): """启动连接""" self._stopped.clear() - self._ws = create_connection(self._websocket_url, enable_multithread=True, - suppress_origin=True) + self._ws = create_connection(self._websocket_url, enable_multithread=True, suppress_origin=True) self._recv_th.start() self._handle_event_th.start() return True diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 63f0ff0..ce88ff5 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -80,7 +80,7 @@ class ChromiumElement(DrissionElement): def __call__(self, loc_or_str, timeout=None): """在内部查找元素 :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :return: ChromiumElement对象或属性、文本 """ return self.ele(loc_or_str, timeout) @@ -214,31 +214,31 @@ class ChromiumElement(DrissionElement): """ return super().parent(level_or_loc, index) - def child(self, filter_loc='', index=1, timeout=0, ele_only=True): + def child(self, filter_loc='', index=1, timeout=None, ele_only=True): """返回当前元素的一个符合条件的直接子元素，可用查询语法筛选，可指定返回筛选结果的第几个 :param filter_loc: 用于筛选的查询语法 :param index: 第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 直接子元素或节点文本 """ return super().child(filter_loc, index, timeout, ele_only=ele_only) - def prev(self, filter_loc='', index=1, timeout=0, ele_only=True): + def prev(self, filter_loc='', index=1, timeout=None, ele_only=True): """返回当前元素前面一个符合条件的同级元素，可用查询语法筛选，可指定返回筛选结果的第几个 :param filter_loc: 用于筛选的查询语法 :param index: 前面第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 兄弟元素或节点文本 """ return super().prev(filter_loc, index, timeout, ele_only=ele_only) - def next(self, filter_loc='', index=1, timeout=0, ele_only=True): + def next(self, filter_loc='', index=1, timeout=None, ele_only=True): """返回当前元素后面一个符合条件的同级元素，可用查询语法筛选，可指定返回筛选结果的第几个 :param filter_loc: 用于筛选的查询语法 :param index: 第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 兄弟元素或节点文本 """ @@ -249,7 +249,7 @@ class ChromiumElement(DrissionElement): 查找范围不限同级元素，而是整个DOM文档 :param filter_loc: 用于筛选的查询语法 :param index: 前面第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素前面的某个元素或节点 """ @@ -260,34 +260,34 @@ class ChromiumElement(DrissionElement): 查找范围不限同级元素，而是整个DOM文档 :param filter_loc: 用于筛选的查询语法 :param index: 第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素后面的某个元素或节点 """ return super().after(filter_loc, index, timeout, ele_only=ele_only) - def children(self, filter_loc='', timeout=0, ele_only=True): + def children(self, filter_loc='', timeout=None, ele_only=True): """返回当前元素符合条件的直接子元素或节点组成的列表，可用查询语法筛选 :param filter_loc: 用于筛选的查询语法 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 直接子元素或节点文本组成的列表 """ return super().children(filter_loc, timeout, ele_only=ele_only) - def prevs(self, filter_loc='', timeout=0, ele_only=True): + def prevs(self, filter_loc='', timeout=None, ele_only=True): """返回当前元素前面符合条件的同级元素或节点组成的列表，可用查询语法筛选 :param filter_loc: 用于筛选的查询语法 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 兄弟元素或节点文本组成的列表 """ return super().prevs(filter_loc, timeout, ele_only=ele_only) - def nexts(self, filter_loc='', timeout=0, ele_only=True): + def nexts(self, filter_loc='', timeout=None, ele_only=True): """返回当前元素后面符合条件的同级元素或节点组成的列表，可用查询语法筛选 :param filter_loc: 用于筛选的查询语法 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 兄弟元素或节点文本组成的列表 """ @@ -297,7 +297,7 @@ class ChromiumElement(DrissionElement): """返回文档中当前元素前面符合条件的元素或节点组成的列表，可用查询语法筛选 查找范围不限同级元素，而是整个DOM文档 :param filter_loc: 用于筛选的查询语法 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素前面的元素或节点组成的列表 """ @@ -307,7 +307,7 @@ class ChromiumElement(DrissionElement): """返回文档中当前元素后面符合条件的元素或节点组成的列表，可用查询语法筛选 查找范围不限同级元素，而是整个DOM文档 :param filter_loc: 用于筛选的查询语法 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素后面的元素或节点组成的列表 """ @@ -367,7 +367,7 @@ class ChromiumElement(DrissionElement): :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 - :param timeout: js超时时间，为None则使用页面timeouts.script设置 + :param timeout: js超时时间（秒），为None则使用页面timeouts.script设置 :return: 运行的结果 """ return run_js(self, script, as_expr, self.page.timeouts.script if timeout is None else timeout, args) @@ -377,7 +377,7 @@ class ChromiumElement(DrissionElement): :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 - :param timeout: js超时时间，为None则使用页面timeouts.script设置 + :param timeout: js超时时间（秒），为None则使用页面timeouts.script设置 :return: None """ from threading import Thread @@ -387,7 +387,7 @@ class ChromiumElement(DrissionElement): def ele(self, loc_or_str, timeout=None): """返回当前元素下级符合条件的第一个元素、属性或节点文本 :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 查找元素超时时间，默认与元素所在页面等待时间一致 + :param timeout: 查找元素超时时间（秒），默认与元素所在页面等待时间一致 :return: ChromiumElement对象或属性、文本 """ return self._ele(loc_or_str, timeout, method='ele()') @@ -395,7 +395,7 @@ class ChromiumElement(DrissionElement): def eles(self, loc_or_str, timeout=None): """返回当前元素下级所有符合条件的子元素、属性或节点文本 :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 查找元素超时时间，默认与元素所在页面等待时间一致 + :param timeout: 查找元素超时时间（秒），默认与元素所在页面等待时间一致 :return: ChromiumElement对象或属性、文本组成的列表 """ return self._ele(loc_or_str, timeout=timeout, single=False) @@ -429,7 +429,7 @@ class ChromiumElement(DrissionElement): def _find_elements(self, loc_or_str, timeout=None, single=True, relative=False, raise_err=None): """返回当前元素下级符合条件的子元素、属性或节点文本，默认返回第一个 :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 查找元素超时时间 + :param timeout: 查找元素超时时间（秒） :param single: True则返回第一个，False则返回全部 :param relative: WebPage用的表示是否相对定位的参数 :param raise_err: 找不到元素是是否抛出异常，为None时根据全局设置 @@ -449,7 +449,7 @@ class ChromiumElement(DrissionElement): def get_src(self, timeout=None, base64_to_bytes=True): """返回元素src资源，base64的可转为bytes返回，其它返回str - :param timeout: 等待资源加载的超时时间 + :param timeout: 等待资源加载的超时时间（秒） :param base64_to_bytes: 为True时，如果是base64数据，转换为bytes格式 :return: 资源内容 """ @@ -532,7 +532,7 @@ class ChromiumElement(DrissionElement): """保存图片或其它有src属性的元素的资源 :param path: 文件保存路径，为None时保存到当前文件夹 :param name: 文件名称，为None时从资源url获取 - :param timeout: 等待资源加载的超时时间 + :param timeout: 等待资源加载的超时时间（秒） :return: 返回保存路径 """ data = self.get_src(timeout=timeout) @@ -603,7 +603,6 @@ class ChromiumElement(DrissionElement): if clear and vals not in ('\n', '\ue007'): self.clear(by_js=False) else: - self.wait.has_rect() self._input_focus() input_text_or_keys(self.page, vals) @@ -793,7 +792,7 @@ class ShadowRoot(BaseElement): """在内部查找元素 例：ele2 = ele1('@id=ele_id') :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :return: 元素对象或属性、文本 """ return self.ele(loc_or_str, timeout) @@ -825,7 +824,7 @@ class ShadowRoot(BaseElement): :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 - :param timeout: js超时时间，为None则使用页面timeouts.script设置 + :param timeout: js超时时间（秒），为None则使用页面timeouts.script设置 :return: 运行的结果 """ return run_js(self, script, as_expr, self.page.timeouts.script if timeout is None else timeout, args) @@ -835,7 +834,7 @@ class ShadowRoot(BaseElement): :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 - :param timeout: js超时时间，为None则使用页面timeouts.script设置 + :param timeout: js超时时间（秒），为None则使用页面timeouts.script设置 :return: None """ from threading import Thread @@ -986,7 +985,7 @@ class ShadowRoot(BaseElement): def ele(self, loc_or_str, timeout=None): """返回当前元素下级符合条件的第一个元素 :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 查找元素超时时间，默认与元素所在页面等待时间一致 + :param timeout: 查找元素超时时间（秒），默认与元素所在页面等待时间一致 :return: ChromiumElement对象 """ return self._ele(loc_or_str, timeout, method='ele()') @@ -994,7 +993,7 @@ class ShadowRoot(BaseElement): def eles(self, loc_or_str, timeout=None): """返回当前元素下级所有符合条件的子元素 :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 查找元素超时时间，默认与元素所在页面等待时间一致 + :param timeout: 查找元素超时时间（秒），默认与元素所在页面等待时间一致 :return: ChromiumElement对象组成的列表 """ return self._ele(loc_or_str, timeout=timeout, single=False) @@ -1020,7 +1019,7 @@ class ShadowRoot(BaseElement): def _find_elements(self, loc_or_str, timeout=None, single=True, relative=False, raise_err=None): """返回当前元素下级符合条件的子元素、属性或节点文本，默认返回第一个 :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 查找元素超时时间 + :param timeout: 查找元素超时时间（秒） :param single: True则返回第一个，False则返回全部 :param relative: WebPage用的表示是否相对定位的参数 :param raise_err: 找不到元素是是否抛出异常，为None时根据全局设置 @@ -1030,42 +1029,47 @@ class ShadowRoot(BaseElement): if loc[0] == 'css selector' and str(loc[1]).startswith(':root'): loc = loc[0], loc[1][5:] - result = None - timeout = timeout if timeout is not None else self.page.timeout - end_time = perf_counter() + timeout - while perf_counter() <= end_time: + def do_find(): if loc[0] == 'css selector': if single: nod_id = self.page.run_cdp('DOM.querySelector', nodeId=self._node_id, selector=loc[1])['nodeId'] - result = make_chromium_ele(self.page, node_id=nod_id) if nod_id else NoneElement(self.page) + if nod_id: + r = make_chromium_ele(self.page, node_id=nod_id) + return None if r is False else r else: nod_ids = self.page.run_cdp('DOM.querySelectorAll', nodeId=self._node_id, selector=loc[1])['nodeId'] - result = make_chromium_eles(self.page, node_ids=nod_ids, single=False) + r = make_chromium_eles(self.page, node_ids=nod_ids, single=False) + return None if r is False else r else: eles = make_session_ele(self.html).eles(loc) if not eles: - result = NoneElement(self.page) if single else eles - continue + return None css = [i.css_path[61:] for i in eles] if single: node_id = self.page.run_cdp('DOM.querySelector', nodeId=self._node_id, selector=css[0])['nodeId'] - result = make_chromium_ele(self.page, node_id=node_id) if node_id else NoneElement(self.page) - + r = make_chromium_ele(self.page, node_id=node_id) + return None if r is False else r else: - result = [] - for i in css: - node_id = self.page.run_cdp('DOM.querySelector', nodeId=self._node_id, selector=i)['nodeId'] - if node_id: - result.append(make_chromium_ele(self.page, node_id=node_id)) + node_ids = [self.page.run_cdp('DOM.querySelector', nodeId=self._node_id, selector=i)['nodeId'] + for i in css] + if 0 in node_ids: + return None + r = make_chromium_eles(self.page, node_ids=node_ids, single=False) + return None if r is False else r - if result: - break + timeout = timeout if timeout is not None else self.page.timeout + end_time = perf_counter() + timeout + result = do_find() + while result is None and perf_counter() <= end_time: sleep(.1) + result = do_find() - return result + if result: + return result + return NoneElement(self.page) if single else [] def _get_node_id(self, obj_id): """返回元素node id""" @@ -1087,7 +1091,7 @@ def find_in_chromium_ele(ele, loc, single=True, timeout=None, relative=True): :param ele: ChromiumElement对象 :param loc: 元素定位元组 :param single: True则返回第一个，False则返回全部 - :param timeout: 查找元素超时时间 + :param timeout: 查找元素超时时间（秒） :param relative: WebPage用于标记是否相对定位使用 :return: 返回ChromiumElement元素或它们组成的列表 """ @@ -1119,45 +1123,57 @@ def find_by_xpath(ele, xpath, single, timeout, relative=True): :param ele: 在此元素中查找 :param xpath: 查找语句 :param single: 是否只返回第一个结果 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :param relative: 是否相对定位 :return: ChromiumElement或其组成的列表 """ type_txt = '9' if single else '7' node_txt = 'this.contentDocument' if ele.tag in __FRAME_ELEMENT__ and not relative else 'this' js = make_js_for_find_ele_by_xpath(xpath, type_txt, node_txt) - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, - returnByValue=False, awaitPromise=True, userGesture=True) - if r['result']['type'] == 'string': - return r['result']['value'] + ele.page.wait.load_complete() + + def do_find(): + res = ele.page.run_cdp('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, + returnByValue=False, awaitPromise=True, userGesture=True) + if res['result']['type'] == 'string': + return res['result']['value'] + if 'exceptionDetails' in res: + if 'The result is not a node set' in res['result']['description']: + js1 = make_js_for_find_ele_by_xpath(xpath, '1', node_txt) + res = ele.page.run_cdp('Runtime.callFunctionOn', functionDeclaration=js1, objectId=ele._obj_id, + returnByValue=False, awaitPromise=True, userGesture=True) + return res['result']['value'] + else: + raise SyntaxError(f'查询语句错误：\n{res}') + + if res['result']['subtype'] == 'null' or res['result']['description'] in ('NodeList(0)', 'Array(0)'): + return None + + if single: + r = make_chromium_ele(ele.page, obj_id=res['result']['objectId']) + return None if r is False else r - if 'exceptionDetails' in r: - if 'The result is not a node set' in r['result']['description']: - js = make_js_for_find_ele_by_xpath(xpath, '1', node_txt) - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, - returnByValue=False, awaitPromise=True, userGesture=True) - return r['result']['value'] else: - raise SyntaxError(f'查询语句错误：\n{r}') + # from pprint import pprint + # for i in ele.page.run_cdp('Runtime.getProperties', + # objectId=res['result']['objectId'], + # ownProperties=True)['result'][:-1]: + # pprint(i) + r = [make_chromium_ele(ele.page, obj_id=i['value']['objectId']) if i['value']['type'] == 'object' else + i['value']['value'] for i in ele.page.run_cdp('Runtime.getProperties', + objectId=res['result']['objectId'], + ownProperties=True)['result'][:-1]] + return None if not r or r is False in r else r end_time = perf_counter() + timeout - while ((r['result']['subtype'] == 'null' or r['result']['description'] in ('NodeList(0)', 'Array(0)')) - and perf_counter() < end_time): - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, - returnByValue=False, awaitPromise=True, userGesture=True) + result = do_find() + while result is None and perf_counter() < end_time: + sleep(.1) + result = do_find() - if single: - return NoneElement(ele.page) if r['result']['subtype'] == 'null' \ - else make_chromium_ele(ele.page, obj_id=r['result']['objectId']) - - if r['result']['description'] in ('NodeList(0)', 'Array(0)'): - return [] - else: - r = ele.page.run_cdp_loaded('Runtime.getProperties', objectId=r['result']['objectId'], - ownProperties=True)['result'] - return [make_chromium_ele(ele.page, obj_id=i['value']['objectId']) - if i['value']['type'] == 'object' else i['value']['value'] - for i in r[:-1]] + if result: + return result + return NoneElement(ele.page) if single else [] def find_by_css(ele, selector, single, timeout): @@ -1165,35 +1181,45 @@ def find_by_css(ele, selector, single, timeout): :param ele: 在此元素中查找 :param selector: 查找语句 :param single: 是否只返回第一个结果 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :return: ChromiumElement或其组成的列表 """ selector = selector.replace('"', r'\"') find_all = '' if single else 'All' node_txt = 'this.contentDocument' if ele.tag in ('iframe', 'frame', 'shadow-root') else 'this' js = f'function(){{return {node_txt}.querySelector{find_all}("{selector}");}}' - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, - returnByValue=False, awaitPromise=True, userGesture=True) + + ele.page.wait.load_complete() + + def do_find(): + res = ele.page.run_cdp('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, + returnByValue=False, awaitPromise=True, userGesture=True) + + if 'exceptionDetails' in res: + raise SyntaxError(f'查询语句错误：\n{res}') + if res['result']['subtype'] == 'null' or res['result']['description'] in ('NodeList(0)', 'Array(0)'): + return None + + if single: + r = make_chromium_ele(ele.page, obj_id=res['result']['objectId']) + return None if r is False else r + + else: + node_ids = [i['value']['objectId'] for i in ele.page.run_cdp('Runtime.getProperties', + objectId=res['result']['objectId'], + ownProperties=True)['result'][:-1]] + r = make_chromium_eles(ele.page, obj_ids=node_ids, single=False, ele_only=False) + return None if r is False else r end_time = perf_counter() + timeout - while ('exceptionDetails' in r or r['result']['subtype'] == 'null' or - r['result']['description'] in ('NodeList(0)', 'Array(0)')) and perf_counter() < end_time: - r = ele.page.run_cdp_loaded('Runtime.callFunctionOn', functionDeclaration=js, objectId=ele._obj_id, - returnByValue=False, awaitPromise=True, userGesture=True) + result = do_find() + while result is None and perf_counter() < end_time: + sleep(.1) + result = do_find() - if 'exceptionDetails' in r: - raise SyntaxError(f'查询语句错误：\n{r}') - - if single: - return NoneElement(ele.page) if r['result']['subtype'] == 'null' \ - else make_chromium_ele(ele.page, obj_id=r['result']['objectId']) - - if r['result']['description'] in ('NodeList(0)', 'Array(0)'): - return [] - else: - r = ele.page.run_cdp_loaded('Runtime.getProperties', objectId=r['result']['objectId'], - ownProperties=True)['result'] - return [make_chromium_ele(ele.page, obj_id=i['value']['objectId']) for i in r] + if result: + return result + return NoneElement(ele.page) if single else [] def make_chromium_ele(page, node_id=None, obj_id=None): @@ -1201,18 +1227,24 @@ def make_chromium_ele(page, node_id=None, obj_id=None): :param page: ChromiumPage对象 :param node_id: 元素的node id :param obj_id: 元素的object id - :return: ChromiumElement对象或ChromiumFrame对象 + :return: ChromiumElement对象或ChromiumFrame对象，生成失败返回False """ if node_id: - node = page.run_cdp('DOM.describeNode', nodeId=node_id) + node = page.driver.run('DOM.describeNode', nodeId=node_id) + if 'error' in node: + return False if node['node']['nodeName'] in ('#text', '#comment'): # todo: Node() return node['node']['nodeValue'] backend_id = node['node']['backendNodeId'] obj_id = page.run_cdp('DOM.resolveNode', nodeId=node_id)['object']['objectId'] + if 'error' in obj_id: + return False elif obj_id: - node = page.run_cdp('DOM.describeNode', objectId=obj_id) + node = page.driver.run('DOM.describeNode', objectId=obj_id) + if 'error' in node: + return False if node['node']['nodeName'] in ('#text', '#comment'): # todo: Node() return node['node']['nodeValue'] @@ -1220,12 +1252,12 @@ def make_chromium_ele(page, node_id=None, obj_id=None): node_id = node['node']['nodeId'] else: - raise ElementLostError + return False ele = ChromiumElement(page, obj_id=obj_id, node_id=node_id, backend_id=backend_id) if ele.tag in __FRAME_ELEMENT__: from .._pages.chromium_frame import ChromiumFrame - ele = ChromiumFrame(page, ele) + ele = ChromiumFrame(page, ele, node) return ele @@ -1237,14 +1269,16 @@ def make_chromium_eles(page, node_ids=None, obj_ids=None, single=True, ele_only= :param obj_ids: 元素的object id :param single: 是否获取但个元素 :param ele_only: 是否只要ele - :return: ChromiumElement对象或ChromiumFrame对象 + :return: ChromiumElement对象或ChromiumFrame对象，生成失败返回False """ nodes = [] if node_ids: for node_id in node_ids: if not node_id: return False - node = page.run_cdp('DOM.describeNode', nodeId=node_id) + node = page.driver.run('DOM.describeNode', nodeId=node_id) + if 'error' in node: + return False if node['node']['nodeName'] in ('#text', '#comment'): if ele_only: continue @@ -1254,11 +1288,14 @@ def make_chromium_eles(page, node_ids=None, obj_ids=None, single=True, ele_only= else: nodes.append(node['node']['nodeValue']) - obj_id = page.run_cdp('DOM.resolveNode', nodeId=node_id)['object']['objectId'] + obj_id = page.driver.run('DOM.resolveNode', nodeId=node_id) + if 'error' in obj_id: + return False + obj_id = obj_id['object']['objectId'] ele = ChromiumElement(page, obj_id=obj_id, node_id=node_id, backend_id=node['node']['backendNodeId']) if ele.tag in __FRAME_ELEMENT__: from .._pages.chromium_frame import ChromiumFrame - ele = ChromiumFrame(page, ele) + ele = ChromiumFrame(page, ele, node) if single: return ele nodes.append(ele) @@ -1267,7 +1304,9 @@ def make_chromium_eles(page, node_ids=None, obj_ids=None, single=True, ele_only= for obj_id in obj_ids: if not obj_id: return False - node = page.run_cdp('DOM.describeNode', objectId=obj_id) + node = page.driver.run('DOM.describeNode', objectId=obj_id) + if 'error' in node: + return False if node['node']['nodeName'] in ('#text', '#comment'): if ele_only: continue @@ -1281,7 +1320,7 @@ def make_chromium_eles(page, node_ids=None, obj_ids=None, single=True, ele_only= backend_id=node['node']['backendNodeId']) if ele.tag in __FRAME_ELEMENT__: from .._pages.chromium_frame import ChromiumFrame - ele = ChromiumFrame(page, ele) + ele = ChromiumFrame(page, ele, node) if single: return ele nodes.append(ele) @@ -1336,7 +1375,7 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): :param page_or_ele: 页面对象或元素对象 :param script: js文本 :param as_expr: 是否作为表达式运行，为True时args无效 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :return: js执行结果 """ @@ -1409,16 +1448,17 @@ def parse_js_result(page, ele, result): elif class_name == 'HTMLDocument': return result else: - return make_chromium_eles(page, obj_ids=(result['objectId'],)) + r = make_chromium_ele(page, obj_id=result['objectId']) + if r is False: + raise ElementLostError + return r elif sub_type == 'array': - r = page.run_cdp('Runtime.getProperties', objectId=result['objectId'], - ownProperties=True)['result'] + r = page.run_cdp('Runtime.getProperties', objectId=result['objectId'], ownProperties=True)['result'] return [parse_js_result(page, ele, result=i['value']) for i in r[:-1]] elif 'objectId' in result and result['className'].lower() == 'object': # dict - r = page.run_cdp('Runtime.getProperties', objectId=result['objectId'], - ownProperties=True)['result'] + r = page.run_cdp('Runtime.getProperties', objectId=result['objectId'], ownProperties=True)['result'] return {i['name']: parse_js_result(page, ele, result=i['value']) for i in r} else: diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index b381132..44b6c8d 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -99,19 +99,19 @@ class ChromiumElement(DrissionElement): def child(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, - timeout: float = 0, + timeout: float = None, ele_only: bool = True) -> Union[ChromiumElement, str, NoneElement]: ... def prev(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, - timeout: float = 0, + timeout: float = None, ele_only: bool = True) -> Union[ChromiumElement, str, NoneElement]: ... def next(self, filter_loc: Union[tuple, str, int] = '', index: int = 1, - timeout: float = 0, + timeout: float = None, ele_only: bool = True) -> Union[ChromiumElement, str, NoneElement]: ... def before(self, @@ -128,17 +128,17 @@ class ChromiumElement(DrissionElement): def children(self, filter_loc: Union[tuple, str] = '', - timeout: float = 0, + timeout: float = None, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... def prevs(self, filter_loc: Union[tuple, str] = '', - timeout: float = 0, + timeout: float = None, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... def nexts(self, filter_loc: Union[tuple, str] = '', - timeout: float = 0, + timeout: float = None, ele_only: bool = True) -> List[Union[ChromiumElement, str]]: ... def befores(self, @@ -315,12 +315,12 @@ def find_by_css(ele: ChromiumElement, selector: str, single: bool, timeout: float) -> Union[ChromiumElement, List[ChromiumElement], NoneElement]: ... -def make_chromium_ele(page: Union[ChromiumPage, WebPage, ChromiumTab, ChromiumFrame], +def make_chromium_ele(page: Union[ChromiumBase, ChromiumPage, WebPage, ChromiumTab, ChromiumFrame], node_id: int = ..., obj_id: str = ...) -> Union[ChromiumElement, ChromiumFrame, str]: ... -def make_chromium_eles(page: Union[ChromiumPage, WebPage, ChromiumTab, ChromiumFrame], +def make_chromium_eles(page: Union[ChromiumBase, ChromiumPage, WebPage, ChromiumTab, ChromiumFrame], node_ids: Union[tuple, list] = None, obj_ids: Union[tuple, list] = None, single: bool = True, diff --git a/DrissionPage/_functions/browser.py b/DrissionPage/_functions/browser.py index f5f4cca..33f0df6 100644 --- a/DrissionPage/_functions/browser.py +++ b/DrissionPage/_functions/browser.py @@ -194,7 +194,7 @@ def test_connect(ip, port, timeout=30): """测试浏览器是否可用 :param ip: 浏览器ip :param port: 浏览器端口 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :return: None """ end_time = perf_counter() + timeout diff --git a/DrissionPage/_functions/keys.py b/DrissionPage/_functions/keys.py index 7e4a300..8c0a243 100644 --- a/DrissionPage/_functions/keys.py +++ b/DrissionPage/_functions/keys.py @@ -419,7 +419,7 @@ def send_key(page, modifier, key): 'unmodifiedText': text, 'location': description['location'], 'isKeypad': description['location'] == 3, - '_ignore': AlertExistsError, '_timeout': 1} + '_ignore': AlertExistsError} page.run_cdp('Input.dispatchKeyEvent', **data) data['type'] = 'keyUp' diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index 6ab1229..4db38b5 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -192,7 +192,7 @@ def wait_until(page, condition, timeout=10, poll=0.1, raise_err=True): """等待返回值不为False或空，直到超时 :param page: DrissionPage对象 :param condition: 等待条件，返回值不为False则停止等待 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :param poll: 轮询间隔 :param raise_err: 是否抛出异常 :return: DP Element or bool @@ -267,7 +267,7 @@ def raise_error(result, ignore=None): 'No node with given id found', 'Node with given id does not belong to the document', 'No node found for given backend id'): r = ElementLostError() - elif error in ('tab closed', 'No target with given id found'): + elif error in ('page closed', 'No target with given id found'): r = PageClosedError() elif error == 'timeout': r = TimeoutError(f'超时。\n错误：{result["error"]}\nmethod：{result["method"]}\nargs：{result["args"]}\n' diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 12eb386..eb21aee 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -38,7 +38,7 @@ class ChromiumBase(BasePage): """ :param address: 浏览器 ip:port :param tab_id: 要控制的标签页id，不指定默认为激活的 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） """ super().__init__() self._is_loading = None @@ -147,7 +147,7 @@ class ChromiumBase(BasePage): def _get_document(self, timeout=10): """获取页面文档 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :return: 是否获取成功 """ if self._debug: @@ -282,7 +282,7 @@ class ChromiumBase(BasePage): """在内部查找元素 例：ele = page('@id=ele_id') :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :return: ChromiumElement对象 """ return self.ele(loc_or_str, timeout) @@ -468,7 +468,7 @@ class ChromiumBase(BasePage): :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 - :param timeout: js超时时间，为None则使用页面timeouts.script设置 + :param timeout: js超时时间（秒），为None则使用页面timeouts.script设置 :return: 运行的结果 """ return run_js(self, script, as_expr, self.timeouts.script if timeout is None else timeout, args) @@ -478,7 +478,7 @@ class ChromiumBase(BasePage): :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 - :param timeout: js超时时间，为None则使用页面timeouts.script设置 + :param timeout: js超时时间（秒），为None则使用页面timeouts.script设置 :return: 运行的结果 """ self.wait.load_complete() @@ -489,7 +489,7 @@ class ChromiumBase(BasePage): :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 - :param timeout: js超时时间，为None则使用页面timeouts.script设置 + :param timeout: js超时时间（秒），为None则使用页面timeouts.script设置 :return: None """ from threading import Thread @@ -502,7 +502,7 @@ class ChromiumBase(BasePage): :param show_errmsg: 是否显示和抛出异常 :param retry: 重试次数 :param interval: 重试间隔（秒） - :param timeout: 连接超时时间 + :param timeout: 连接超时时间（秒） :return: 目标url是否可用 """ retry, interval = self._before_connect(url, retry, interval) @@ -531,7 +531,7 @@ class ChromiumBase(BasePage): def ele(self, loc_or_ele, timeout=None): """获取第一个符合条件的元素对象 :param loc_or_ele: 定位符或元素对象 - :param timeout: 查找超时时间 + :param timeout: 查找超时时间（秒） :return: ChromiumElement对象 """ return self._ele(loc_or_ele, timeout=timeout, method='ele()') @@ -539,7 +539,7 @@ class ChromiumBase(BasePage): def eles(self, loc_or_str, timeout=None): """获取所有符合条件的元素对象 :param loc_or_str: 定位符或元素对象 - :param timeout: 查找超时时间 + :param timeout: 查找超时时间（秒） :return: ChromiumElement对象组成的列表 """ return self._ele(loc_or_str, timeout=timeout, single=False) @@ -568,7 +568,7 @@ class ChromiumBase(BasePage): def _find_elements(self, loc_or_ele, timeout=None, single=True, relative=False, raise_err=None): """执行元素查找 :param loc_or_ele: 定位符或元素对象 - :param timeout: 查找超时时间 + :param timeout: 查找超时时间（秒） :param single: 是否只返回第一个 :param relative: WebPage用的表示是否相对定位的参数 :param raise_err: 找不到元素是是否抛出异常，为None时根据全局设置 @@ -581,56 +581,39 @@ class ChromiumBase(BasePage): else: raise ValueError('loc_or_str参数只能是tuple、str、ChromiumElement类型。') - ok = False - nodeIds = None - + self.wait.load_complete() timeout = timeout if timeout is not None else self.timeout end_time = perf_counter() + timeout search_ids = [] - try: - search_result = self.run_cdp_loaded('DOM.performSearch', query=loc, _timeout=timeout, - includeUserAgentShadowDOM=True) - count = search_result['resultCount'] - search_ids.append(search_result['searchId']) - except ContextLostError: - search_result = None - count = 0 + timeout = .5 if timeout <= 0 else timeout + result = self.driver.run('DOM.performSearch', query=loc, _timeout=timeout, includeUserAgentShadowDOM=True) + if not result or __ERROR__ in result: + num = 0 + else: + num = result['resultCount'] + search_ids.append(result['searchId']) while True: - if count > 0: - count = 1 if single else count - try: - nodeIds = self.run_cdp_loaded('DOM.getSearchResults', searchId=search_result['searchId'], - fromIndex=0, toIndex=count) - if nodeIds['nodeIds'][0] != 0: - ok = True - - except Exception: - pass - - if ok: - r = make_chromium_eles(self, node_ids=nodeIds['nodeIds'], single=single) - if r is not False: - break - else: - ok = False - - try: - timeout = end_time - perf_counter() - if timeout <= 0: - timeout = .5 - search_result = self.run_cdp_loaded('DOM.performSearch', query=loc, _timeout=timeout, - includeUserAgentShadowDOM=True) - count = search_result['resultCount'] - search_ids.append(search_result['searchId']) - except ContextLostError: - pass + if num > 0: + num = 1 if single else num + nIds = self._driver.run('DOM.getSearchResults', searchId=result['searchId'], fromIndex=0, toIndex=num) + if __ERROR__ not in nIds: + if nIds['nodeIds'][0] != 0: + r = make_chromium_eles(self, node_ids=nIds['nodeIds'], single=single) + if r is not False: + break if perf_counter() >= end_time: return NoneElement(self) if single else [] sleep(.1) + timeout = end_time - perf_counter() + timeout = .5 if timeout <= 0 else timeout + result = self.driver.run('DOM.performSearch', query=loc, _timeout=timeout, includeUserAgentShadowDOM=True) + if not result or __ERROR__ not in result: + num = result['resultCount'] + search_ids.append(result['searchId']) for _id in search_ids: self._driver.run('DOM.discardSearchResults', searchId=_id) @@ -712,7 +695,7 @@ class ChromiumBase(BasePage): def get_frame(self, loc_ind_ele, timeout=None): """获取页面中一个frame对象，可传入定位符、iframe序号、ChromiumFrame对象，序号从1开始 :param loc_ind_ele: 定位符、iframe序号、ChromiumFrame对象 - :param timeout: 查找元素超时时间 + :param timeout: 查找元素超时时间（秒） :return: ChromiumFrame对象 """ if isinstance(loc_ind_ele, str): @@ -752,7 +735,7 @@ class ChromiumBase(BasePage): def get_frames(self, loc=None, timeout=None): """获取所有符合条件的frame对象 :param loc: 定位符，为None时返回所有 - :param timeout: 查找超时时间 + :param timeout: 查找超时时间（秒） :return: ChromiumFrame对象组成的列表 """ loc = loc or 'xpath://*[name()="iframe" or name()="frame"]' @@ -839,6 +822,11 @@ class ChromiumBase(BasePage): if cookies: self.run_cdp_loaded('Network.clearBrowserCookies') + def disconnect(self): + """断开与页面的连接，不关闭页面""" + if self._driver: + self.driver.stop() + def handle_alert(self, accept=True, send=None, timeout=None, next_one=False): r = self._handle_alert(accept=accept, send=send, timeout=timeout, next_one=next_one) while self._has_alert: @@ -849,7 +837,7 @@ class ChromiumBase(BasePage): """处理提示框，可以自动等待提示框出现 :param accept: True表示确认，False表示取消，其它值不会按按钮但依然返回文本值 :param send: 处理prompt提示框时可输入文本 - :param timeout: 等待提示框出现的超时时间，为None则使用self.timeout属性的值 + :param timeout: 等待提示框出现的超时时间（秒），为None则使用self.timeout属性的值 :param next_one: 是否处理下一个出现的提示框，为True时timeout参数无效 :return: 提示框内容文本，未等到提示框则返回False """ @@ -901,12 +889,9 @@ class ChromiumBase(BasePage): def _wait_loaded(self, timeout=None): """等待页面加载完成，超时触发停止加载 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :return: 是否成功，超时返回False """ - if self._load_mode == 'none': - return True - timeout = timeout if timeout is not None else self.timeouts.page_load end_time = perf_counter() + timeout while perf_counter() < end_time: @@ -948,10 +933,11 @@ class ChromiumBase(BasePage): :param times: 重试次数 :param interval: 重试间隔（秒） :param show_errmsg: 是否抛出异常 - :param timeout: 连接超时时间 + :param timeout: 连接超时时间（秒） :return: 是否成功，返回None表示不确定 """ err = None + self._is_loading = True timeout = timeout if timeout is not None else self.timeouts.page_load for t in range(times + 1): err = None diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 98c98fb..5781211 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -224,6 +224,8 @@ class ChromiumBase(BasePage): def clear_cache(self, session_storage: bool = True, local_storage: bool = True, cache: bool = True, cookies: bool = True) -> None: ... + def disconnect(self) -> None: ... + def handle_alert(self, accept: bool = True, send: str = None, timeout: float = None, next_one: bool = False) -> Union[str, False]: ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 873f331..f2d8820 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -19,10 +19,11 @@ from ..errors import ContextLostError, ElementLostError, GetDocumentError, PageC class ChromiumFrame(ChromiumBase): - def __init__(self, page, ele): + def __init__(self, page, ele, info=None): """ :param page: frame所在的页面对象 :param ele: frame所在元素 + :param info: frame所在元素信息 """ page_type = str(type(page)) if 'ChromiumPage' in page_type or 'WebPage' in page_type: @@ -41,7 +42,7 @@ class ChromiumFrame(ChromiumBase): self._states = None self._reloading = False - node = page.run_cdp('DOM.describeNode', backendNodeId=ele._backend_id)['node'] + node = info['node'] if not info else page.run_cdp('DOM.describeNode', backendNodeId=ele._backend_id)['node'] self._frame_id = node['frameId'] if self._is_inner_frame(): self._is_diff_domain = False @@ -65,7 +66,7 @@ class ChromiumFrame(ChromiumBase): """在内部查找元素 例：ele2 = ele1('@id=ele_id') :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :return: ChromiumElement对象或属性、文本 """ return self.ele(loc_or_str, timeout) @@ -159,7 +160,7 @@ class ChromiumFrame(ChromiumBase): def _get_document(self, timeout=10): """刷新cdp使用的document数据 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :return: 是否获取成功 """ if self._is_reading: @@ -392,7 +393,7 @@ class ChromiumFrame(ChromiumBase): :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 - :param timeout: js超时时间，为None则使用页面timeouts.script设置 + :param timeout: js超时时间（秒），为None则使用页面timeouts.script设置 :return: 运行的结果 """ if script.startswith('this.scrollIntoView'): @@ -412,7 +413,7 @@ class ChromiumFrame(ChromiumBase): """返回当前元素前面一个符合条件的同级元素，可用查询语法筛选，可指定返回筛选结果的第几个 :param filter_loc: 用于筛选的查询语法 :param index: 前面第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 同级元素或节点 """ @@ -422,7 +423,7 @@ class ChromiumFrame(ChromiumBase): """返回当前元素后面一个符合条件的同级元素，可用查询语法筛选，可指定返回筛选结果的第几个 :param filter_loc: 用于筛选的查询语法 :param index: 后面第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 同级元素或节点 """ @@ -433,7 +434,7 @@ class ChromiumFrame(ChromiumBase): 查找范围不限同级元素，而是整个DOM文档 :param filter_loc: 用于筛选的查询语法 :param index: 前面第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素前面的某个元素或节点 """ @@ -444,7 +445,7 @@ class ChromiumFrame(ChromiumBase): 查找范围不限同级元素，而是整个DOM文档 :param filter_loc: 用于筛选的查询语法 :param index: 后面第几个查询结果，1开始 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 本元素后面的某个元素或节点 """ @@ -453,7 +454,7 @@ class ChromiumFrame(ChromiumBase): def prevs(self, filter_loc='', timeout=0, ele_only=True): """返回当前元素前面符合条件的同级元素或节点组成的列表，可用查询语法筛选 :param filter_loc: 用于筛选的查询语法 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 同级元素或节点文本组成的列表 """ @@ -462,7 +463,7 @@ class ChromiumFrame(ChromiumBase): def nexts(self, filter_loc='', timeout=0, ele_only=True): """返回当前元素后面符合条件的同级元素或节点组成的列表，可用查询语法筛选 :param filter_loc: 用于筛选的查询语法 - :param timeout: 查找节点的超时时间 + :param timeout: 查找节点的超时时间（秒） :param ele_only: 是否只获取元素，为False时把文本、注释节点也纳入 :return: 同级元素或节点文本组成的列表 """ diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index 8dc1380..f2cf074 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -24,7 +24,8 @@ class ChromiumFrame(ChromiumBase): def __init__(self, page: Union[ChromiumPage, WebPage, ChromiumTab, ChromiumFrame], - ele: ChromiumElement): + ele: ChromiumElement, + info: dict = None): self._page: ChromiumPage = ... self._target_page: ChromiumBase = ... self.tab: ChromiumTab = ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 2c53511..73777de 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -25,7 +25,7 @@ class ChromiumPage(ChromiumBase): """ :param addr_or_opts: 浏览器地址:端口、ChromiumOptions对象或端口数字（int） :param tab_id: 要控制的标签页id，不指定默认为激活的 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） """ addr_or_opts = addr_or_opts or addr_driver_opts self._page = self @@ -242,7 +242,7 @@ class ChromiumPage(ChromiumBase): def quit(self, timeout=5, force=True): """关闭浏览器 - :param timeout: 等待浏览器关闭超时时间 + :param timeout: 等待浏览器关闭超时时间（秒） :param force: 关闭超时是否强制终止进程 :return: None """ diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 983ee4a..dacf435 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -79,7 +79,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): """在内部查找元素 例：ele = page('@id=ele_id') :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :return: 子元素对象 """ if self._mode == 'd': @@ -221,7 +221,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): def ele(self, loc_or_ele, timeout=None): """返回第一个符合条件的元素、属性或节点文本 :param loc_or_ele: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 - :param timeout: 查找元素超时时间，默认与页面等待时间一致 + :param timeout: 查找元素超时时间（秒），默认与页面等待时间一致 :return: 元素对象或属性、文本节点文本 """ if self._mode == 's': @@ -232,7 +232,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): def eles(self, loc_or_str, timeout=None): """返回页面中所有符合条件的元素、属性或节点文本 :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 查找元素超时时间，默认与页面等待时间一致 + :param timeout: 查找元素超时时间（秒），默认与页面等待时间一致 :return: 元素对象或属性、文本组成的列表 """ if self._mode == 's': @@ -345,7 +345,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): def _find_elements(self, loc_or_ele, timeout=None, single=True, relative=False, raise_err=None): """返回页面中符合条件的元素、属性或节点文本，默认返回第一个 :param loc_or_ele: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 - :param timeout: 查找元素超时时间，d模式专用 + :param timeout: 查找元素超时时间（秒），d模式专用 :param single: True则返回第一个，False则返回全部 :param relative: WebPage用的表示是否相对定位的参数 :param raise_err: 找不到元素是是否抛出异常，为None时根据全局设置 diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index 9dfcbf5..0170a1c 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -25,7 +25,7 @@ class SessionPage(BasePage): def __init__(self, session_or_options=None, timeout=None): """ :param session_or_options: Session对象或SessionOptions对象 - :param timeout: 连接超时时间，为None时从ini文件读取 + :param timeout: 连接超时时间（秒），为None时从ini文件读取 """ super(SessionPage, SessionPage).__init__(self) self._headers = None diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index a62d30c..22940e2 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -18,7 +18,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def __init__(self, mode='d', timeout=None, chromium_options=None, session_or_options=None, driver_or_options=None): """初始化函数 :param mode: 'd' 或 's'，即driver模式和session模式 - :param timeout: 超时时间，d模式时为寻找元素时间，s模式时为连接时间，默认10秒 + :param timeout: 超时时间（秒），d模式时为寻找元素时间，s模式时为连接时间，默认10秒 :param chromium_options: Driver对象，只使用s模式时应传入False :param session_or_options: Session对象或SessionOptions对象，只使用d模式时应传入False """ @@ -40,7 +40,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): """在内部查找元素 例：ele = page('@id=ele_id') :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 超时时间 + :param timeout: 超时时间（秒） :return: 子元素对象 """ if self._mode == 'd': @@ -182,7 +182,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def ele(self, loc_or_ele, timeout=None): """返回第一个符合条件的元素、属性或节点文本 :param loc_or_ele: 元素的定位信息，可以是元素对象，loc元组，或查询字符串 - :param timeout: 查找元素超时时间，默认与页面等待时间一致 + :param timeout: 查找元素超时时间（秒），默认与页面等待时间一致 :return: 元素对象或属性、文本节点文本 """ if self._mode == 's': @@ -193,7 +193,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): def eles(self, loc_or_str, timeout=None): """返回页面中所有符合条件的元素、属性或节点文本 :param loc_or_str: 元素的定位信息，可以是loc元组，或查询字符串 - :param timeout: 查找元素超时时间，默认与页面等待时间一致 + :param timeout: 查找元素超时时间（秒），默认与页面等待时间一致 :return: 元素对象或属性、文本组成的列表 """ if self._mode == 's': diff --git a/DrissionPage/_units/actions.py b/DrissionPage/_units/actions.py index 4f4b282..d8a90c3 100644 --- a/DrissionPage/_units/actions.py +++ b/DrissionPage/_units/actions.py @@ -252,7 +252,6 @@ class Actions: return self data = self._get_key_data(key, 'keyDown') - data['_timeout'] = 1 data['_ignore'] = AlertExistsError self.page.run_cdp('Input.dispatchKeyEvent', **data) return self @@ -268,7 +267,6 @@ class Actions: return self data = self._get_key_data(key, 'keyUp') - data['_timeout'] = 1 data['_ignore'] = AlertExistsError self.page.run_cdp('Input.dispatchKeyEvent', **data) return self diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index 9a85215..6565e50 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -21,7 +21,7 @@ class Clicker(object): """点击元素 如果遇到遮挡，可选择是否用js点击 :param by_js: 是否用js点击，为None时先用模拟点击，遇到遮挡改用js，为True时直接用js点击，为False时只用模拟点击 - :param timeout: 模拟点击的超时时间，等待元素可见、可用、进入视口 + :param timeout: 模拟点击的超时时间（秒），等待元素可见、可用、进入视口 :param wait_stop: 是否等待元素运动结束再执行点击 :return: 是否点击成功 """ @@ -30,7 +30,7 @@ class Clicker(object): def left(self, by_js=False, timeout=1.5, wait_stop=True): """点击元素，可选择是否用js点击 :param by_js: 是否用js点击，为None时先用模拟点击，遇到遮挡改用js，为True时直接用js点击，为False时只用模拟点击 - :param timeout: 模拟点击的超时时间，等待元素可见、可用、进入视口 + :param timeout: 模拟点击的超时时间（秒），等待元素可见、可用、进入视口 :param wait_stop: 是否等待元素运动结束再执行点击 :return: 是否点击成功 """ From 465ef1573491b418cc679a1591734abd7347d31b Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 25 Dec 2023 21:16:21 +0800 Subject: [PATCH 154/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dtype()=E7=BB=84?= =?UTF-8?q?=E5=90=88=E9=94=AE=E6=97=A0=E6=95=88=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_units/actions.py | 10 ++++++++-- DrissionPage/_units/actions.pyi | 9 +++++---- requirements.txt | 4 ++-- setup.py | 10 +++++----- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/DrissionPage/_units/actions.py b/DrissionPage/_units/actions.py index d8a90c3..11c9ef7 100644 --- a/DrissionPage/_units/actions.py +++ b/DrissionPage/_units/actions.py @@ -276,11 +276,17 @@ class Actions: :param keys: 要按下的按键，特殊字符和多个文本可用list或tuple传入 :return: self """ + modifiers = [] for i in keys: for character in i: self.key_down(character) - sleep(.05) - self.key_up(character) + if character in ('\ue009', '\ue008', '\ue00a', '\ue03d'): + modifiers.append(character) + else: + sleep(.01) + self.key_up(character) + for m in modifiers: + self.key_up(m) return self def input(self, text): diff --git a/DrissionPage/_units/actions.pyi b/DrissionPage/_units/actions.pyi index 982ead1..8a56363 100644 --- a/DrissionPage/_units/actions.pyi +++ b/DrissionPage/_units/actions.pyi @@ -36,7 +36,8 @@ KEYS = Literal['NULL', 'CANCEL', 'HELP', 'BACKSPACE', 'BACK_SPACE', 'meta', 'g', 'h', 'j', 'k', 'l', ';', '\'', 'z', 'x', 'c', 'v', 'b', 'n', 'm', ',', '.', '/', '~', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '_', '+', 'Q', 'W', 'E', 'R', 'T', 'Y', 'U', 'I', 'O', 'P', '{', '}', 'A', 'S', 'D', -'F', 'G', 'H', 'J', 'K', 'L', ':', '"', 'Z', 'X', 'C', 'V', 'B', 'N', 'M', '<', '>', '?'] +'F', 'G', 'H', 'J', 'K', 'L', ':', '"', 'Z', 'X', 'C', 'V', 'B', 'N', 'M', '<', '>', '?' +] class Actions: @@ -89,11 +90,11 @@ class Actions: def right(self, pixel: int) -> Actions: ... - def key_down(self, key: KEYS) -> Actions: ... + def key_down(self, key: Union[KEYS, str]) -> Actions: ... - def key_up(self, key: KEYS) -> Actions: ... + def key_up(self, key: Union[KEYS, str]) -> Actions: ... - def type(self, keys: Union[str, list, tuple]) -> Actions: ... + def type(self, keys: Union[KEYS, str, list, tuple]) -> Actions: ... def input(self, text: Any) -> Actions: ... diff --git a/requirements.txt b/requirements.txt index b195908..57cde23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ requests lxml cssselect -DownloadKit>=2.0.0b1 -websocket-client +DownloadKit>=2.0.0b2 +websocket-client>=1.7.0 click tldextract psutil \ No newline at end of file diff --git a/setup.py b/setup.py index 0a417d0..66aeaf1 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b28", + version="4.0.0b29", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", @@ -22,19 +22,19 @@ setup( 'lxml', 'requests', 'cssselect', - 'DownloadKit>=2.0.0b1', - 'websocket-client', + 'DownloadKit>=2.0.0b2', + 'websocket-client>=1.7.0', 'click', 'tldextract', 'psutil' ], classifiers=[ - "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.8", "Development Status :: 4 - Beta", "Topic :: Utilities", "License :: OSI Approved :: BSD License", ], - python_requires='>=3.6', + python_requires='>=3.8', entry_points={ 'console_scripts': [ 'dp = DrissionPage.commons.cli:main', From 23881d03a34eb3bfb7a89a8cb2bc1ee736956038 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 26 Dec 2023 15:51:32 +0800 Subject: [PATCH 155/182] =?UTF-8?q?4.0.0b30=E4=BC=98=E5=8C=96=5Freload()?= =?UTF-8?q?=EF=BC=9B=E4=BF=AE=E5=A4=8Dtype()=E3=80=81select()=E5=92=8Cclos?= =?UTF-8?q?e()=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_base/browser.py | 3 +- DrissionPage/_pages/chromium_base.py | 2 +- DrissionPage/_pages/chromium_frame.py | 42 +++++++++++---------------- DrissionPage/_units/selector.py | 2 +- requirements.txt | 2 +- setup.py | 8 ++--- 7 files changed, 27 insertions(+), 34 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 098d123..8631fbb 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b29' +__version__ = '4.0.0b30' diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 2d8b891..2171bd1 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -92,8 +92,9 @@ class Browser(object): :param cmd_args: 参数 :return: 执行的结果 """ + ignore = cmd_args.pop('_ignore', None) r = self._driver.run(cmd, **cmd_args) - return r if __ERROR__ not in r else raise_error(r) + return r if __ERROR__ not in r else raise_error(r, ignore) @property def driver(self): diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index eb21aee..025ab24 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -143,7 +143,7 @@ class ChromiumBase(BasePage): self._driver.set_callback('Page.loadEventFired', self._onLoadEventFired) self._driver.set_callback('Page.frameStoppedLoading', self._onFrameStoppedLoading) self._driver.set_callback('Page.frameAttached', self._onFrameAttached) - self._driver.set_callback('Page.frameDetached', self._onFrameDetached) + self._driver.set_callback('Page.frameDetached', self._onFrameDetached, immediate=True) def _get_document(self, timeout=10): """获取页面文档 diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index f2d8820..b8f09cb 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -95,7 +95,7 @@ class ChromiumFrame(ChromiumBase): except: self.browser.driver.get(f'http://{self.address}/json') super()._driver_init(tab_id) - self._driver.set_callback('Inspector.detached', self._onInspectorDetached) + self._driver.set_callback('Inspector.detached', self._onInspectorDetached, immediate=True) def _reload(self): """重新获取document""" @@ -136,19 +136,23 @@ class ChromiumFrame(ChromiumBase): self._is_diff_domain = True if self._listener: self._listener._to_target(node['frameId'], self.address, self) - super().__init__(self.address, node['frameId'], self._target_page.timeout) end_time = perf_counter() + self.timeouts.page_load - while perf_counter() < end_time: - try: - obj_id = super().run_js('document;', as_expr=True)['objectId'] - self.doc_ele = ChromiumElement(self, obj_id=obj_id) - break - except Exception as e: - sleep(.1) - if self._debug: - print(f'获取doc失败，重试 {e}') - else: - raise GetDocumentError + super().__init__(self.address, node['frameId'], self._target_page.timeout) + timeout = end_time - perf_counter() + if timeout <= 0: + timeout = .5 + self._wait_loaded(timeout) + # while perf_counter() < end_time: + # try: + # obj_id = super().run_js('document;', as_expr=True)['objectId'] + # self.doc_ele = ChromiumElement(self, obj_id=obj_id) + # break + # except Exception as e: + # sleep(.1) + # if self._debug: + # print(f'获取doc失败，重试 {e}') + # else: + # raise GetDocumentError self._debug = debug self.driver._debug = d_debug @@ -201,26 +205,14 @@ class ChromiumFrame(ChromiumBase): def _onInspectorDetached(self, **kwargs): """异域转同域或退出""" - if self._debug: - print(f'{self._frame_id}触发InspectorDetached') - self._reload() - if self._debug: - print(f'{self._frame_id}执行InspectorDetached完毕') - def _onFrameDetached(self, **kwargs): """同域变异域""" self.browser._frames.pop(kwargs['frameId'], None) if kwargs['frameId'] == self._frame_id: - if self._debug: - print(f'{self._frame_id}触发FrameDetached') - self._reload() - if self._debug: - print(f'{self._frame_id}执行FrameDetached完毕') - # ----------挂件---------- @property diff --git a/DrissionPage/_units/selector.py b/DrissionPage/_units/selector.py index 8d945d2..e7a84d3 100644 --- a/DrissionPage/_units/selector.py +++ b/DrissionPage/_units/selector.py @@ -252,7 +252,7 @@ class SelectElement(object): """ if isinstance(option, (list, tuple, set)): if not self.is_multi and len(option) > 1: - raise TypeError("只能对多项选框执行多选。") + option = option[:1] for o in option: o.run_js(f'this.selected={mode};') self._dispatch_change() diff --git a/requirements.txt b/requirements.txt index 57cde23..e02789b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ requests lxml cssselect -DownloadKit>=2.0.0b2 +DownloadKit>=2.0.0b3 websocket-client>=1.7.0 click tldextract diff --git a/setup.py b/setup.py index 66aeaf1..784d826 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b29", + version="4.0.0b30", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", @@ -22,19 +22,19 @@ setup( 'lxml', 'requests', 'cssselect', - 'DownloadKit>=2.0.0b2', + 'DownloadKit>=2.0.0b3', 'websocket-client>=1.7.0', 'click', 'tldextract', 'psutil' ], classifiers=[ - "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.6", "Development Status :: 4 - Beta", "Topic :: Utilities", "License :: OSI Approved :: BSD License", ], - python_requires='>=3.8', + python_requires='>=3.6', entry_points={ 'console_scripts': [ 'dp = DrissionPage.commons.cli:main', From 8c870c1c3e673617fafb19acd35fae081d3b91c7 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 26 Dec 2023 17:11:48 +0800 Subject: [PATCH 156/182] =?UTF-8?q?=E6=88=AA=E5=9B=BE=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E9=87=8D=E5=91=BD=E5=90=8D=EF=BC=9B=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E5=BD=95=E5=B1=8F=E6=9C=AA=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 2 +- DrissionPage/_pages/chromium_base.py | 5 +++-- DrissionPage/_pages/chromium_page.py | 2 +- DrissionPage/_units/screencast.py | 4 +++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 2171bd1..6556480 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -123,7 +123,7 @@ class Browser(object): :param url: 要匹配url的文本 :param tab_type: tab类型，可用列表输入多个 :param single: 是否返回首个结果的id，为False返回所有信息 - :return: tab id或tab dict + :return: tab id或tab列表 """ tabs = self._driver.get(f'http://{self.address}/json').json() # 不要改用cdp diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 025ab24..0d51807 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -5,6 +5,7 @@ """ from json import loads, JSONDecodeError from os.path import sep +from pathlib import Path from re import findall, match from threading import Thread from time import perf_counter, sleep @@ -16,7 +17,7 @@ from .._elements.none_element import NoneElement from .._elements.session_element import make_session_ele from .._functions.locator import get_loc, is_loc from .._functions.settings import Settings -from .._functions.tools import get_usable_path, raise_error +from .._functions.tools import raise_error from .._functions.web import location_in_viewport from .._units.actions import Actions from .._units.listener import Listener @@ -1021,7 +1022,7 @@ class ChromiumBase(BasePage): name = f'{name}.jpg' path = f'{path}{sep}{name}' - path = get_usable_path(path) + path = Path(path) pic_type = path.suffix.lower() pic_type = 'jpeg' if pic_type == '.jpg' else pic_type[1:] diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 73777de..e01419a 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -162,7 +162,7 @@ class ChromiumPage(ChromiumBase): :param url: 要匹配url的文本 :param tab_type: tab类型，可用列表输入多个 :param single: 是否返回首个结果的id，为False返回所有信息 - :return: tab id或tab dict + :return: tab id或tab列表 """ return self._browser.find_tabs(title, url, tab_type, single) diff --git a/DrissionPage/_units/screencast.py b/DrissionPage/_units/screencast.py index 4b927a8..086e14c 100644 --- a/DrissionPage/_units/screencast.py +++ b/DrissionPage/_units/screencast.py @@ -33,7 +33,9 @@ class Screencast(object): self.set_save_path(save_path) if self._path is None: raise ValueError('save_path必须设置。') - clean_folder(self._path) + tmp = self._path / 'tmp' + tmp.mkdir(parents=True, exist_ok=True) + clean_folder(tmp) if self._mode.startswith('frugal'): self._page.driver.set_callback('Page.screencastFrame', self._onScreencastFrame) self._page.run_cdp('Page.startScreencast', everyNthFrame=1, quality=100) From bd47aee4cabcc33c4f328deacf680f11d7b12742 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 27 Dec 2023 07:23:38 +0800 Subject: [PATCH 157/182] =?UTF-8?q?=E5=AE=8C=E5=96=84Driver=E9=80=BB?= =?UTF-8?q?=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/driver.py | 12 +++++++++--- DrissionPage/_base/driver.pyi | 2 ++ DrissionPage/_pages/chromium_frame.py | 12 +++++++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/DrissionPage/_base/driver.py b/DrissionPage/_base/driver.py index 46a52fb..eae9b8e 100644 --- a/DrissionPage/_base/driver.py +++ b/DrissionPage/_base/driver.py @@ -6,7 +6,7 @@ from json import dumps, loads, JSONDecodeError from queue import Queue, Empty from threading import Thread, Event -from time import perf_counter +from time import perf_counter, sleep from requests import get from websocket import (WebSocketTimeoutException, WebSocketConnectionClosedException, create_connection, @@ -104,7 +104,7 @@ class Driver(object): except WebSocketTimeoutException: continue except (WebSocketException, OSError, WebSocketConnectionClosedException, JSONDecodeError): - self.stop() + self._stop() return if self._debug: @@ -174,6 +174,13 @@ class Driver(object): return True def stop(self): + """中断连接""" + self._stop() + while self._handle_event_th.is_alive() or self._recv_th.is_alive(): + sleep(.1) + return True + + def _stop(self): """中断连接""" if self._stopped.is_set(): return False @@ -195,7 +202,6 @@ class Driver(object): self.event_handlers.clear() self.method_results.clear() self.event_queue.queue.clear() - return True def set_callback(self, event, callback, immediate=False): """绑定cdp event和回调方法 diff --git a/DrissionPage/_base/driver.pyi b/DrissionPage/_base/driver.pyi index 6ea93b4..2d23d91 100644 --- a/DrissionPage/_base/driver.pyi +++ b/DrissionPage/_base/driver.pyi @@ -54,6 +54,8 @@ class Driver(object): def stop(self) -> bool: ... + def _stop(self) -> None: ... + def set_callback(self, event: str, callback: Union[Callable, None], immediate: bool = False) -> None: ... def __str__(self) -> str: ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index b8f09cb..434942a 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -15,7 +15,7 @@ from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter from .._units.states import FrameStates from .._units.waiter import FrameWaiter -from ..errors import ContextLostError, ElementLostError, GetDocumentError, PageClosedError, JavaScriptError +from ..errors import ContextLostError, ElementLostError, PageClosedError, JavaScriptError class ChromiumFrame(ChromiumBase): @@ -205,13 +205,19 @@ class ChromiumFrame(ChromiumBase): def _onInspectorDetached(self, **kwargs): """异域转同域或退出""" - self._reload() + try: + self._reload() + except PageClosedError: + pass def _onFrameDetached(self, **kwargs): """同域变异域""" self.browser._frames.pop(kwargs['frameId'], None) if kwargs['frameId'] == self._frame_id: - self._reload() + try: + self._reload() + except PageClosedError: + pass # ----------挂件---------- From eaad58da9ec948247652a187b7a15ad17b0516e7 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 27 Dec 2023 17:21:52 +0800 Subject: [PATCH 158/182] =?UTF-8?q?4.0.0b31=E5=AE=8C=E5=96=84Driver?= =?UTF-8?q?=E7=9A=84stop()=E9=80=BB=E8=BE=91=EF=BC=9B=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E5=BD=95=E5=83=8F=E4=BF=9D=E5=AD=98=E9=80=BB=E8=BE=91=EF=BC=9B?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E5=AF=B9=E8=B1=A1=E5=A2=9E=E5=8A=A0save()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_base/driver.py | 33 ++++++++++++++++++--------- DrissionPage/_pages/chromium_base.py | 25 ++++++++++++++++---- DrissionPage/_pages/chromium_base.pyi | 4 ++++ DrissionPage/_pages/chromium_frame.py | 10 ++------ DrissionPage/_pages/chromium_page.py | 17 ++++++++++---- DrissionPage/_pages/chromium_page.pyi | 3 +++ DrissionPage/_pages/chromium_tab.py | 18 +++++++++++---- DrissionPage/_pages/chromium_tab.pyi | 3 +++ DrissionPage/_pages/session_page.py | 6 ++--- DrissionPage/_pages/web_page.py | 6 ++--- DrissionPage/_units/listener.py | 4 +++- DrissionPage/_units/screencast.py | 33 ++++++++++++++++++--------- DrissionPage/_units/screencast.pyi | 1 + setup.py | 2 +- 15 files changed, 114 insertions(+), 53 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 8631fbb..9d09b17 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b30' +__version__ = '4.0.0b31' diff --git a/DrissionPage/_base/driver.py b/DrissionPage/_base/driver.py index eae9b8e..2412b6f 100644 --- a/DrissionPage/_base/driver.py +++ b/DrissionPage/_base/driver.py @@ -12,6 +12,8 @@ from requests import get from websocket import (WebSocketTimeoutException, WebSocketConnectionClosedException, create_connection, WebSocketException) +from ..errors import PageClosedError + class Driver(object): def __init__(self, tab_id, tab_type, address): @@ -122,7 +124,7 @@ class Driver(object): self.alert_flag = msg['method'].endswith('Opening') function = self.immediate_event_handlers.get(msg['method']) if function: - Thread(target=function, kwargs=msg['params']).start() + Thread(target=run_function, args=(function, msg['params'])).start() # function(**msg['params']) else: self.event_queue.put(msg) @@ -159,11 +161,13 @@ class Driver(object): timeout = kwargs.pop('_timeout', 15) result = self._send({'method': _method, 'params': kwargs}, timeout=timeout) - if 'result' not in result and 'error' in result: + if result is None: + return {'error': {'message': 'page closed'}} + elif 'result' not in result and 'error' in result: return {'error': result['error']['message'], 'type': result.get('type', 'call_method_error'), 'method': _method, 'args': kwargs} - - return result['result'] + else: + return result['result'] def start(self): """启动连接""" @@ -190,14 +194,14 @@ class Driver(object): self._ws.close() self._ws = None - while not self.event_queue.empty(): - event = self.event_queue.get_nowait() - function = self.event_handlers.get(event['method']) - if function: - try: + try: + while not self.event_queue.empty(): + event = self.event_queue.get_nowait() + function = self.event_handlers.get(event['method']) + if function: function(**event['params']) - except: - pass + except: + pass self.event_handlers.clear() self.method_results.clear() @@ -249,3 +253,10 @@ class BrowserDriver(Driver): def stop(self): super().stop() self.browser._on_quit() + + +def run_function(function, kwargs): + try: + function(**kwargs) + except PageClosedError: + pass diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 0d51807..1a41abc 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -479,7 +479,7 @@ class ChromiumBase(BasePage): :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 - :param timeout: js超时时间（秒），为None则使用页面timeouts.script设置 + :param timeout: js超时时间（秒），为None则使用页面timeouts.script属性值 :return: 运行的结果 """ self.wait.load_complete() @@ -490,7 +490,7 @@ class ChromiumBase(BasePage): :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 - :param timeout: js超时时间（秒），为None则使用页面timeouts.script设置 + :param timeout: js超时时间（秒），为None则使用页面timeouts.script属性值 :return: None """ from threading import Thread @@ -501,9 +501,9 @@ class ChromiumBase(BasePage): """访问url :param url: 目标url :param show_errmsg: 是否显示和抛出异常 - :param retry: 重试次数 - :param interval: 重试间隔（秒） - :param timeout: 连接超时时间（秒） + :param retry: 重试次数，为None时使用页面对象retry_times属性值 + :param interval: 重试间隔（秒），为None时使用页面对象retry_interval属性值 + :param timeout: 连接超时时间（秒），为None时使用页面对象timeouts.page_load属性值 :return: 目标url是否可用 """ retry, interval = self._before_connect(url, retry, interval) @@ -1154,3 +1154,18 @@ def close_privacy_dialog(page, tid): except: pass + + +def get_mhtml(page, path=None, name=None): + """把当前页面保存为mhtml文件 + :param page: 要保存的页面对象 + :param path: 保存路径，为None保存在当前路径 + :param name: 文件名，为None则用title属性值 + :return: mhtml文本 + """ + r = page.run_cdp('Page.captureSnapshot')['data'] + path = path or '.' + name = name or page.title + with open(f'{path}{sep}{name}.mhtml', 'w', encoding='utf-8') as f: + f.write(r) + return r diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 5781211..abef890 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -6,6 +6,7 @@ from pathlib import Path from typing import Union, Tuple, List, Any, Optional, Literal +from .chromium_tab import ChromiumTab from .._base.base import BasePage from .._base.browser import Browser from .._base.driver import Driver @@ -263,3 +264,6 @@ class Alert(object): self.handle_next: Optional[bool] = ... self.next_text: str = ... self.auto: Optional[bool] = ... + + +def get_mhtml(page: Union[ChromiumPage, ChromiumTab], path: Union[str, Path] = None, name: str = None) -> str: ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 434942a..f27e0d1 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -205,19 +205,13 @@ class ChromiumFrame(ChromiumBase): def _onInspectorDetached(self, **kwargs): """异域转同域或退出""" - try: - self._reload() - except PageClosedError: - pass + self._reload() def _onFrameDetached(self, **kwargs): """同域变异域""" self.browser._frames.pop(kwargs['frameId'], None) if kwargs['frameId'] == self._frame_id: - try: - self._reload() - except PageClosedError: - pass + self._reload() # ----------挂件---------- diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index e01419a..9de3d29 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -11,7 +11,7 @@ from requests import get from .._base.browser import Browser from .._functions.browser import connect_browser from .._configs.chromium_options import ChromiumOptions -from .._pages.chromium_base import ChromiumBase, Timeout +from .._pages.chromium_base import ChromiumBase, get_mhtml, Timeout from .._pages.chromium_tab import ChromiumTab from .._units.setter import ChromiumPageSetter from .._units.waiter import PageWaiter @@ -66,12 +66,13 @@ class ChromiumPage(ChromiumBase): ws = get(f'http://{self._chromium_options.address}/json/version', headers={'Connection': 'close'}) if not ws: raise BrowserConnectError('\n浏览器连接失败，请检查是否启用全局代理。如是，须设置不代理127.0.0.1地址。') - except : + ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] + except KeyError: + raise BrowserConnectError('浏览器版本太旧，请升级。') + except: raise BrowserConnectError('\n浏览器连接失败，请检查是否启用全局代理。如是，须设置不代理127.0.0.1地址。') - ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] self._browser = Browser(self._chromium_options.address, ws, self) - if (is_exist and self._chromium_options._headless is False and 'headless' in self._browser.run_cdp('Browser.getVersion')['userAgent'].lower()): self._browser.quit(3) @@ -140,6 +141,14 @@ class ChromiumPage(ChromiumBase): """返回浏览器进程id""" return self.browser.process_id + def save(self, path=None, name=None): + """把当前页面保存为mhtml文件 + :param path: 保存路径，为None保存在当前路径 + :param name: 文件名，为None则用title属性值 + :return: mhtml文本 + """ + return get_mhtml(self, path, name) + def get_tab(self, id_or_num=None): """获取一个标签页对象 :param id_or_num: 要获取的标签页id或序号，为None时获取当前tab，序号不是视觉排列顺序，而是激活顺序 diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index 49e519b..ac29e10 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -3,6 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ +from pathlib import Path from typing import Union, Tuple, List, Optional from .._base.browser import Browser @@ -54,6 +55,8 @@ class ChromiumPage(ChromiumBase): @property def set(self) -> ChromiumPageSetter: ... + def save(self, path: Union[str, Path] = None, name: str = None) -> str: ... + def get_tab(self, tab_id: Union[str, ChromiumTab, int] = None) -> ChromiumTab: ... def find_tabs(self, title: str = None, url: str = None, diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index dacf435..4038abd 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -6,9 +6,9 @@ from copy import copy from .._base.base import BasePage -from .._functions.web import set_session_cookies, set_browser_cookies from .._configs.session_options import SessionOptions -from .._pages.chromium_base import ChromiumBase +from .._functions.web import set_session_cookies, set_browser_cookies +from .._pages.chromium_base import ChromiumBase, get_mhtml from .._pages.session_page import SessionPage from .._units.setter import TabSetter, WebPageTabSetter from .._units.waiter import TabWaiter @@ -58,6 +58,14 @@ class ChromiumTab(ChromiumBase): self._wait = TabWaiter(self) return self._wait + def save(self, path=None, name=None): + """把当前页面保存为mhtml文件 + :param path: 保存路径，为None保存在当前路径 + :param name: 文件名，为None则用title属性值 + :return: mhtml文本 + """ + return get_mhtml(self, path, name) + def __repr__(self): return f'' @@ -191,9 +199,9 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): """跳转到一个url :param url: 目标url :param show_errmsg: 是否显示和抛出异常 - :param retry: 重试次数 - :param interval: 重试间隔（秒） - :param timeout: 连接超时时间（秒） + :param retry: 重试次数，为None时使用页面对象retry_times属性值 + :param interval: 重试间隔（秒），为None时使用页面对象retry_interval属性值 + :param timeout: 连接超时时间（秒），为None时使用页面对象timeouts.page_load属性值 :param kwargs: 连接参数，s模式专用 :return: url是否可用，d模式返回None时表示不确定 """ diff --git a/DrissionPage/_pages/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi index 327b037..ec3798a 100644 --- a/DrissionPage/_pages/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -3,6 +3,7 @@ @Author : g1879 @Contact : g1879@qq.com """ +from pathlib import Path from typing import Union, Tuple, Any, List, Optional from requests import Session, Response @@ -41,6 +42,8 @@ class ChromiumTab(ChromiumBase): @property def wait(self) -> TabWaiter: ... + def save(self, path: Union[str, Path] = None, name: str = None) -> str: ... + class WebPageTab(SessionPage, ChromiumTab): def __init__(self, page: WebPage, tab_id: str): diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index 0170a1c..0dce0af 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -141,9 +141,9 @@ class SessionPage(BasePage): """用get方式跳转到url，可输入文件路径 :param url: 目标url，可指定本地文件路径 :param show_errmsg: 是否显示和抛出异常 - :param retry: 重试次数 - :param interval: 重试间隔（秒） - :param timeout: 连接超时时间（秒） + :param retry: 重试次数，为None时使用页面对象retry_times属性值 + :param interval: 重试间隔（秒），为None时使用页面对象retry_interval属性值 + :param timeout: 连接超时时间（秒），为None时使用页面对象timeout属性值 :param kwargs: 连接参数 :return: url是否可用 """ diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 22940e2..e20931b 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -152,9 +152,9 @@ class WebPage(SessionPage, ChromiumPage, BasePage): """跳转到一个url :param url: 目标url :param show_errmsg: 是否显示和抛出异常 - :param retry: 重试次数 - :param interval: 重试间隔（秒） - :param timeout: 连接超时时间（秒） + :param retry: 重试次数，为None时使用页面对象retry_times属性值 + :param interval: 重试间隔（秒），为None时使用页面对象retry_interval属性值 + :param timeout: 连接超时时间（秒），为None时使用页面对象timeouts.page_load属性值 :param kwargs: 连接参数，s模式专用 :return: url是否可用，d模式返回None时表示不确定 """ diff --git a/DrissionPage/_units/listener.py b/DrissionPage/_units/listener.py index d18759f..d576480 100644 --- a/DrissionPage/_units/listener.py +++ b/DrissionPage/_units/listener.py @@ -295,7 +295,9 @@ class Listener(object): packet = self._request_ids.get(rid) if packet: r = self._driver.run('Network.getResponseBody', requestId=rid) - if 'body' in r: + if 'error' in r: + return + elif 'body' in r: packet._raw_body = r['body'] packet._base64_body = r['base64Encoded'] else: diff --git a/DrissionPage/_units/screencast.py b/DrissionPage/_units/screencast.py index 086e14c..ab60b83 100644 --- a/DrissionPage/_units/screencast.py +++ b/DrissionPage/_units/screencast.py @@ -6,16 +6,17 @@ from base64 import b64decode from os.path import sep from pathlib import Path +from random import randint +from shutil import rmtree from threading import Thread from time import sleep, time -from .._functions.tools import clean_folder - class Screencast(object): def __init__(self, page): self._page = page self._path = None + self._tmp_path = None self._running = False self._enable = False self._mode = 'video' @@ -33,9 +34,11 @@ class Screencast(object): self.set_save_path(save_path) if self._path is None: raise ValueError('save_path必须设置。') - tmp = self._path / 'tmp' - tmp.mkdir(parents=True, exist_ok=True) - clean_folder(tmp) + + if self._mode in ('frugal_video', 'video'): + self._tmp_path = self._path / f'screencast_tmp_{time()}_{randint(0, 100)}' + self._tmp_path.mkdir(parents=True, exist_ok=True) + if self._mode.startswith('frugal'): self._page.driver.set_callback('Page.screencastFrame', self._onScreencastFrame) self._page.run_cdp('Page.startScreencast', everyNthFrame=1, quality=100) @@ -45,7 +48,7 @@ class Screencast(object): self._enable = True Thread(target=self._run).start() - else: + else: # js模式 js = ''' async function () { stream = await navigator.mediaDevices.getDisplayMedia({video: true, audio: true}) @@ -104,7 +107,7 @@ class Screencast(object): if self._mode.endswith('imgs'): return str(Path(self._path).absolute()) - if not str(video_name).isascii() or not str(self._path).isascii(): + if not str(self._path).isascii(): raise TypeError('转换成视频仅支持英文路径和文件名。') try: @@ -113,7 +116,7 @@ class Screencast(object): except ModuleNotFoundError: raise ModuleNotFoundError('请先安装cv2，pip install opencv-python') - pic_list = Path(self._path).glob('*.jpg') + pic_list = Path(self._tmp_path or self._path).glob('*.jpg') img = imread(str(next(pic_list))) imgInfo = img.shape size = (imgInfo[1], imgInfo[0]) @@ -124,7 +127,8 @@ class Screencast(object): img = imread(str(i)) videoWrite.write(img) - clean_folder(self._path, ignore=(name,)) + rmtree(self._tmp_path) + self._tmp_path = None return f'{self._path}{sep}{name}' def set_save_path(self, save_path=None): @@ -142,14 +146,16 @@ class Screencast(object): def _run(self): """非节俭模式运行方法""" self._running = True + path = self._tmp_path or self._path while self._enable: - self._page.get_screenshot(path=self._path, name=f'{time()}.jpg') + self._page.get_screenshot(path=path, name=f'{time()}.jpg') sleep(.04) self._running = False def _onScreencastFrame(self, **kwargs): """节俭模式运行方法""" - with open(f'{self._path}\\{kwargs["metadata"]["timestamp"]}.jpg', 'wb') as f: + path = self._tmp_path or self._path + with open(f'{path}{sep}{kwargs["metadata"]["timestamp"]}.jpg', 'wb') as f: f.write(b64decode(kwargs['data'])) self._page.run_cdp('Page.screencastFrameAck', sessionId=kwargs['sessionId']) @@ -159,16 +165,21 @@ class ScreencastMode(object): self._screencast = screencast def video_mode(self): + """持续视频模式，生成的视频没有声音""" self._screencast._mode = 'video' def frugal_video_mode(self): + """设置节俭视频模式，页面有变化时才录制，生成的视频没有声音""" self._screencast._mode = 'frugal_video' def js_video_mode(self): + """设置使用js录制视频模式，可生成有声音的视频，但需要手动启动""" self._screencast._mode = 'js_video' def frugal_imgs_mode(self): + """设置节俭视频模式，页面有变化时才截图""" self._screencast._mode = 'frugal_imgs' def imgs_mode(self): + """设置图片模式，持续对页面进行截图""" self._screencast._mode = 'imgs' diff --git a/DrissionPage/_units/screencast.pyi b/DrissionPage/_units/screencast.pyi index 6c23592..c185375 100644 --- a/DrissionPage/_units/screencast.pyi +++ b/DrissionPage/_units/screencast.pyi @@ -13,6 +13,7 @@ class Screencast(object): def __init__(self, page: ChromiumBase): self._page: ChromiumBase = ... self._path: Path = ... + self._tmp_path: Path = ... self._running: bool = ... self._enable: bool = ... self._mode: str = ... diff --git a/setup.py b/setup.py index 784d826..713e0f2 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b30", + version="4.0.0b31", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From 221204b2f29d7edea4ba514b0479a143b5b818c8 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 27 Dec 2023 17:29:42 +0800 Subject: [PATCH 159/182] =?UTF-8?q?4.0.0b31=E5=AE=8C=E5=96=84Driver?= =?UTF-8?q?=E7=9A=84stop()=E9=80=BB=E8=BE=91=EF=BC=9B=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E5=BD=95=E5=83=8F=E4=BF=9D=E5=AD=98=E9=80=BB=E8=BE=91=EF=BC=9B?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E5=AF=B9=E8=B1=A1=E5=A2=9E=E5=8A=A0save()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 1a41abc..30830ac 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -17,7 +17,7 @@ from .._elements.none_element import NoneElement from .._elements.session_element import make_session_ele from .._functions.locator import get_loc, is_loc from .._functions.settings import Settings -from .._functions.tools import raise_error +from .._functions.tools import raise_error, make_valid_name from .._functions.web import location_in_viewport from .._units.actions import Actions from .._units.listener import Listener @@ -1165,7 +1165,8 @@ def get_mhtml(page, path=None, name=None): """ r = page.run_cdp('Page.captureSnapshot')['data'] path = path or '.' - name = name or page.title + Path(path).mkdir(parents=True, exist_ok=True) + name = make_valid_name(name or page.title) with open(f'{path}{sep}{name}.mhtml', 'w', encoding='utf-8') as f: f.write(r) return r From 9a6bd9c2b4fcb10deddc5593cbc8c1973dc409e1 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 27 Dec 2023 17:38:44 +0800 Subject: [PATCH 160/182] =?UTF-8?q?=E5=BE=AE=E8=B0=83=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 2 +- DrissionPage/_pages/chromium_frame.py | 4 ++-- DrissionPage/_pages/chromium_page.py | 2 +- DrissionPage/_pages/chromium_tab.py | 4 ++-- DrissionPage/_pages/session_page.py | 2 +- DrissionPage/_pages/web_page.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 30830ac..b2862ef 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -307,7 +307,7 @@ class ChromiumBase(BasePage): @property def set(self): - """返回用于等待的对象""" + """返回用于设置的对象""" if self._set is None: self._set = ChromiumBaseSetter(self) return self._set diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index f27e0d1..44bc702 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -217,7 +217,7 @@ class ChromiumFrame(ChromiumBase): @property def scroll(self): - """返回用于等待的对象""" + """返回用于滚动的对象""" self.wait.load_complete() if self._scroll is None: self._scroll = FrameScroller(self) @@ -225,7 +225,7 @@ class ChromiumFrame(ChromiumBase): @property def set(self): - """返回用于等待的对象""" + """返回用于设置的对象""" if self._set is None: self._set = ChromiumFrameSetter(self) return self._set diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 9de3d29..dd3ee7a 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -102,7 +102,7 @@ class ChromiumPage(ChromiumBase): @property def set(self): - """返回用于等待的对象""" + """返回用于设置的对象""" if self._set is None: self._set = ChromiumPageSetter(self) return self._set diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 4038abd..ce8d901 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -46,7 +46,7 @@ class ChromiumTab(ChromiumBase): @property def set(self): - """返回用于等待的对象""" + """返回用于设置的对象""" if self._set is None: self._set = TabSetter(self) return self._set @@ -97,7 +97,7 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): @property def set(self): - """返回用于等待的对象""" + """返回用于设置的对象""" if self._set is None: self._set = WebPageTabSetter(self) return self._set diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index 0dce0af..5abcf09 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -132,7 +132,7 @@ class SessionPage(BasePage): @property def set(self): - """返回用于等待的对象""" + """返回用于设置的对象""" if self._set is None: self._set = SessionPageSetter(self) return self._set diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index e20931b..c12ed00 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -50,7 +50,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): @property def set(self): - """返回用于等待的对象""" + """返回用于设置的对象""" if self._set is None: self._set = WebPageSetter(self) return self._set From 655895c560630334b581d7b541fe26b43a595ea3 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 27 Dec 2023 23:45:58 +0800 Subject: [PATCH 161/182] =?UTF-8?q?4.0.0b32(+)=20=E4=BC=98=E5=8C=96WebPage?= =?UTF-8?q?=E7=9A=84post()=E8=BF=94=E5=9B=9E=E5=80=BC=EF=BC=9B=20=E4=BC=98?= =?UTF-8?q?=E5=8C=96run=5Fasync=5Fjs()=E9=80=BB=E8=BE=91=EF=BC=8C=E5=88=A0?= =?UTF-8?q?=E9=99=A4timeout=E5=8F=82=E6=95=B0=EF=BC=9B=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E5=88=A4=E6=96=AD=E8=A6=86=E7=9B=96=E5=A4=B1=E6=95=88?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=9B=20=E4=BF=AE=E5=A4=8D=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E6=BB=9A=E5=8A=A8=E6=9C=89=E6=97=B6=E6=8A=A5=E9=94=99?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=9B=20=E4=BC=98=E5=8C=96=5Fmake=5Frespo?= =?UTF-8?q?nse()=E8=BF=94=E5=9B=9E=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_elements/chromium_element.py | 3 +- DrissionPage/_pages/chromium_base.py | 7 +-- DrissionPage/_pages/chromium_base.pyi | 2 +- DrissionPage/_pages/chromium_tab.py | 13 +++-- DrissionPage/_pages/chromium_tab.pyi | 2 +- DrissionPage/_pages/session_page.py | 66 +++++++++++----------- DrissionPage/_pages/session_page.pyi | 4 +- DrissionPage/_pages/web_page.py | 13 +++-- DrissionPage/_pages/web_page.pyi | 2 +- DrissionPage/_units/scroller.py | 3 +- DrissionPage/_units/scroller.pyi | 6 +- DrissionPage/_units/states.py | 2 +- setup.py | 6 +- 14 files changed, 63 insertions(+), 68 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 9d09b17..3a5cb9f 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b31' +__version__ = '4.0.0b32' diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index ce88ff5..0916987 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -1409,7 +1409,8 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): res = page.run_cdp('Runtime.callFunctionOn', functionDeclaration=script, objectId=obj_id, arguments=[convert_argument(arg) for arg in args], returnByValue=False, awaitPromise=True, userGesture=True, _timeout=timeout, _ignore=AlertExistsError) - + except TimeoutError: + raise TimeoutError('执行js超时。') except ContextLostError: if is_page: raise ContextLostError('页面已被刷新，请尝试等待页面加载完成再执行操作。') diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index b2862ef..ea9a04d 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -485,17 +485,14 @@ class ChromiumBase(BasePage): self.wait.load_complete() return run_js(self, script, as_expr, self.timeouts.script if timeout is None else timeout, args) - def run_async_js(self, script, *args, as_expr=False, timeout=None): + def run_async_js(self, script, *args, as_expr=False): """以异步方式执行js代码 :param script: js文本 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 - :param timeout: js超时时间（秒），为None则使用页面timeouts.script属性值 :return: None """ - from threading import Thread - Thread(target=run_js, args=(self, script, as_expr, self.timeouts.script if timeout is None else timeout, - args)).start() + run_js(self, script, as_expr, 0, args) def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None): """访问url diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index abef890..4ed83cb 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -167,7 +167,7 @@ class ChromiumBase(BasePage): def run_js_loaded(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> Any: ... - def run_async_js(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> None: ... + def run_async_js(self, script: str, *args, as_expr: bool = False) -> None: ... def get(self, url: str, show_errmsg: bool = False, retry: int = None, interval: float = None, timeout: float = None) -> Union[None, bool]: ... diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index ce8d901..124cc4d 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -212,19 +212,20 @@ class WebPageTab(SessionPage, ChromiumTab, BasePage): timeout = self.timeouts.page_load if self._has_driver else self.timeout return super().get(url, show_errmsg, retry, interval, timeout, **kwargs) - def post(self, url: str, data=None, show_errmsg=False, retry=None, interval=None, **kwargs): + def post(self, url, show_errmsg=False, retry=None, interval=None, **kwargs): """用post方式跳转到url，会切换到s模式 :param url: 目标url - :param data: post方式时提交的数据 :param show_errmsg: 是否显示和抛出异常 - :param retry: 重试次数 - :param interval: 重试间隔（秒） + :param retry: 重试次数，为None时使用页面对象retry_times属性值 + :param interval: 重试间隔（秒），为None时使用页面对象retry_interval属性值 :param kwargs: 连接参数 - :return: url是否可用 + :return: s模式时返回url是否可用，d模式时返回获取到的Response对象 """ if self.mode == 'd': self.cookies_to_session() - return super().post(url, data, show_errmsg, retry, interval, **kwargs) + super().post(url, show_errmsg, retry, interval, **kwargs) + return self.response + return super().post(url, show_errmsg, retry, interval, **kwargs) def ele(self, loc_or_ele, timeout=None): """返回第一个符合条件的元素、属性或节点文本 diff --git a/DrissionPage/_pages/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi index ec3798a..f80d0a6 100644 --- a/DrissionPage/_pages/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -165,7 +165,7 @@ class WebPageTab(SessionPage, ChromiumTab): hooks: Any | None = ..., stream: Any | None = ..., verify: Any | None = ..., - cert: Any | None = ...) -> bool: ... + cert: Any | None = ...) -> Union[bool, Response]: ... @property def set(self) -> WebPageTabSetter: ... diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index 5abcf09..0f3ddce 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -25,7 +25,7 @@ class SessionPage(BasePage): def __init__(self, session_or_options=None, timeout=None): """ :param session_or_options: Session对象或SessionOptions对象 - :param timeout: 连接超时时间（秒），为None时从ini文件读取 + :param timeout: 连接超时时间（秒），为None时从ini文件读取或默认10 """ super(SessionPage, SessionPage).__init__(self) self._headers = None @@ -41,7 +41,7 @@ class SessionPage(BasePage): def _s_set_start_options(self, session_or_options): """启动配置 - :param session_or_options: Session、SessionOptions + :param session_or_options: Session、SessionOptions对象 :return: None """ if not session_or_options or isinstance(session_or_options, SessionOptions): @@ -117,12 +117,12 @@ class SessionPage(BasePage): @property def session(self): - """返回session对象""" + """返回Session对象""" return self._session @property def response(self): - """返回访问url得到的response对象""" + """返回访问url得到的Response对象""" return self._response @property @@ -159,7 +159,18 @@ class SessionPage(BasePage): r.status_code = 200 self._response = r return - return self._s_connect(url, 'get', None, show_errmsg, retry, interval, **kwargs) + return self._s_connect(url, 'get', show_errmsg, retry, interval, **kwargs) + + def post(self, url, show_errmsg=False, retry=None, interval=None, **kwargs): + """用post方式跳转到url + :param url: 目标url + :param show_errmsg: 是否显示和抛出异常 + :param retry: 重试次数，为None时使用页面对象retry_times属性值 + :param interval: 重试间隔（秒），为None时使用页面对象timeout属性值 + :param kwargs: 连接参数 + :return: url是否可用 + """ + return self._s_connect(url, 'post', show_errmsg, retry, interval, **kwargs) def ele(self, loc_or_ele, timeout=None): """返回页面中符合条件的第一个元素、属性或节点文本 @@ -230,18 +241,6 @@ class SessionPage(BasePage): r.append({'name': c['name'], 'value': c['value'], 'domain': c['domain']}) return r - def post(self, url, data=None, show_errmsg=False, retry=None, interval=None, **kwargs): - """用post方式跳转到url - :param url: 目标url - :param data: 提交的数据 - :param show_errmsg: 是否显示和抛出异常 - :param retry: 重试次数 - :param interval: 重试间隔（秒） - :param kwargs: 连接参数 - :return: url是否可用 - """ - return self._s_connect(url, 'post', data, show_errmsg, retry, interval, **kwargs) - def close(self): """关闭Session对象""" self._session.close() @@ -260,11 +259,10 @@ class SessionPage(BasePage): interval = interval if interval is not None else self.retry_interval return retry, interval - def _s_connect(self, url, mode, data=None, show_errmsg=False, retry=None, interval=None, **kwargs): + def _s_connect(self, url, mode, show_errmsg=False, retry=None, interval=None, **kwargs): """执行get或post连接 :param url: 目标url :param mode: 'get' 或 'post' - :param data: 提交的数据 :param show_errmsg: 是否显示和抛出异常 :param retry: 重试次数 :param interval: 重试间隔（秒） @@ -272,7 +270,7 @@ class SessionPage(BasePage): :return: url是否可用 """ retry, interval = self._before_connect(url, retry, interval) - self._response, info = self._make_response(self._url, mode, data, retry, interval, show_errmsg, **kwargs) + self._response, info = self._make_response(self._url, mode, retry, interval, show_errmsg, **kwargs) if self._response is None: self._url_available = False @@ -288,11 +286,10 @@ class SessionPage(BasePage): return self._url_available - def _make_response(self, url, mode='get', data=None, retry=None, interval=None, show_errmsg=False, **kwargs): + def _make_response(self, url, mode='get', retry=None, interval=None, show_errmsg=False, **kwargs): """生成Response对象 :param url: 目标url :param mode: 'get' 或 'post' - :param data: post方式要提交的数据 :param show_errmsg: 是否显示和抛出异常 :param kwargs: 其它参数 :return: tuple，第一位为Response或None，第二位为出错信息或 'Success' @@ -325,7 +322,7 @@ class SessionPage(BasePage): if mode == 'get': r = self.session.get(url, **kwargs) elif mode == 'post': - r = self.session.post(url, data=data, **kwargs) + r = self.session.post(url, **kwargs) if r and r.content: if self._encoding: @@ -344,18 +341,19 @@ class SessionPage(BasePage): if show_errmsg: print(f'重试 {url}') - if r is None: - if show_errmsg: - if err: - raise err - else: - raise ConnectionError('连接失败') - return None, '连接失败' if err is None else err + if show_errmsg: + if err: + raise err + elif r is not None: + raise ConnectionError(f'状态码：{r.status_code}') if r.content else ConnectionError('返回内容为空。') + else: + raise ConnectionError('连接失败') - if not r.ok: - if show_errmsg: - raise ConnectionError(f'状态码：{r.status_code}') - return r, f'状态码：{r.status_code}' + else: + if r is not None: + return (r, f'状态码：{r.status_code}') if r.content else (None, '返回内容为空') + else: + return None, '连接失败' if err is None else err def __repr__(self): return f'' diff --git a/DrissionPage/_pages/session_page.pyi b/DrissionPage/_pages/session_page.pyi index b3116b2..dd4da4e 100644 --- a/DrissionPage/_pages/session_page.pyi +++ b/DrissionPage/_pages/session_page.pyi @@ -128,10 +128,10 @@ class SessionPage(BasePage): def post(self, url: str, - data: Union[dict, str, None] = ..., show_errmsg: bool = False, retry: int | None = None, interval: float | None = None, + data: Union[dict, str, None] = ..., timeout: float | None = ..., params: dict | None = ..., json: Union[dict, str, None] = ..., @@ -153,7 +153,6 @@ class SessionPage(BasePage): def _s_connect(self, url: str, mode: str, - data: Union[dict, str, None] = None, show_errmsg: bool = False, retry: int = None, interval: float = None, @@ -162,7 +161,6 @@ class SessionPage(BasePage): def _make_response(self, url: str, mode: str = 'get', - data: Union[dict, str] = None, retry: int = None, interval: float = None, show_errmsg: bool = False, diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index c12ed00..7fec00d 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -165,19 +165,20 @@ class WebPage(SessionPage, ChromiumPage, BasePage): timeout = self.timeouts.page_load if self._has_driver else self.timeout return super().get(url, show_errmsg, retry, interval, timeout, **kwargs) - def post(self, url: str, data=None, show_errmsg=False, retry=None, interval=None, **kwargs): + def post(self, url, show_errmsg=False, retry=None, interval=None, **kwargs): """用post方式跳转到url，会切换到s模式 :param url: 目标url - :param data: post方式时提交的数据 :param show_errmsg: 是否显示和抛出异常 - :param retry: 重试次数 - :param interval: 重试间隔（秒） + :param retry: 重试次数，为None时使用页面对象retry_times属性值 + :param interval: 重试间隔（秒），为None时使用页面对象retry_interval属性值 :param kwargs: 连接参数 - :return: url是否可用 + :return: s模式时返回url是否可用，d模式时返回获取到的Response对象 """ if self.mode == 'd': self.cookies_to_session() - return super().post(url, data, show_errmsg, retry, interval, **kwargs) + super().post(url, show_errmsg, retry, interval, **kwargs) + return self.response + return super().post(url, show_errmsg, retry, interval, **kwargs) def ele(self, loc_or_ele, timeout=None): """返回第一个符合条件的元素、属性或节点文本 diff --git a/DrissionPage/_pages/web_page.pyi b/DrissionPage/_pages/web_page.pyi index 9c80c71..2b1671d 100644 --- a/DrissionPage/_pages/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -157,7 +157,7 @@ class WebPage(SessionPage, ChromiumPage, BasePage): hooks: Any | None = ..., stream: Any | None = ..., verify: Any | None = ..., - cert: Any | None = ...) -> bool: ... + cert: Any | None = ...) -> Union[bool, Response]: ... @property def set(self) -> WebPageSetter: ... diff --git a/DrissionPage/_units/scroller.py b/DrissionPage/_units/scroller.py index 6e3606e..b1b987e 100644 --- a/DrissionPage/_units/scroller.py +++ b/DrissionPage/_units/scroller.py @@ -81,6 +81,7 @@ class Scroller(object): self._run_js(f'{{}}.scrollBy({pixel}, 0);') def _wait_scrolled(self): + """等待滚动结束""" if not self._wait_complete: return @@ -89,7 +90,7 @@ class Scroller(object): x = r['layoutViewport']['pageX'] y = r['layoutViewport']['pageY'] - end_time = perf_counter() + self._driver.page.timeout + end_time = perf_counter() + page.timeout while perf_counter() < end_time: sleep(.1) r = page.run_cdp('Page.getLayoutMetrics') diff --git a/DrissionPage/_units/scroller.pyi b/DrissionPage/_units/scroller.pyi index 438da32..510543d 100644 --- a/DrissionPage/_units/scroller.pyi +++ b/DrissionPage/_units/scroller.pyi @@ -3,15 +3,13 @@ from typing import Union from .._elements.chromium_element import ChromiumElement from .._pages.chromium_base import ChromiumBase -from .._pages.chromium_frame import ChromiumFrame -from .._pages.chromium_page import ChromiumPage class Scroller(object): - def __init__(self, page_or_ele: Union[ChromiumBase, ChromiumElement, ChromiumFrame]): + def __init__(self, page_or_ele: Union[ChromiumBase, ChromiumElement]): self.t1: str = ... self.t2: str = ... - self._driver: Union[ChromiumPage, ChromiumElement, ChromiumFrame] = ... + self._driver: Union[ChromiumBase, ChromiumElement] = ... self._wait_complete: bool = ... def _run_js(self, js: str): ... diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index 8294fab..1ae731f 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -64,7 +64,7 @@ class ElementStates(object): """返回元素是否被覆盖，与是否在视口中无关，如被覆盖返回覆盖元素的backend id，否则返回False""" lx, ly = self._ele.rect.click_point try: - bid = self._ele.page.run_cdp('DOM.getNodeForLocation', x=lx, y=ly).get('backendNodeId') + bid = self._ele.page.run_cdp('DOM.getNodeForLocation', x=int(lx), y=int(ly)).get('backendNodeId') return bid if bid != self._ele._backend_id else False except CDPError: return False diff --git a/setup.py b/setup.py index 713e0f2..102fcd8 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b31", + version="4.0.0b32", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", @@ -29,12 +29,12 @@ setup( 'psutil' ], classifiers=[ - "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.8", "Development Status :: 4 - Beta", "Topic :: Utilities", "License :: OSI Approved :: BSD License", ], - python_requires='>=3.6', + python_requires='>=3.8', entry_points={ 'console_scripts': [ 'dp = DrissionPage.commons.cli:main', From a75cb3a0b37513eaac7d6e03ae2e2968457678c2 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 27 Dec 2023 23:57:53 +0800 Subject: [PATCH 162/182] =?UTF-8?q?=E5=88=A0=E9=99=A4run=5Fasync=5Fjs()tim?= =?UTF-8?q?eout=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 11 ++++------- DrissionPage/_elements/chromium_element.pyi | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 0916987..8eca461 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -364,7 +364,7 @@ class ChromiumElement(DrissionElement): def run_js(self, script, *args, as_expr=False, timeout=None): """对本元素执行javascript代码 - :param script: js文本 + :param script: js文本，文本中用this表示本元素 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 :param timeout: js超时时间（秒），为None则使用页面timeouts.script设置 @@ -372,17 +372,14 @@ class ChromiumElement(DrissionElement): """ return run_js(self, script, as_expr, self.page.timeouts.script if timeout is None else timeout, args) - def run_async_js(self, script, *args, as_expr=False, timeout=None): + def run_async_js(self, script, *args, as_expr=False): """以异步方式对本元素执行javascript代码 - :param script: js文本 + :param script: js文本，文本中用this表示本元素 :param args: 参数，按顺序在js文本中对应arguments[0]、arguments[1]... :param as_expr: 是否作为表达式运行，为True时args无效 - :param timeout: js超时时间（秒），为None则使用页面timeouts.script设置 :return: None """ - from threading import Thread - Thread(target=run_js, args=(self, script, as_expr, self.page.timeouts.script if timeout is None else timeout, - args, True)).start() + run_js(self, script, as_expr, 0, args) def ele(self, loc_or_str, timeout=None): """返回当前元素下级符合条件的第一个元素、属性或节点文本 diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 44b6c8d..388a30c 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -167,7 +167,7 @@ class ChromiumElement(DrissionElement): def run_js(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> Any: ... - def run_async_js(self, script: str, *args, as_expr: bool = False, timeout: float = None) -> None: ... + def run_async_js(self, script: str, *args, as_expr: bool = False) -> None: ... def ele(self, loc_or_str: Union[Tuple[str, str], str], From 63266cce76692cbd282d48acade4234a49edcf13 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 28 Dec 2023 17:23:47 +0800 Subject: [PATCH 163/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8DPage=E5=85=B3?= =?UTF-8?q?=E9=97=AD=E5=90=8Enew=5Ftab()=E6=8A=A5=E9=94=99=E9=97=AE?= =?UTF-8?q?=E9=A2=98=EF=BC=9B=E5=BE=AE=E8=B0=83Driver=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/driver.py | 16 ++++++++-------- DrissionPage/_elements/chromium_element.py | 2 +- DrissionPage/_functions/tools.py | 2 +- DrissionPage/_pages/chromium_base.py | 3 +-- DrissionPage/_pages/chromium_page.py | 2 +- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/DrissionPage/_base/driver.py b/DrissionPage/_base/driver.py index 2412b6f..d932805 100644 --- a/DrissionPage/_base/driver.py +++ b/DrissionPage/_base/driver.py @@ -77,7 +77,7 @@ class Driver(object): except (OSError, WebSocketConnectionClosedException): self.method_results.pop(ws_id, None) - return {'error': {'message': 'page closed'}} + return {'error': {'message': 'page closed'}, 'type': 'page_closed'} while not self._stopped.is_set(): try: @@ -87,15 +87,17 @@ class Driver(object): except Empty: if self.alert_flag and message['method'].startswith(('Input.', 'Runtime.')): - return {'error': {'message': 'alert exists.'}} + return {'error': {'message': 'alert exists.'}, 'type': 'alert_exists'} if timeout is not None and perf_counter() > end_time: self.method_results.pop(ws_id, None) - return {'error': {'message': 'alert exists.'}} \ - if self.alert_flag else {'error': {'message': 'timeout'}} + return {'error': {'message': 'alert exists.'}, 'type': 'alert_exists'} \ + if self.alert_flag else {'error': {'message': 'timeout'}, 'type': 'timeout'} continue + return {'error': 'page closed', 'type': 'page_closed'} + def _recv_loop(self): """接收浏览器信息的守护线程方法""" while not self._stopped.is_set(): @@ -161,11 +163,9 @@ class Driver(object): timeout = kwargs.pop('_timeout', 15) result = self._send({'method': _method, 'params': kwargs}, timeout=timeout) - if result is None: - return {'error': {'message': 'page closed'}} - elif 'result' not in result and 'error' in result: + if 'result' not in result and 'error' in result: return {'error': result['error']['message'], 'type': result.get('type', 'call_method_error'), - 'method': _method, 'args': kwargs} + 'method': _method, 'args': kwargs, 'timeout': timeout} else: return result['result'] diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 8eca461..2a0a4f0 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -1407,7 +1407,7 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): arguments=[convert_argument(arg) for arg in args], returnByValue=False, awaitPromise=True, userGesture=True, _timeout=timeout, _ignore=AlertExistsError) except TimeoutError: - raise TimeoutError('执行js超时。') + raise TimeoutError(f'执行js超时（{timeout}秒）。') except ContextLostError: if is_page: raise ContextLostError('页面已被刷新，请尝试等待页面加载完成再执行操作。') diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index 4db38b5..9df03db 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -219,7 +219,7 @@ def wait_until(page, condition, timeout=10, poll=0.1, raise_err=True): break if raise_err: - raise TimeoutError('等待超时') + raise TimeoutError(f'等待超时（{timeout}秒）') else: return False diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index ea9a04d..5ec4c7f 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -438,8 +438,7 @@ class ChromiumBase(BasePage): def _js_ready_state(self): """返回js获取的ready state信息""" try: - return self.run_cdp('Runtime.evaluate', expression='document.readyState;', - _timeout=3)['result']['value'] + return self.run_cdp('Runtime.evaluate', expression='document.readyState;', _timeout=3)['result']['value'] except ContextLostError: return None except TimeoutError: diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index dd3ee7a..3f04953 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -207,7 +207,7 @@ class ChromiumPage(ChromiumBase): if bid: kwargs['browserContextId'] = bid - return self.run_cdp('Target.createTarget', **kwargs)['targetId'] + return self.browser.run_cdp('Target.createTarget', **kwargs)['targetId'] def close_tabs(self, tabs_or_ids=None, others=False): """关闭传入的标签页，默认关闭当前页。可传入多个 From 6f5020f955f306f55c7c256e8cce15d746fb5a0c Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 29 Dec 2023 19:20:15 +0800 Subject: [PATCH 164/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=9B=91=E5=90=AC?= =?UTF-8?q?=E4=B8=80=E4=B8=AA=E9=97=AE=E9=A2=98=EF=BC=9BPageClosedError?= =?UTF-8?q?=E6=94=B9=E4=B8=BAPageDisconnectedError?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 6 +++--- DrissionPage/_base/driver.py | 10 +++++----- DrissionPage/_functions/tools.py | 8 ++++---- DrissionPage/_pages/chromium_base.py | 4 ++-- DrissionPage/_pages/chromium_frame.py | 4 ++-- DrissionPage/_units/listener.py | 4 +--- DrissionPage/_units/states.py | 6 +++--- DrissionPage/errors.py | 4 ++-- 8 files changed, 22 insertions(+), 24 deletions(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 6556480..2d12911 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -8,7 +8,7 @@ from time import sleep, perf_counter from .driver import BrowserDriver, Driver from .._functions.tools import stop_process_on_port, raise_error from .._units.downloader import DownloadManager -from ..errors import PageClosedError +from ..errors import PageDisconnectedError __ERROR__ = 'error' @@ -143,7 +143,7 @@ class Browser(object): :param tab_id: 标签页id :return: None """ - self.run_cdp('Target.closeTarget', targetId=tab_id, _ignore=PageClosedError) + self.run_cdp('Target.closeTarget', targetId=tab_id, _ignore=PageDisconnectedError) def activate_tab(self, tab_id): """使标签页变为活动状态 @@ -168,7 +168,7 @@ class Browser(object): try: self.run_cdp('Browser.close') self.driver.stop() - except PageClosedError: + except PageDisconnectedError: self.driver.stop() return diff --git a/DrissionPage/_base/driver.py b/DrissionPage/_base/driver.py index d932805..0b63f80 100644 --- a/DrissionPage/_base/driver.py +++ b/DrissionPage/_base/driver.py @@ -12,7 +12,7 @@ from requests import get from websocket import (WebSocketTimeoutException, WebSocketConnectionClosedException, create_connection, WebSocketException) -from ..errors import PageClosedError +from ..errors import PageDisconnectedError class Driver(object): @@ -77,7 +77,7 @@ class Driver(object): except (OSError, WebSocketConnectionClosedException): self.method_results.pop(ws_id, None) - return {'error': {'message': 'page closed'}, 'type': 'page_closed'} + return {'error': {'message': 'connection disconnected'}, 'type': 'connection_error'} while not self._stopped.is_set(): try: @@ -96,7 +96,7 @@ class Driver(object): continue - return {'error': 'page closed', 'type': 'page_closed'} + return {'error': {'message': 'connection disconnected'}, 'type': 'connection_error'} def _recv_loop(self): """接收浏览器信息的守护线程方法""" @@ -159,7 +159,7 @@ class Driver(object): :return: 执行结果 """ if self._stopped.is_set(): - return {'error': 'page closed', 'type': 'page_closed'} + return {'error': 'connection disconnected', 'type': 'connection_error'} timeout = kwargs.pop('_timeout', 15) result = self._send({'method': _method, 'params': kwargs}, timeout=timeout) @@ -258,5 +258,5 @@ class BrowserDriver(Driver): def run_function(function, kwargs): try: function(**kwargs) - except PageClosedError: + except PageDisconnectedError: pass diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index 9df03db..edfc8ec 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -12,8 +12,8 @@ from time import perf_counter, sleep from psutil import process_iter, AccessDenied, NoSuchProcess, ZombieProcess from .._configs.options_manage import OptionsManager -from ..errors import (ContextLostError, ElementLostError, CDPError, PageClosedError, NoRectError, AlertExistsError, - WrongURLError, StorageError, CookieFormatError) +from ..errors import (ContextLostError, ElementLostError, CDPError, PageDisconnectedError, NoRectError, + AlertExistsError, WrongURLError, StorageError, CookieFormatError) def get_usable_path(path, is_file=True, parents=True): @@ -267,8 +267,8 @@ def raise_error(result, ignore=None): 'No node with given id found', 'Node with given id does not belong to the document', 'No node found for given backend id'): r = ElementLostError() - elif error in ('page closed', 'No target with given id found'): - r = PageClosedError() + elif error in ('connection disconnected', 'No target with given id found'): + r = PageDisconnectedError() elif error == 'timeout': r = TimeoutError(f'超时。\n错误：{result["error"]}\nmethod：{result["method"]}\nargs：{result["args"]}\n' f'出现这个错误可能意味着程序有bug，请把错误信息和重现方法告知作者，谢谢。\n' diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 5ec4c7f..9ca83ed 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -27,7 +27,7 @@ from .._units.scroller import PageScroller from .._units.setter import ChromiumBaseSetter from .._units.states import PageStates from .._units.waiter import BaseWaiter -from ..errors import ContextLostError, CDPError, PageClosedError, ElementNotFoundError +from ..errors import ContextLostError, CDPError, PageDisconnectedError, ElementNotFoundError __ERROR__ = 'error' @@ -672,7 +672,7 @@ class ChromiumBase(BasePage): print('停止页面加载') try: self.run_cdp('Page.stopLoading') - except (PageClosedError, CDPError): + except (PageDisconnectedError, CDPError): pass end_time = perf_counter() + self.timeouts.page_load while self._ready_state != 'complete' and perf_counter() < end_time: diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 44bc702..9abe951 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -15,7 +15,7 @@ from .._units.scroller import FrameScroller from .._units.setter import ChromiumFrameSetter from .._units.states import FrameStates from .._units.waiter import FrameWaiter -from ..errors import ContextLostError, ElementLostError, PageClosedError, JavaScriptError +from ..errors import ContextLostError, ElementLostError, PageDisconnectedError, JavaScriptError class ChromiumFrame(ChromiumBase): @@ -119,7 +119,7 @@ class ChromiumFrame(ChromiumBase): else: return - except (ElementLostError, PageClosedError): + except (ElementLostError, PageDisconnectedError): return if self._is_inner_frame(): diff --git a/DrissionPage/_units/listener.py b/DrissionPage/_units/listener.py index d576480..d18759f 100644 --- a/DrissionPage/_units/listener.py +++ b/DrissionPage/_units/listener.py @@ -295,9 +295,7 @@ class Listener(object): packet = self._request_ids.get(rid) if packet: r = self._driver.run('Network.getResponseBody', requestId=rid) - if 'error' in r: - return - elif 'body' in r: + if 'body' in r: packet._raw_body = r['body'] packet._base64_body = r['base64Encoded'] else: diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index 1ae731f..2abbcf5 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from .._functions.web import location_in_viewport -from ..errors import CDPError, NoRectError, PageClosedError, ElementLostError +from ..errors import CDPError, NoRectError, PageDisconnectedError, ElementLostError class ElementStates(object): @@ -120,7 +120,7 @@ class PageStates(object): try: self._page.run_cdp('Page.getLayoutMetrics') return True - except PageClosedError: + except PageDisconnectedError: return False @property @@ -152,7 +152,7 @@ class FrameStates(object): try: node = self._frame._target_page.run_cdp('DOM.describeNode', backendNodeId=self._frame._frame_ele._backend_id)['node'] - except (ElementLostError, PageClosedError): + except (ElementLostError, PageDisconnectedError): return False return 'frameId' in node diff --git a/DrissionPage/errors.py b/DrissionPage/errors.py index 5e8677a..155453e 100644 --- a/DrissionPage/errors.py +++ b/DrissionPage/errors.py @@ -45,8 +45,8 @@ class CDPError(BaseError): _info = '方法调用错误。' -class PageClosedError(BaseError): - _info = '页面已关闭。' +class PageDisconnectedError(BaseError): + _info = '与页面的连接已断开。' class JavaScriptError(BaseError): From 3005468ce070763a4f553c21def614a48926dc31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8F=88=E9=98=B3=E6=9C=88?= <2497732985@qq.com> Date: Sat, 30 Dec 2023 00:47:59 +0800 Subject: [PATCH 165/182] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=8E=A5=E5=8F=A3=20?= =?UTF-8?q?add=5Finit=5Fscript=EF=BC=9A=E5=9C=A8=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E8=BF=9B=E8=A1=8C=E4=BB=BB=E4=BD=95=E8=84=9A=E6=9C=AC=E7=9A=84?= =?UTF-8?q?=E5=8A=A0=E8=BD=BD=E5=89=8D=EF=BC=8C=E6=89=A7=E8=A1=8Cjs?= =?UTF-8?q?=E8=AF=AD=E5=8F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 9ca83ed..8ff490c 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -493,6 +493,19 @@ class ChromiumBase(BasePage): """ run_js(self, script, as_expr, 0, args) + def add_init_script(self, script: str, raise_error=True): + '''添加初始化脚本，在页面加载任何脚本前执行 + :param script: js文本 + :return: identifier 添加的脚本的标识符，用于删除，失败时返回False，或raise Error + ''' + result = self.driver.run('Page.addScriptToEvaluateOnNewDocument', source=script) + if not result or __ERROR__ not in result: + return result['identifier'] + else: + if raise_error: + raise_error(str(result)) + return False + def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None): """访问url :param url: 目标url From 290baa680a23529029c77bf93570ac8ea3bf8bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8F=88=E9=98=B3=E6=9C=88?= <2497732985@qq.com> Date: Sat, 30 Dec 2023 12:19:05 +0800 Subject: [PATCH 166/182] =?UTF-8?q?=E7=A7=BB=E5=8A=A8add=5Finit=5Fscript?= =?UTF-8?q?=E5=88=B0TabSetter=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 13 ------------- DrissionPage/_units/setter.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 8ff490c..48b818a 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -492,19 +492,6 @@ class ChromiumBase(BasePage): :return: None """ run_js(self, script, as_expr, 0, args) - - def add_init_script(self, script: str, raise_error=True): - '''添加初始化脚本，在页面加载任何脚本前执行 - :param script: js文本 - :return: identifier 添加的脚本的标识符，用于删除，失败时返回False，或raise Error - ''' - result = self.driver.run('Page.addScriptToEvaluateOnNewDocument', source=script) - if not result or __ERROR__ not in result: - return result['identifier'] - else: - if raise_error: - raise_error(str(result)) - return False def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None): """访问url diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index 5c27059..683ee10 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -10,6 +10,7 @@ from requests.structures import CaseInsensitiveDict from .cookies_setter import SessionCookiesSetter, CookiesSetter from .._functions.tools import show_or_hide_browser +__ERROR__ = 'error' class BasePageSetter(object): def __init__(self, page): @@ -198,6 +199,19 @@ class TabSetter(ChromiumBaseSetter): """使标签页处于最前面""" self._page.browser.activate_tab(self._page.tab_id) + def add_init_script(self, script: str, raise_error=True): + '''添加初始化脚本，在页面加载任何脚本前执行 + :param script: js文本 + :return: identifier 添加的脚本的标识符，失败时返回False，或raise Error + ''' + result = self.driver.run('Page.addScriptToEvaluateOnNewDocument', source=script) + if not result or __ERROR__ not in result: + return result['identifier'] + else: + if raise_error: + raise_error(str(result)) + return False + class ChromiumPageSetter(TabSetter): From 34c5ad818b1b0ce01e97e8977989aa29959c918c Mon Sep 17 00:00:00 2001 From: g1879 Date: Sat, 30 Dec 2023 14:40:20 +0800 Subject: [PATCH 167/182] =?UTF-8?q?=E5=85=83=E7=B4=A0=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=E9=97=B4=E5=8F=AF=E7=94=A8=3D=3D=E5=88=A4=E6=96=AD=E6=98=AF?= =?UTF-8?q?=E5=90=A6=E6=8C=87=E5=90=91=E5=90=8C=E4=B8=80=E4=B8=AA=E5=85=83?= =?UTF-8?q?=E7=B4=A0=EF=BC=9BPage=E7=9A=84save()=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E8=BF=94=E5=9B=9E=E6=96=87=E6=9C=AC=E4=B8=8D=E4=BF=9D=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 6 ++++++ DrissionPage/_elements/chromium_element.pyi | 4 ++++ DrissionPage/_elements/session_element.py | 3 +++ DrissionPage/_elements/session_element.pyi | 2 ++ DrissionPage/_pages/chromium_base.py | 8 +++++--- DrissionPage/_pages/chromium_frame.py | 3 +++ DrissionPage/_pages/chromium_frame.pyi | 2 ++ DrissionPage/_pages/chromium_page.py | 6 +++--- DrissionPage/_pages/chromium_tab.py | 6 +++--- 9 files changed, 31 insertions(+), 9 deletions(-) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 2a0a4f0..48d64e2 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -85,6 +85,9 @@ class ChromiumElement(DrissionElement): """ return self.ele(loc_or_str, timeout) + def __eq__(self, other): + return self._backend_id == getattr(other, '_backend_id', None) + @property def tag(self): """返回元素tag""" @@ -794,6 +797,9 @@ class ShadowRoot(BaseElement): """ return self.ele(loc_or_str, timeout) + def __eq__(self, other): + return self._backend_id == getattr(other, '_backend_id', None) + @property def tag(self): """返回元素标签名""" diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 388a30c..4e3f886 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -48,6 +48,8 @@ class ChromiumElement(DrissionElement): def __call__(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> Union[ChromiumElement, NoneElement]: ... + def __eq__(self, other: ChromiumElement) -> bool: ... + @property def tag(self) -> str: ... @@ -242,6 +244,8 @@ class ShadowRoot(BaseElement): def __call__(self, loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> ChromiumElement: ... + def __eq__(self, other: ShadowRoot) -> bool: ... + @property def states(self) -> ShadowRootStates: ... diff --git a/DrissionPage/_elements/session_element.py b/DrissionPage/_elements/session_element.py index 430e6c4..82820a9 100644 --- a/DrissionPage/_elements/session_element.py +++ b/DrissionPage/_elements/session_element.py @@ -43,6 +43,9 @@ class SessionElement(DrissionElement): """ return self.ele(loc_or_str) + def __eq__(self, other): + return self.xpath == getattr(other, 'xpath', None) + @property def tag(self): """返回元素类型""" diff --git a/DrissionPage/_elements/session_element.pyi b/DrissionPage/_elements/session_element.pyi index 190d06b..32452c6 100644 --- a/DrissionPage/_elements/session_element.pyi +++ b/DrissionPage/_elements/session_element.pyi @@ -30,6 +30,8 @@ class SessionElement(DrissionElement): loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> Union[SessionElement, NoneElement]: ... + def __eq__(self, other: SessionElement) -> bool: ... + @property def tag(self) -> str: ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 9ca83ed..df4476e 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -1153,13 +1153,15 @@ def close_privacy_dialog(page, tid): def get_mhtml(page, path=None, name=None): - """把当前页面保存为mhtml文件 + """把当前页面保存为mhtml文件，如果path和name参数都为None，只返回mhtml文本 :param page: 要保存的页面对象 - :param path: 保存路径，为None保存在当前路径 - :param name: 文件名，为None则用title属性值 + :param path: 保存路径，为None且name不为None时保存在当前路径 + :param name: 文件名，为None且path不为None时用title属性值 :return: mhtml文本 """ r = page.run_cdp('Page.captureSnapshot')['data'] + if path is None and name is None: + return r path = path or '.' Path(path).mkdir(parents=True, exist_ok=True) name = make_valid_name(name or page.title) diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 9abe951..80e0460 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -71,6 +71,9 @@ class ChromiumFrame(ChromiumBase): """ return self.ele(loc_or_str, timeout) + def __eq__(self, other): + return self._frame_id == getattr(other, '_frame_id', None) + def __repr__(self): attrs = self._frame_ele.attrs attrs = [f"{attr}='{attrs[attr]}'" for attr in attrs] diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index f2cf074..a2149bb 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -44,6 +44,8 @@ class ChromiumFrame(ChromiumBase): loc_or_str: Union[Tuple[str, str], str], timeout: float = None) -> Union[ChromiumElement, NoneElement]: ... + def __eq__(self, other: ChromiumFrame) -> bool: ... + def _check_alive(self) -> None: ... def __repr__(self) -> str: ... diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 3f04953..2bfb04d 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -142,9 +142,9 @@ class ChromiumPage(ChromiumBase): return self.browser.process_id def save(self, path=None, name=None): - """把当前页面保存为mhtml文件 - :param path: 保存路径，为None保存在当前路径 - :param name: 文件名，为None则用title属性值 + """把当前页面保存为mhtml文件，如果path和name参数都为None，只返回mhtml文本 + :param path: 保存路径，为None且name不为None时保存在当前路径 + :param name: 文件名，为None且path不为None时用title属性值 :return: mhtml文本 """ return get_mhtml(self, path, name) diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 124cc4d..06ae0fa 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -59,9 +59,9 @@ class ChromiumTab(ChromiumBase): return self._wait def save(self, path=None, name=None): - """把当前页面保存为mhtml文件 - :param path: 保存路径，为None保存在当前路径 - :param name: 文件名，为None则用title属性值 + """把当前页面保存为mhtml文件，如果path和name参数都为None，只返回mhtml文本 + :param path: 保存路径，为None且name不为None时保存在当前路径 + :param name: 文件名，为None且path不为None时用title属性值 :return: mhtml文本 """ return get_mhtml(self, path, name) From 54d2e3b9ded033894897de05e17b0a33b110cd00 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sat, 30 Dec 2023 20:51:06 +0800 Subject: [PATCH 168/182] =?UTF-8?q?=E5=85=83=E7=B4=A0=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=E9=97=B4=E5=8F=AF=E7=94=A8=3D=3D=E5=88=A4=E6=96=AD=E6=98=AF?= =?UTF-8?q?=E5=90=A6=E6=8C=87=E5=90=91=E5=90=8C=E4=B8=80=E4=B8=AA=E5=85=83?= =?UTF-8?q?=E7=B4=A0=EF=BC=9BPage=E7=9A=84save()=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E8=BF=94=E5=9B=9E=E6=96=87=E6=9C=AC=EF=BC=8C=E4=B8=8D=E4=BF=9D?= =?UTF-8?q?=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/driver.py | 2 +- DrissionPage/_elements/chromium_element.py | 2 +- DrissionPage/_functions/tools.py | 6 +++--- DrissionPage/_pages/chromium_base.py | 4 ++-- DrissionPage/_units/listener.py | 2 +- DrissionPage/_units/waiter.py | 22 +++++++++++----------- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/DrissionPage/_base/driver.py b/DrissionPage/_base/driver.py index 0b63f80..28b8642 100644 --- a/DrissionPage/_base/driver.py +++ b/DrissionPage/_base/driver.py @@ -161,7 +161,7 @@ class Driver(object): if self._stopped.is_set(): return {'error': 'connection disconnected', 'type': 'connection_error'} - timeout = kwargs.pop('_timeout', 15) + timeout = kwargs.pop('_timeout', 30) result = self._send({'method': _method, 'params': kwargs}, timeout=timeout) if 'result' not in result and 'error' in result: return {'error': result['error']['message'], 'type': result.get('type', 'call_method_error'), diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 48d64e2..790801f 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -1413,7 +1413,7 @@ def run_js(page_or_ele, script, as_expr=False, timeout=None, args=None): arguments=[convert_argument(arg) for arg in args], returnByValue=False, awaitPromise=True, userGesture=True, _timeout=timeout, _ignore=AlertExistsError) except TimeoutError: - raise TimeoutError(f'执行js超时（{timeout}秒）。') + raise TimeoutError(f'执行js超时（等待{timeout}秒）。') except ContextLostError: if is_page: raise ContextLostError('页面已被刷新，请尝试等待页面加载完成再执行操作。') diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index edfc8ec..640d0cf 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -219,7 +219,7 @@ def wait_until(page, condition, timeout=10, poll=0.1, raise_err=True): break if raise_err: - raise TimeoutError(f'等待超时（{timeout}秒）') + raise TimeoutError(f'等待超时（等待{timeout}秒）。') else: return False @@ -270,8 +270,8 @@ def raise_error(result, ignore=None): elif error in ('connection disconnected', 'No target with given id found'): r = PageDisconnectedError() elif error == 'timeout': - r = TimeoutError(f'超时。\n错误：{result["error"]}\nmethod：{result["method"]}\nargs：{result["args"]}\n' - f'出现这个错误可能意味着程序有bug，请把错误信息和重现方法告知作者，谢谢。\n' + r = TimeoutError(f'超时（等待{result["timeout"]}秒）。\n错误：{result["error"]}\nmethod：{result["method"]}\nargs：' + f'{result["args"]}\n出现这个错误可能意味着程序有bug，请把错误信息和重现方法告知作者，谢谢。\n' '报告网站：https://gitee.com/g1879/DrissionPage/issues') elif error == 'alert exists.': r = AlertExistsError() diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index df4476e..9c99b3b 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -944,7 +944,7 @@ class ChromiumBase(BasePage): if 'errorText' in result: err = ConnectionError(result['errorText']) except TimeoutError: - err = TimeoutError('页面连接超时。') + err = TimeoutError(f'页面连接超时（等待{timeout}秒）。') if err: if t < times: @@ -963,7 +963,7 @@ class ChromiumBase(BasePage): yu = end_time - perf_counter() ok = self._wait_loaded(1 if yu <= 0 else yu) if not ok: - err = TimeoutError('页面连接超时。') + err = TimeoutError(f'页面连接超时（等待{timeout}秒）。') if t < times: sleep(interval) if self._debug or show_errmsg: diff --git a/DrissionPage/_units/listener.py b/DrissionPage/_units/listener.py index d18759f..c8a8ffb 100644 --- a/DrissionPage/_units/listener.py +++ b/DrissionPage/_units/listener.py @@ -118,7 +118,7 @@ class Listener(object): if fail: if fit_count or not self._caught.qsize(): if raise_err is True or Settings.raise_when_wait_failed is True: - raise WaitTimeoutError('等待数据包失败。') + raise WaitTimeoutError(f'等待数据包失败（等待{timeout}秒）。') else: return False else: diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 7a45607..3fed08e 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -43,7 +43,7 @@ class BaseWaiter(object): timeout = end_time - perf_counter() if timeout <= 0: if raise_err is True or Settings.raise_when_wait_failed is True: - raise WaitTimeoutError('等待元素显示失败。') + raise WaitTimeoutError(f'等待元素显示失败（等待{timeout}秒）。') else: return False return ele.wait.displayed(timeout, raise_err=raise_err) @@ -62,7 +62,7 @@ class BaseWaiter(object): timeout = end_time - perf_counter() if timeout <= 0: if raise_err is True or Settings.raise_when_wait_failed is True: - raise WaitTimeoutError('等待元素显示失败。') + raise WaitTimeoutError(f'等待元素显示失败（等待{timeout}秒）。') else: return False return ele.wait.hidden(timeout, raise_err=raise_err) @@ -78,7 +78,7 @@ class BaseWaiter(object): if ele: return ele if raise_err is True or Settings.raise_when_wait_failed is True: - raise WaitTimeoutError('等待元素加载失败。') + raise WaitTimeoutError(f'等待元素加载失败（等待{timeout}秒）。') else: return False @@ -173,7 +173,7 @@ class BaseWaiter(object): sleep(.05) if raise_err is True or Settings.raise_when_wait_failed is True: - raise WaitTimeoutError(f'等待{arg}改变失败。') + raise WaitTimeoutError(f'等待{arg}改变失败（等待{timeout}秒）。') else: return False @@ -195,7 +195,7 @@ class BaseWaiter(object): sleep(gap) if raise_err is True or Settings.raise_when_wait_failed is True: - raise WaitTimeoutError('等待页面加载失败。') + raise WaitTimeoutError(f'等待页面加载失败（等待{timeout}秒）。') else: return False @@ -266,7 +266,7 @@ class PageWaiter(TabWaiter): sleep(.01) if raise_err is True or Settings.raise_when_wait_failed is True: - raise WaitTimeoutError('等待新标签页失败。') + raise WaitTimeoutError(f'等待新标签页失败（等待{timeout}秒）。') else: return False @@ -386,7 +386,7 @@ class ElementWaiter(object): sleep(.05) if raise_err is True or Settings.raise_when_wait_failed is True: - raise WaitTimeoutError('等待元素隐藏或被删除失败。') + raise WaitTimeoutError(f'等待元素隐藏或被删除失败（等待{timeout}秒）。') else: return False @@ -418,7 +418,7 @@ class ElementWaiter(object): location = self._ele.rect.location if raise_err is True or Settings.raise_when_wait_failed is True: - raise WaitTimeoutError('等待元素停止运动失败。') + raise WaitTimeoutError(f'等待元素停止运动失败（等待{timeout}秒）。') else: return False @@ -428,7 +428,7 @@ class ElementWaiter(object): :param raise_err: 等待失败时是否报错，为None时根据Settings设置 :return: 是否等待成功 """ - return self._wait_state('has_rect', True, timeout, raise_err, err_text='等待元素拥有大小及位置属性失败。') + return self._wait_state('has_rect', True, timeout, raise_err, err_text='等待元素拥有大小及位置属性失败（等待{}秒）。') def _wait_state(self, attr, mode=False, timeout=None, raise_err=None, err_text=None): """等待元素某个元素状态到达指定状态 @@ -439,7 +439,7 @@ class ElementWaiter(object): :param err_text: 抛出错误时显示的信息 :return: 是否等待成功 """ - err_text = err_text or '等待元素状态改变失败。' + err_text = err_text or '等待元素状态改变失败（等待{}秒）。' if timeout is None: timeout = self._page.timeout end_time = perf_counter() + timeout @@ -449,7 +449,7 @@ class ElementWaiter(object): sleep(.05) if raise_err is True or Settings.raise_when_wait_failed is True: - raise WaitTimeoutError(err_text) + raise WaitTimeoutError(err_text.format(timeout)) else: return False From c0f50e2bbfa9bf3e4fae62d836cb6adbd6460b1f Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 31 Dec 2023 00:00:40 +0800 Subject: [PATCH 169/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8DWebPage=E7=9A=84set.c?= =?UTF-8?q?ookies=E9=97=AE=E9=A2=98=EF=BC=9B=E6=B7=BB=E5=8A=A0add=5Finit?= =?UTF-8?q?=5Fjs()=E5=92=8Cremove=5Finit=5Fjs()=EF=BC=9B=E5=B0=9D=E8=AF=95?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=8E=B7=E5=8F=96=E5=85=83=E7=B4=A0=E5=A4=A7?= =?UTF-8?q?=E5=B0=8F=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_functions/tools.py | 12 +++---- DrissionPage/_pages/chromium_base.py | 27 ++++++++++++++- DrissionPage/_pages/chromium_base.pyi | 5 +++ DrissionPage/_units/cookies_setter.py | 35 ++++++++++++++++++++ DrissionPage/_units/cookies_setter.pyi | 26 ++++++++++++--- DrissionPage/_units/rect.py | 6 ++-- DrissionPage/_units/setter.py | 46 ++++++++------------------ DrissionPage/_units/setter.pyi | 8 +++-- DrissionPage/_units/waiter.py | 2 +- 9 files changed, 115 insertions(+), 52 deletions(-) diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index 640d0cf..81e9515 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -269,10 +269,6 @@ def raise_error(result, ignore=None): r = ElementLostError() elif error in ('connection disconnected', 'No target with given id found'): r = PageDisconnectedError() - elif error == 'timeout': - r = TimeoutError(f'超时（等待{result["timeout"]}秒）。\n错误：{result["error"]}\nmethod：{result["method"]}\nargs：' - f'{result["args"]}\n出现这个错误可能意味着程序有bug，请把错误信息和重现方法告知作者，谢谢。\n' - '报告网站：https://gitee.com/g1879/DrissionPage/issues') elif error == 'alert exists.': r = AlertExistsError() elif error in ('Node does not have a layout object', 'Could not compute box model.'): @@ -283,10 +279,12 @@ def raise_error(result, ignore=None): r = StorageError() elif error == 'Sanitizing cookie failed': r = CookieFormatError(f'cookie格式不正确：{result["args"]}') - elif result['type'] == 'call_method_error': + elif result['type'] in ('call_method_error', 'timeout'): + from DrissionPage import __version__ + from time import process_time r = CDPError(f'\n错误：{result["error"]}\nmethod：{result["method"]}\nargs：{result["args"]}\n' - f'出现这个错误可能意味着程序有bug，请把错误信息和重现方法告知作者，谢谢。' - f'\n报告网站：https://gitee.com/g1879/DrissionPage/issues') + f'版本：{__version__}\n运行时间：{process_time()}\n出现这个错误可能意味着程序有bug，请把错误信息和重现方法' + '告知作者，谢谢。\n报告网站：https://gitee.com/g1879/DrissionPage/issues') else: r = RuntimeError(result) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 3920137..d1eadd0 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -58,6 +58,7 @@ class ChromiumBase(BasePage): self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc self._download_path = None self._load_end_time = 0 + self._init_jss = [] if not hasattr(self, '_listener'): self._listener = None @@ -492,7 +493,7 @@ class ChromiumBase(BasePage): :return: None """ run_js(self, script, as_expr, 0, args) - + def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None): """访问url :param url: 目标url @@ -796,6 +797,30 @@ class ChromiumBase(BasePage): return self._get_screenshot(path=path, name=name, as_bytes=as_bytes, as_base64=as_base64, full_page=full_page, left_top=left_top, right_bottom=right_bottom) + def add_init_js(self, js): + """添加初始化脚本，在页面加载任何脚本前执行 + :param js: js文本 + :return: 添加的脚本的id + """ + js_id = self.run_cdp('Page.addScriptToEvaluateOnNewDocument', source=js, + includeCommandLineAPI=True)['identifier'] + self._init_jss.append(js_id) + return js_id + + def remove_init_js(self, js_id=None): + """删除初始化脚本，js_id传入None时删除所有 + :param js_id: 脚本的id + :return: None + """ + if js_id is None: + for js_id in self._init_jss: + self.run_cdp('Page.removeScriptToEvaluateOnNewDocument', identifier=js_id) + self._init_jss.clear() + + elif js_id in self._init_jss: + self.run_cdp('Page.removeScriptToEvaluateOnNewDocument', identifier=js_id) + self._init_jss.remove(js_id) + def clear_cache(self, session_storage=True, local_storage=True, cache=True, cookies=True): """清除缓存，可选要清除的项 :param session_storage: 是否清除sessionStorage diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 4ed83cb..5e107c4 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -58,6 +58,7 @@ class ChromiumBase(BasePage): self._has_alert: bool = ... self._doc_got: bool = ... self._load_end_time: float = ... + self._init_jss: list = ... self._ready_state: Optional[str] = ... self._rect: TabRect = ... @@ -214,6 +215,10 @@ class ChromiumBase(BasePage): def get_local_storage(self, item: str = None) -> Union[str, dict, None]: ... + def add_init_js(self, js: str) -> str: ... + + def remove_init_js(self, js_id: str = None) -> None: ... + def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: PIC_TYPE = None, as_base64: PIC_TYPE = None, full_page: bool = False, left_top: Tuple[int, int] = None, right_bottom: Tuple[int, int] = None) -> Union[str, bytes]: ... diff --git a/DrissionPage/_units/cookies_setter.py b/DrissionPage/_units/cookies_setter.py index e22878a..3eda66a 100644 --- a/DrissionPage/_units/cookies_setter.py +++ b/DrissionPage/_units/cookies_setter.py @@ -66,3 +66,38 @@ class SessionCookiesSetter(object): def clear(self): """清除cookies""" self._page.session.cookies.clear() + + +class WebPageCookiesSetter(CookiesSetter, SessionCookiesSetter): + + def __call__(self, cookies): + """设置多个cookie，注意不要传入单个 + :param cookies: cookies信息 + :return: None + """ + if self._page.mode == 'd' and self._page._has_driver: + super().__call__(cookies) + elif self._page.mode == 's' and self._page._has_session: + super(CookiesSetter, self).__call__(cookies) + + def remove(self, name, url=None, domain=None, path=None): + """删除一个cookie + :param name: cookie的name字段 + :param url: cookie的url字段，可选，d模式时才有效 + :param domain: cookie的domain字段，可选，d模式时才有效 + :param path: cookie的path字段，可选，d模式时才有效 + :return: None + """ + if self._page.mode == 'd' and self._page._has_driver: + super().remove(name, url, domain, path) + elif self._page.mode == 's' and self._page._has_session: + if url or domain or path: + raise AttributeError('url、domain、path参数只有d模式下有效。') + super(CookiesSetter, self).remove(name) + + def clear(self): + """清除cookies""" + if self._page.mode == 'd' and self._page._has_driver: + super().clear() + elif self._page.mode == 's' and self._page._has_session: + super(CookiesSetter, self).clear() diff --git a/DrissionPage/_units/cookies_setter.pyi b/DrissionPage/_units/cookies_setter.pyi index 4bfc3e9..f1b5877 100644 --- a/DrissionPage/_units/cookies_setter.pyi +++ b/DrissionPage/_units/cookies_setter.pyi @@ -4,13 +4,16 @@ from typing import Union from requests.cookies import RequestsCookieJar -from .._pages.session_page import SessionPage from .._pages.chromium_base import ChromiumBase +from .._pages.chromium_tab import WebPageTab +from .._pages.session_page import SessionPage +from .._pages.web_page import WebPage class CookiesSetter(object): - def __init__(self, page: ChromiumBase): - self._page: ChromiumBase = ... + _page: ChromiumBase + + def __init__(self, page: ChromiumBase): ... def __call__(self, cookies: Union[RequestsCookieJar, Cookie, list, tuple, str, dict]) -> None: ... @@ -20,11 +23,24 @@ class CookiesSetter(object): class SessionCookiesSetter(object): - def __init__(self, page: SessionPage): - self._page: SessionPage = ... + _page: SessionPage + + def __init__(self, page: SessionPage): ... def __call__(self, cookies: Union[RequestsCookieJar, Cookie, list, tuple, str, dict]) -> None: ... def remove(self, name: str) -> None: ... def clear(self) -> None: ... + + +class WebPageCookiesSetter(CookiesSetter, SessionCookiesSetter): + _page: Union[WebPage, WebPageTab] + + def __init__(self, page: SessionPage): ... + + def __call__(self, cookies: Union[RequestsCookieJar, Cookie, list, tuple, str, dict]) -> None: ... + + def remove(self, name: str, url: str = None, domain: str = None, path: str = None) -> None: ... + + def clear(self) -> None: ... diff --git a/DrissionPage/_units/rect.py b/DrissionPage/_units/rect.py index 93ac879..ac1a8a9 100644 --- a/DrissionPage/_units/rect.py +++ b/DrissionPage/_units/rect.py @@ -30,7 +30,8 @@ class ElementRect(object): @property def size(self): """返回元素大小，格式(宽, 高)""" - border = self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele._backend_id)['model']['border'] + border = self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele._backend_id, + nodeId=self._ele._node_id, objectId=self._ele._obj_id)['model']['border'] return border[2] - border[0], border[5] - border[1] @property @@ -98,7 +99,8 @@ class ElementRect(object): :param quad: 方框类型，margin border padding :return: 四个角坐标 """ - return self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele._backend_id)['model'][quad] + return self._ele.page.run_cdp('DOM.getBoxModel', backendNodeId=self._ele._backend_id, + nodeId=self._ele._node_id, objectId=self._ele._obj_id)['model'][quad] def _get_page_coord(self, x, y): """根据视口坐标获取绝对坐标""" diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index 683ee10..8f8b1e1 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -7,10 +7,9 @@ from pathlib import Path from requests.structures import CaseInsensitiveDict -from .cookies_setter import SessionCookiesSetter, CookiesSetter +from .cookies_setter import SessionCookiesSetter, CookiesSetter, WebPageCookiesSetter from .._functions.tools import show_or_hide_browser -__ERROR__ = 'error' class BasePageSetter(object): def __init__(self, page): @@ -199,19 +198,6 @@ class TabSetter(ChromiumBaseSetter): """使标签页处于最前面""" self._page.browser.activate_tab(self._page.tab_id) - def add_init_script(self, script: str, raise_error=True): - '''添加初始化脚本，在页面加载任何脚本前执行 - :param script: js文本 - :return: identifier 添加的脚本的标识符，失败时返回False，或raise Error - ''' - result = self.driver.run('Page.addScriptToEvaluateOnNewDocument', source=script) - if not result or __ERROR__ not in result: - return result['identifier'] - else: - if raise_error: - raise_error(str(result)) - return False - class ChromiumPageSetter(TabSetter): @@ -384,15 +370,12 @@ class WebPageSetter(ChromiumPageSetter): self._session_setter = SessionPageSetter(self._page) self._chromium_setter = ChromiumPageSetter(self._page) - def cookies(self, cookies): - """添加cookies信息到浏览器或session对象 - :param cookies: 可以接收`CookieJar`、`list`、`tuple`、`str`、`dict`格式的`cookies` - :return: None - """ - if self._page.mode == 'd' and self._page._has_driver: - self._chromium_setter.cookies(cookies) - elif self._page.mode == 's' and self._page._has_session: - self._session_setter.cookies(cookies) + @property + def cookies(self): + """返回用于设置cookies的对象""" + if self._cookies_setter is None: + self._cookies_setter = WebPageCookiesSetter(self._page) + return self._cookies_setter def headers(self, headers) -> None: """设置固定发送的headers @@ -418,15 +401,12 @@ class WebPageTabSetter(TabSetter): self._session_setter = SessionPageSetter(self._page) self._chromium_setter = ChromiumBaseSetter(self._page) - def cookies(self, cookies): - """添加多个cookies信息到浏览器或session对象，注意不要传入单个 - :param cookies: 可以接收`CookieJar`、`list`、`tuple`、`str`、`dict`格式的`cookies` - :return: None - """ - if self._page.mode == 'd' and self._page._has_driver: - self._chromium_setter.cookies(cookies) - elif self._page.mode == 's' and self._page._has_session: - self._session_setter.cookies(cookies) + @property + def cookies(self): + """返回用于设置cookies的对象""" + if self._cookies_setter is None: + self._cookies_setter = WebPageCookiesSetter(self._page) + return self._cookies_setter def headers(self, headers) -> None: """设置固定发送的headers diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index 415decb..0ac2f98 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -9,7 +9,7 @@ from typing import Union, Tuple, Literal, Any, Optional from requests.adapters import HTTPAdapter from requests.auth import HTTPBasicAuth -from .cookies_setter import SessionCookiesSetter, CookiesSetter +from .cookies_setter import SessionCookiesSetter, CookiesSetter, WebPageCookiesSetter from .scroller import PageScroller from .._base.base import BasePage from .._elements.chromium_element import ChromiumElement @@ -143,7 +143,8 @@ class WebPageSetter(ChromiumPageSetter): def headers(self, headers: dict) -> None: ... - def cookies(self, cookies) -> None: ... + @property + def cookies(self) -> WebPageCookiesSetter: ... class WebPageTabSetter(TabSetter): @@ -155,7 +156,8 @@ class WebPageTabSetter(TabSetter): def headers(self, headers: dict) -> None: ... - def cookies(self, cookies) -> None: ... + @property + def cookies(self) -> WebPageCookiesSetter: ... class ChromiumElementSetter(object): diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 3fed08e..84a6676 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -412,7 +412,7 @@ class ElementWaiter(object): while perf_counter() < end_time: sleep(gap) - if self._ele.rect.size == size and location == self._ele.rect.location: + if self._ele.rect.size == size and self._ele.rect.location == location: return True size = self._ele.rect.size location = self._ele.rect.location From d1a87654a1f30003514b1204d94bd2106e762a35 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 31 Dec 2023 11:02:27 +0800 Subject: [PATCH 170/182] =?UTF-8?q?=E5=8F=AF=E6=8C=87=E5=AE=9A=E4=B8=B4?= =?UTF-8?q?=E6=97=B6=E6=96=87=E4=BB=B6=E5=A4=B9(+)=20auto=5Fport()?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0tmp=5Fpath=E5=8F=82=E6=95=B0=EF=BC=9B=20ini?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0tmp=5Fpath=E9=A1=B9=EF=BC=9B=20co=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0set=5Ftmp=5Fpath()=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_configs/chromium_options.py | 30 +++++++++++++++++----- DrissionPage/_configs/chromium_options.pyi | 8 +++++- DrissionPage/_configs/configs.ini | 3 ++- DrissionPage/_pages/chromium_page.py | 4 +-- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index c055d0b..d05d09c 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -32,6 +32,7 @@ class ChromiumOptions(object): options = om.chromium_options self._download_path = om.paths.get('download_path', None) or None + self._tmp_path = om.paths.get('tmp_path', None) or None self._arguments = options.get('arguments', []) self._browser_path = options.get('browser_path', '') self._extensions = options.get('extensions', []) @@ -76,6 +77,7 @@ class ChromiumOptions(object): self._browser_path = "chrome" self._arguments = [] self._download_path = None + self._tmp_path = None self._extensions = [] self._prefs = {} self._flags = {} @@ -445,6 +447,14 @@ class ChromiumOptions(object): self._download_path = str(path) return self + def set_tmp_path(self, path): + """设置临时文件文件保存路径 + :param path: 下载路径 + :return: 当前对象 + """ + self._tmp_path = str(path) + return self + def set_user_data_path(self, path): """设置用户文件夹路径 :param path: 用户文件夹路径 @@ -472,13 +482,15 @@ class ChromiumOptions(object): self._system_user_path = on_off return self - def auto_port(self, on_off=True): + def auto_port(self, on_off=True, tmp_path=None): """自动获取可用端口 :param on_off: 是否开启自动获取端口号 + :param tmp_path: 临时文件保存路径，为None时保存到系统临时文件夹 :return: 当前对象 """ if on_off: - port, path = PortFinder().get_port() + tmp_path = tmp_path or self._tmp_path + port, path = PortFinder(tmp_path).get_port() self.set_paths(local_port=port, user_data_path=path) self._auto_port = True else: @@ -527,6 +539,7 @@ class ChromiumOptions(object): om.set_item('proxies', 'https', self._proxy) # 设置路径 om.set_item('paths', 'download_path', self._download_path or '') + om.set_item('paths', 'tmp_path', self._tmp_path or '') # 设置timeout om.set_item('timeouts', 'base', self._timeouts['base']) om.set_item('timeouts', 'page_load', self._timeouts['pageLoad']) @@ -546,6 +559,9 @@ class ChromiumOptions(object): """保存当前配置到默认ini文件""" return self.save('default') + def __repr__(self): + return f'' + # ---------------即将废弃-------------- @property @@ -593,16 +609,16 @@ class ChromiumOptions(object): on_off = None if on_off else False return self.set_argument('--mute-audio', on_off) - def __repr__(self): - return f'' - class PortFinder(object): used_port = {} lock = Lock() - def __init__(self): - self.tmp_dir = Path(gettempdir()) / 'DrissionPage' / 'TempFolder' + def __init__(self, path=None): + """ + :param path: 临时文件保存路径，为None时使用系统临时文件夹 + """ + self.tmp_dir = Path(path) if path else Path(gettempdir()) / 'DrissionPage' / 'UserTempFolder' self.tmp_dir.mkdir(parents=True, exist_ok=True) if not PortFinder.used_port: clean_folder(self.tmp_dir) diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index 5531f57..201a107 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -14,6 +14,7 @@ class ChromiumOptions(object): self._driver_path: str = ... self._user_data_path: str = ... self._download_path: str = ... + self._tmp_path: str = ... self._arguments: list = ... self._browser_path: str = ... self._user: str = ... @@ -132,6 +133,8 @@ class ChromiumOptions(object): def set_download_path(self, path: Union[str, Path]) -> ChromiumOptions: ... + def set_tmp_path(self, path: Union[str, Path]) -> ChromiumOptions: ... + def set_user_data_path(self, path: Union[str, Path]) -> ChromiumOptions: ... def set_cache_path(self, path: Union[str, Path]) -> ChromiumOptions: ... @@ -142,7 +145,7 @@ class ChromiumOptions(object): def use_system_user_path(self, on_off: bool = True) -> ChromiumOptions: ... - def auto_port(self, on_off: bool = True) -> ChromiumOptions: ... + def auto_port(self, on_off: bool = True, tmp_path: Union[str, Path] = None) -> ChromiumOptions: ... def existing_only(self, on_off: bool = True) -> ChromiumOptions: ... @@ -154,6 +157,9 @@ class ChromiumOptions(object): class PortFinder(object): used_port: dict = ... lock: Lock = ... + tmp_dir: Path = ... + + def __init__(self, path: Union[str, Path] = None): ... @staticmethod def get_port() -> Tuple[int, str]: ... diff --git a/DrissionPage/_configs/configs.ini b/DrissionPage/_configs/configs.ini index 4355442..2eb7adb 100644 --- a/DrissionPage/_configs/configs.ini +++ b/DrissionPage/_configs/configs.ini @@ -1,5 +1,6 @@ [paths] -download_path = +download_path = +tmp_path = [chromium_options] address = 127.0.0.1:9222 diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 2bfb04d..6a3de65 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -65,12 +65,12 @@ class ChromiumPage(ChromiumBase): try: ws = get(f'http://{self._chromium_options.address}/json/version', headers={'Connection': 'close'}) if not ws: - raise BrowserConnectError('\n浏览器连接失败，请检查是否启用全局代理。如是，须设置不代理127.0.0.1地址。') + raise BrowserConnectError('\n浏览器连接失败，如使用全局代理，须设置不代理127.0.0.1地址。') ws = ws.json()['webSocketDebuggerUrl'].split('/')[-1] except KeyError: raise BrowserConnectError('浏览器版本太旧，请升级。') except: - raise BrowserConnectError('\n浏览器连接失败，请检查是否启用全局代理。如是，须设置不代理127.0.0.1地址。') + raise BrowserConnectError('\n浏览器连接失败，如使用全局代理，须设置不代理127.0.0.1地址。') self._browser = Browser(self._chromium_options.address, ws, self) if (is_exist and self._chromium_options._headless is False and From f2e147a7e2be398929e0ccac0c5300bf29560308 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 1 Jan 2024 23:34:34 +0800 Subject: [PATCH 171/182] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dis=5Fdisplayed?= =?UTF-8?q?=E5=B0=8F=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_units/states.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index 2abbcf5..9736a33 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -27,9 +27,8 @@ class ElementStates(object): @property def is_displayed(self): """返回元素是否显示""" - return not (self._ele.style('visibility') == 'hidden' - or self._ele.run_js('return this.offsetParent === null;') - or self._ele.style('display') == 'none') + return not (self._ele.style('visibility') == 'hidden' or self._ele.run_js('return this.offsetParent === null;') + or self._ele.style('display') == 'none' or self._ele.prop('hidden')) @property def is_enabled(self): From a20fafebd7bde69555583f394105247c1288bd2d Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 2 Jan 2024 15:26:13 +0800 Subject: [PATCH 172/182] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=A4=84=E7=90=86?= =?UTF-8?q?=E7=AB=8B=E5=8D=B3=E6=89=A7=E8=A1=8C=E7=9A=84=E5=8A=A8=E4=BD=9C?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/driver.py | 37 ++++++++++++-------- DrissionPage/_base/driver.pyi | 10 +++--- DrissionPage/_pages/chromium_base.py | 50 ++------------------------- DrissionPage/_pages/chromium_base.pyi | 1 - DrissionPage/_pages/chromium_frame.py | 17 ++------- 5 files changed, 34 insertions(+), 81 deletions(-) diff --git a/DrissionPage/_base/driver.py b/DrissionPage/_base/driver.py index 28b8642..3a86da4 100644 --- a/DrissionPage/_base/driver.py +++ b/DrissionPage/_base/driver.py @@ -36,6 +36,7 @@ class Driver(object): self._handle_event_th = Thread(target=self._handle_event_loop) self._recv_th.daemon = True self._handle_event_th.daemon = True + self._handle_immediate_event_th = None self._stopped = Event() @@ -43,6 +44,7 @@ class Driver(object): self.immediate_event_handlers = {} self.method_results = {} self.event_queue = Queue() + self.immediate_event_queue = Queue() self.start() @@ -126,8 +128,7 @@ class Driver(object): self.alert_flag = msg['method'].endswith('Opening') function = self.immediate_event_handlers.get(msg['method']) if function: - Thread(target=run_function, args=(function, msg['params'])).start() - # function(**msg['params']) + self._handle_immediate_event(function, msg['params']) else: self.event_queue.put(msg) @@ -151,6 +152,26 @@ class Driver(object): self.event_queue.task_done() + def _handle_immediate_event_loop(self): + while not self._stopped.is_set() and not self.immediate_event_queue.empty(): + function, kwargs = self.immediate_event_queue.get(timeout=1) + try: + function(**kwargs) + except PageDisconnectedError: + pass + + def _handle_immediate_event(self, function, kwargs): + """处理立即执行的动作 + :param function: 要运行下方法 + :param kwargs: 方法参数 + :return: None + """ + self.immediate_event_queue.put((function, kwargs)) + if self._handle_immediate_event_th is None or not self._handle_immediate_event_th.is_alive(): + self._handle_immediate_event_th = Thread(target=self._handle_immediate_event_loop) + self._handle_immediate_event_th.daemon = True + self._handle_immediate_event_th.start() + def run(self, _method, **kwargs): """执行cdp方法 :param _method: cdp方法名 @@ -220,11 +241,6 @@ class Driver(object): else: handler.pop(event, None) - def __str__(self): - return f'' - - __repr__ = __str__ - class BrowserDriver(Driver): BROWSERS = {} @@ -253,10 +269,3 @@ class BrowserDriver(Driver): def stop(self): super().stop() self.browser._on_quit() - - -def run_function(function, kwargs): - try: - function(**kwargs) - except PageDisconnectedError: - pass diff --git a/DrissionPage/_base/driver.pyi b/DrissionPage/_base/driver.pyi index 2d23d91..cdabcb2 100644 --- a/DrissionPage/_base/driver.pyi +++ b/DrissionPage/_base/driver.pyi @@ -25,18 +25,20 @@ class Driver(object): id: str address: str type: str - _debug: bool + # _debug: bool alert_flag: bool _websocket_url: str _cur_id: int _ws: Optional[WebSocket] _recv_th: Thread _handle_event_th: Thread + _handle_immediate_event_th: Optional[Thread] _stopped: Event event_handlers: dict immediate_event_handlers: dict method_results: dict event_queue: Queue + immediate_event_queue: Queue def __init__(self, tab_id: str, tab_type: str, address: str): ... @@ -46,7 +48,9 @@ class Driver(object): def _handle_event_loop(self) -> None: ... - def __getattr__(self, item: str) -> Callable: ... + def _handle_immediate_event_loop(self): ... + + def _handle_immediate_event(self, function: Callable, kwargs: dict): ... def run(self, _method: str, **kwargs) -> dict: ... @@ -58,8 +62,6 @@ class Driver(object): def set_callback(self, event: str, callback: Union[Callable, None], immediate: bool = False) -> None: ... - def __str__(self) -> str: ... - class BrowserDriver(Driver): BROWSERS: Dict[str, Driver] = ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index d1eadd0..5f3ab31 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -44,7 +44,6 @@ class ChromiumBase(BasePage): super().__init__() self._is_loading = None self._root_id = None # object id - self._debug = False self._set = None self._screencast = None self._actions = None @@ -114,8 +113,6 @@ class ChromiumBase(BasePage): if self._js_ready_state == 'complete' and self._ready_state is None: self._get_document() self._ready_state = 'complete' - if self._debug: - print(f'{self._frame_id}在connect_browser变成complete') def _driver_init(self, tab_id): """新建页面、页面刷新、切换标签页后要进行的cdp参数初始化 @@ -145,15 +142,13 @@ class ChromiumBase(BasePage): self._driver.set_callback('Page.loadEventFired', self._onLoadEventFired) self._driver.set_callback('Page.frameStoppedLoading', self._onFrameStoppedLoading) self._driver.set_callback('Page.frameAttached', self._onFrameAttached) - self._driver.set_callback('Page.frameDetached', self._onFrameDetached, immediate=True) + self._driver.set_callback('Page.frameDetached', self._onFrameDetached) def _get_document(self, timeout=10): """获取页面文档 :param timeout: 超时时间（秒） :return: 是否获取成功 """ - if self._debug: - print('获取文档开始') if self._is_reading: return timeout = timeout if timeout >= .5 else .5 @@ -169,8 +164,6 @@ class ChromiumBase(BasePage): r = self.run_cdp('Page.getFrameTree') for i in findall(r"'id': '(.*?)'", str(r)): self.browser._frames[i] = self.tab_id - if self._debug: - print('获取文档结束') result = True break @@ -195,10 +188,6 @@ class ChromiumBase(BasePage): """页面开始加载时执行""" self.browser._frames[kwargs['frameId']] = self.tab_id if kwargs['frameId'] == self._frame_id: - if self._debug: - print(f'{self._frame_id}触发FrameStartedLoading') - print('在FrameStartedLoading变成loading') - self._doc_got = False self._ready_state = 'connecting' self._is_loading = True @@ -208,66 +197,35 @@ class ChromiumBase(BasePage): t.daemon = True t.start() - if self._debug: - print(f'{self._frame_id}执行FrameStartedLoading完毕') - def _onFrameNavigated(self, **kwargs): """页面跳转时执行""" if kwargs['frame']['id'] == self._frame_id: - if self._debug: - print(f'{self._frame_id}触发FrameNavigated') - print('在FrameNavigated变成loading') - self._doc_got = False self._ready_state = 'loading' self._is_loading = True - if self._debug: - print(f'>>> FrameNavigated {kwargs}') - def _onDomContentEventFired(self, **kwargs): """在页面刷新、变化后重新读取页面内容""" - if self._debug: - print(f'{self._frame_id}触发DomContentEventFired') - print('在DomContentEventFired变成interactive') - if self._load_mode == 'eager': self.run_cdp('Page.stopLoading') if self._get_document(self._load_end_time - perf_counter() - .1): self._doc_got = True self._ready_state = 'interactive' - if self._debug: - print(f'{self._frame_id}执行DomContentEventFired完毕') - def _onLoadEventFired(self, **kwargs): """在页面刷新、变化后重新读取页面内容""" - if self._debug: - print(f'{self._frame_id}触发LoadEventFired') - print('在LoadEventFired变成complete') - if self._doc_got is False and self._get_document(self._load_end_time - perf_counter() - .1): self._doc_got = True self._ready_state = 'complete' - if self._debug: - print(f'{self._frame_id}执行LoadEventFired完毕') - def _onFrameStoppedLoading(self, **kwargs): """页面加载完成后执行""" self.browser._frames[kwargs['frameId']] = self.tab_id if kwargs['frameId'] == self._frame_id: - if self._debug: - print(f'{self._frame_id}触发FrameStoppedLoading') - print('在FrameStoppedLoading变成complete') - if self._doc_got is False: self._get_document(self._load_end_time - perf_counter() - .1) self._ready_state = 'complete' - if self._debug: - print(f'{self._frame_id}执行FrameStoppedLoading完毕') - def _onFileChooserOpened(self, **kwargs): """文件选择框打开时执行""" if self._upload_list: @@ -669,8 +627,6 @@ class ChromiumBase(BasePage): def stop_loading(self): """页面停止加载""" - if self._debug: - print('停止页面加载') try: self.run_cdp('Page.stopLoading') except (PageDisconnectedError, CDPError): @@ -974,7 +930,7 @@ class ChromiumBase(BasePage): if err: if t < times: sleep(interval) - if self._debug or show_errmsg: + if show_errmsg: print(f'重试{t + 1} {to_url}') end_time1 = end_time - perf_counter() while self._ready_state not in ('loading', 'complete') and perf_counter() < end_time1: # 等待出错信息显示 @@ -991,7 +947,7 @@ class ChromiumBase(BasePage): err = TimeoutError(f'页面连接超时（等待{timeout}秒）。') if t < times: sleep(interval) - if self._debug or show_errmsg: + if show_errmsg: print(f'重试{t + 1} {to_url}') continue diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index 5e107c4..ab120ec 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -46,7 +46,6 @@ class ChromiumBase(BasePage): self._scroll: Scroller = ... self._url: str = ... self._root_id: str = ... - self._debug: bool = ... self._upload_list: list = ... self._wait: BaseWaiter = ... self._set: ChromiumBaseSetter = ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 80e0460..838096f 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -99,16 +99,15 @@ class ChromiumFrame(ChromiumBase): self.browser.driver.get(f'http://{self.address}/json') super()._driver_init(tab_id) self._driver.set_callback('Inspector.detached', self._onInspectorDetached, immediate=True) + self._driver.set_callback('Page.frameDetached', None) + self._driver.set_callback('Page.frameDetached', self._onFrameDetached, immediate=True) def _reload(self): """重新获取document""" self._is_loading = True - debug = self._debug d_debug = self.driver._debug self._reloading = True self._doc_got = False - if debug: - print(f'{self._frame_id} reload 开始') self._driver.stop() try: @@ -132,7 +131,6 @@ class ChromiumFrame(ChromiumBase): if self._listener: self._listener._to_target(self._target_page.tab_id, self.address, self) super().__init__(self.address, self._target_page.tab_id, self._target_page.timeout) - self._debug = debug self.driver._debug = d_debug else: @@ -156,15 +154,11 @@ class ChromiumFrame(ChromiumBase): # print(f'获取doc失败，重试 {e}') # else: # raise GetDocumentError - self._debug = debug self.driver._debug = d_debug self._is_loading = False self._reloading = False - if self._debug: - print(f'{self._frame_id} reload 完毕') - def _get_document(self, timeout=10): """刷新cdp使用的document数据 :param timeout: 超时时间（秒） @@ -173,9 +167,6 @@ class ChromiumFrame(ChromiumBase): if self._is_reading: return - if self._debug: - print('获取文档开始') - self._is_reading = True try: if self._is_diff_domain is False: @@ -192,13 +183,9 @@ class ChromiumFrame(ChromiumBase): r = self.run_cdp('Page.getFrameTree') for i in findall(r"'id': '(.*?)'", str(r)): self.browser._frames[i] = self.tab_id - if self._debug: - print('获取文档结束') return True except: - if self._debug: - print('获取文档失败') return False finally: From 2986e3eeb124e99416673e584b5892977802d6e4 Mon Sep 17 00:00:00 2001 From: g1879 Date: Tue, 2 Jan 2024 22:51:40 +0800 Subject: [PATCH 173/182] =?UTF-8?q?4.0.0b33(+)=20co=E5=A2=9E=E5=8A=A0tmp?= =?UTF-8?q?=5Fpath=E5=92=8Cis=5Fauto=5Fport=E5=B1=9E=E6=80=A7=EF=BC=9B=20a?= =?UTF-8?q?uto=5Fport=E5=9C=A8=E5=88=9B=E5=BB=BA=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=E6=97=B6=E6=89=8D=E7=A1=AE=E5=AE=9A=E7=AB=AF=E5=8F=A3=E5=92=8C?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=EF=BC=9B=20auto=5Fport=E7=9A=84=E5=AF=B9?= =?UTF-8?q?=E8=B1=A1=E5=9C=A8=E6=B5=8F=E8=A7=88=E5=99=A8=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E6=97=B6=E6=83=85=E5=86=B5=E7=94=A8=E6=88=B7=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=A4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_base/browser.py | 13 +++++++ DrissionPage/_base/driver.py | 44 +++++++++++----------- DrissionPage/_configs/chromium_options.py | 20 +++++++--- DrissionPage/_configs/chromium_options.pyi | 8 +++- DrissionPage/_pages/chromium_frame.py | 7 ++-- DrissionPage/_pages/chromium_page.py | 7 +++- DrissionPage/_units/actions.py | 1 - setup.py | 2 +- 9 files changed, 69 insertions(+), 35 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 3a5cb9f..859c24c 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b32' +__version__ = '4.0.0b33' diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 2d12911..ab17493 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -3,6 +3,8 @@ @Author : g1879 @Contact : g1879@qq.com """ +from pathlib import Path +from shutil import rmtree from time import sleep, perf_counter from .driver import BrowserDriver, Driver @@ -196,3 +198,14 @@ class Browser(object): def _on_quit(self): Browser.BROWSERS.pop(self.id, None) + if self.page._chromium_options.is_auto_port and self.page._chromium_options.user_data_path: + path = Path(self.page._chromium_options.user_data_path) + end_time = perf_counter() + 7 + while perf_counter() < end_time: + if not path.exists(): + break + try: + rmtree(path) + break + except (PermissionError, FileNotFoundError): + pass diff --git a/DrissionPage/_base/driver.py b/DrissionPage/_base/driver.py index 3a86da4..809c747 100644 --- a/DrissionPage/_base/driver.py +++ b/DrissionPage/_base/driver.py @@ -59,15 +59,15 @@ class Driver(object): message['id'] = ws_id message_json = dumps(message) - if self._debug: - if self._debug is True or (isinstance(self._debug, str) and - message.get('method', '').startswith(self._debug)): - print(f'发> {message_json}') - elif isinstance(self._debug, (list, tuple, set)): - for m in self._debug: - if message.get('method', '').startswith(m): - print(f'发> {message_json}') - break + # if self._debug: + # if self._debug is True or (isinstance(self._debug, str) and + # message.get('method', '').startswith(self._debug)): + # print(f'发> {message_json}') + # elif isinstance(self._debug, (list, tuple, set)): + # for m in self._debug: + # if message.get('method', '').startswith(m): + # print(f'发> {message_json}') + # break end_time = perf_counter() + timeout if timeout is not None else None self.method_results[ws_id] = Queue() @@ -113,15 +113,15 @@ class Driver(object): self._stop() return - if self._debug: - if self._debug is True or 'id' in msg or (isinstance(self._debug, str) - and msg.get('method', '').startswith(self._debug)): - print(f'<收 {msg_json}') - elif isinstance(self._debug, (list, tuple, set)): - for m in self._debug: - if msg.get('method', '').startswith(m): - print(f'<收 {msg_json}') - break + # if self._debug: + # if self._debug is True or 'id' in msg or (isinstance(self._debug, str) + # and msg.get('method', '').startswith(self._debug)): + # print(f'<收 {msg_json}') + # elif isinstance(self._debug, (list, tuple, set)): + # for m in self._debug: + # if msg.get('method', '').startswith(m): + # print(f'<收 {msg_json}') + # break if 'method' in msg: if msg['method'].startswith('Page.javascriptDialog'): @@ -135,8 +135,8 @@ class Driver(object): elif msg.get('id') in self.method_results: self.method_results[msg['id']].put(msg) - elif self._debug: - print(f'未知信息：{msg}') + # elif self._debug: + # print(f'未知信息：{msg}') def _handle_event_loop(self): """当接收到浏览器信息，执行已绑定的方法""" @@ -266,6 +266,6 @@ class BrowserDriver(Driver): r.close() return r - def stop(self): - super().stop() + def _stop(self): + super()._stop() self.browser._on_quit() diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index d05d09c..342008e 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -106,6 +106,11 @@ class ChromiumOptions(object): """返回用户数据文件夹路径""" return self._user_data_path + @property + def tmp_path(self): + """返回临时文件夹路径""" + return self._tmp_path + @property def user(self): """返回用户配置文件夹名称""" @@ -161,6 +166,11 @@ class ChromiumOptions(object): """返回是否只接管现有浏览器方式""" return self._existing_only + @property + def is_auto_port(self): + """返回是否使用自动端口和用户文件""" + return self._auto_port + @property def retry_times(self): """返回连接失败时的重试次数""" @@ -485,14 +495,13 @@ class ChromiumOptions(object): def auto_port(self, on_off=True, tmp_path=None): """自动获取可用端口 :param on_off: 是否开启自动获取端口号 - :param tmp_path: 临时文件保存路径，为None时保存到系统临时文件夹 + :param tmp_path: 临时文件保存路径，为None时保存到系统临时文件夹，on_off为False时此参数无效 :return: 当前对象 """ if on_off: - tmp_path = tmp_path or self._tmp_path - port, path = PortFinder(tmp_path).get_port() - self.set_paths(local_port=port, user_data_path=path) self._auto_port = True + if tmp_path: + self._tmp_path = str(tmp_path) else: self._auto_port = False return self @@ -618,7 +627,8 @@ class PortFinder(object): """ :param path: 临时文件保存路径，为None时使用系统临时文件夹 """ - self.tmp_dir = Path(path) if path else Path(gettempdir()) / 'DrissionPage' / 'UserTempFolder' + tmp = Path(path) if path else Path(gettempdir()) / 'DrissionPage' + self.tmp_dir = tmp / 'UserTempFolder' self.tmp_dir.mkdir(parents=True, exist_ok=True) if not PortFinder.used_port: clean_folder(self.tmp_dir) diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index 201a107..8fd4ab9 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -5,7 +5,7 @@ """ from pathlib import Path from threading import Lock -from typing import Union, Tuple, Any, Literal +from typing import Union, Tuple, Any, Literal, Optional class ChromiumOptions(object): @@ -43,6 +43,9 @@ class ChromiumOptions(object): @property def user_data_path(self) -> str: ... + @property + def tmp_path(self) -> Optional[str]: ... + @property def user(self) -> str: ... @@ -76,6 +79,9 @@ class ChromiumOptions(object): @property def is_existing_only(self) -> bool: ... + @property + def is_auto_port(self) -> bool: ... + @property def retry_times(self) -> int: ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 838096f..290b9a5 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -105,7 +105,7 @@ class ChromiumFrame(ChromiumBase): def _reload(self): """重新获取document""" self._is_loading = True - d_debug = self.driver._debug + # d_debug = self.driver._debug self._reloading = True self._doc_got = False @@ -131,7 +131,7 @@ class ChromiumFrame(ChromiumBase): if self._listener: self._listener._to_target(self._target_page.tab_id, self.address, self) super().__init__(self.address, self._target_page.tab_id, self._target_page.timeout) - self.driver._debug = d_debug + # self.driver._debug = d_debug else: self._is_diff_domain = True @@ -154,7 +154,8 @@ class ChromiumFrame(ChromiumBase): # print(f'获取doc失败，重试 {e}') # else: # raise GetDocumentError - self.driver._debug = d_debug + + # self.driver._debug = d_debug self._is_loading = False self._reloading = False diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index 6a3de65..ff33dfd 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -10,7 +10,7 @@ from requests import get from .._base.browser import Browser from .._functions.browser import connect_browser -from .._configs.chromium_options import ChromiumOptions +from .._configs.chromium_options import ChromiumOptions, PortFinder from .._pages.chromium_base import ChromiumBase, get_mhtml, Timeout from .._pages.chromium_tab import ChromiumTab from .._units.setter import ChromiumPageSetter @@ -44,6 +44,11 @@ class ChromiumPage(ChromiumBase): self._chromium_options = ChromiumOptions(addr_or_opts) elif isinstance(addr_or_opts, ChromiumOptions): + if addr_or_opts.is_auto_port: + port, path = PortFinder(addr_or_opts.tmp_path).get_port() + addr_or_opts.set_address(f'127.0.0.1:{port}') + addr_or_opts.set_user_data_path(path) + addr_or_opts.auto_port() self._chromium_options = addr_or_opts elif isinstance(addr_or_opts, str): diff --git a/DrissionPage/_units/actions.py b/DrissionPage/_units/actions.py index 11c9ef7..7f47b11 100644 --- a/DrissionPage/_units/actions.py +++ b/DrissionPage/_units/actions.py @@ -283,7 +283,6 @@ class Actions: if character in ('\ue009', '\ue008', '\ue00a', '\ue03d'): modifiers.append(character) else: - sleep(.01) self.key_up(character) for m in modifiers: self.key_up(m) diff --git a/setup.py b/setup.py index 102fcd8..424413e 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b32", + version="4.0.0b33", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From ecfa83dcf84fc93fada7fb4431d23bebe1045480 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 3 Jan 2024 00:06:23 +0800 Subject: [PATCH 174/182] =?UTF-8?q?=E5=BD=95=E5=83=8F=E8=BF=87=E7=A8=8B?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=94=BE=E5=88=B0=E4=B8=B4=E6=97=B6=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=A4=B9=EF=BC=9B=E5=BE=AE=E8=B0=83=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 18 +++++++++--------- DrissionPage/_pages/chromium_base.pyi | 4 ++-- DrissionPage/_units/screencast.py | 7 ++++++- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 5f3ab31..87d0c70 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -753,29 +753,29 @@ class ChromiumBase(BasePage): return self._get_screenshot(path=path, name=name, as_bytes=as_bytes, as_base64=as_base64, full_page=full_page, left_top=left_top, right_bottom=right_bottom) - def add_init_js(self, js): + def add_init_js(self, script): """添加初始化脚本，在页面加载任何脚本前执行 - :param js: js文本 + :param script: js文本 :return: 添加的脚本的id """ - js_id = self.run_cdp('Page.addScriptToEvaluateOnNewDocument', source=js, + js_id = self.run_cdp('Page.addScriptToEvaluateOnNewDocument', source=script, includeCommandLineAPI=True)['identifier'] self._init_jss.append(js_id) return js_id - def remove_init_js(self, js_id=None): + def remove_init_js(self, script_id=None): """删除初始化脚本，js_id传入None时删除所有 - :param js_id: 脚本的id + :param script_id: 脚本的id :return: None """ - if js_id is None: + if script_id is None: for js_id in self._init_jss: self.run_cdp('Page.removeScriptToEvaluateOnNewDocument', identifier=js_id) self._init_jss.clear() - elif js_id in self._init_jss: - self.run_cdp('Page.removeScriptToEvaluateOnNewDocument', identifier=js_id) - self._init_jss.remove(js_id) + elif script_id in self._init_jss: + self.run_cdp('Page.removeScriptToEvaluateOnNewDocument', identifier=script_id) + self._init_jss.remove(script_id) def clear_cache(self, session_storage=True, local_storage=True, cache=True, cookies=True): """清除缓存，可选要清除的项 diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index ab120ec..a24ddd8 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -214,9 +214,9 @@ class ChromiumBase(BasePage): def get_local_storage(self, item: str = None) -> Union[str, dict, None]: ... - def add_init_js(self, js: str) -> str: ... + def add_init_js(self, script: str) -> str: ... - def remove_init_js(self, js_id: str = None) -> None: ... + def remove_init_js(self, script_id: str = None) -> None: ... def get_screenshot(self, path: [str, Path] = None, name: str = None, as_bytes: PIC_TYPE = None, as_base64: PIC_TYPE = None, full_page: bool = False, left_top: Tuple[int, int] = None, diff --git a/DrissionPage/_units/screencast.py b/DrissionPage/_units/screencast.py index ab60b83..b752f27 100644 --- a/DrissionPage/_units/screencast.py +++ b/DrissionPage/_units/screencast.py @@ -8,6 +8,7 @@ from os.path import sep from pathlib import Path from random import randint from shutil import rmtree +from tempfile import gettempdir from threading import Thread from time import sleep, time @@ -36,7 +37,11 @@ class Screencast(object): raise ValueError('save_path必须设置。') if self._mode in ('frugal_video', 'video'): - self._tmp_path = self._path / f'screencast_tmp_{time()}_{randint(0, 100)}' + if self._page.browser.page._chromium_options.tmp_path: + self._tmp_path = Path( + self._page.browser.page._chromium_options.tmp_path) / f'screencast_tmp_{time()}_{randint(0, 100)}' + else: + self._tmp_path = Path(gettempdir()) / 'DrissionPage' / f'screencast_tmp_{time()}_{randint(0, 100)}' self._tmp_path.mkdir(parents=True, exist_ok=True) if self._mode.startswith('frugal'): From 5c8ba2da5829157f32fa0f01b93c0ab0fe81d2b4 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 3 Jan 2024 11:06:39 +0800 Subject: [PATCH 175/182] =?UTF-8?q?4.0.0b34=E4=BF=AE=E5=A4=8Dget()timeout?= =?UTF-8?q?=E5=BE=88=E7=9F=AD=E6=97=B6=E6=8A=A5=E9=94=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_functions/tools.py | 11 +++++++---- setup.py | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 859c24c..7ab5984 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b33' +__version__ = '4.0.0b34' diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index 81e9515..dcdae15 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -13,7 +13,7 @@ from psutil import process_iter, AccessDenied, NoSuchProcess, ZombieProcess from .._configs.options_manage import OptionsManager from ..errors import (ContextLostError, ElementLostError, CDPError, PageDisconnectedError, NoRectError, - AlertExistsError, WrongURLError, StorageError, CookieFormatError) + AlertExistsError, WrongURLError, StorageError, CookieFormatError, JavaScriptError) def get_usable_path(path, is_file=True, parents=True): @@ -279,12 +279,15 @@ def raise_error(result, ignore=None): r = StorageError() elif error == 'Sanitizing cookie failed': r = CookieFormatError(f'cookie格式不正确：{result["args"]}') + elif error == 'Given expression does not evaluate to a function': + r = JavaScriptError(f'传入的js无法解析成函数：\n{result["args"]["functionDeclaration"]}') elif result['type'] in ('call_method_error', 'timeout'): from DrissionPage import __version__ from time import process_time - r = CDPError(f'\n错误：{result["error"]}\nmethod：{result["method"]}\nargs：{result["args"]}\n' - f'版本：{__version__}\n运行时间：{process_time()}\n出现这个错误可能意味着程序有bug，请把错误信息和重现方法' - '告知作者，谢谢。\n报告网站：https://gitee.com/g1879/DrissionPage/issues') + txt = f'\n错误：{result["error"]}\nmethod：{result["method"]}\nargs：{result["args"]}\n' \ + f'版本：{__version__}\n运行时间：{process_time()}\n出现这个错误可能意味着程序有bug，请把错误信息和重现方法' \ + '告知作者，谢谢。\n报告网站：https://gitee.com/g1879/DrissionPage/issues' + r = TimeoutError(txt) if result['type'] == 'timeout' else CDPError(txt) else: r = RuntimeError(result) diff --git a/setup.py b/setup.py index 424413e..e304e1f 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b33", + version="4.0.0b34", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", From 1be8f00c2183afeb309e53c01c0b15547f2b8918 Mon Sep 17 00:00:00 2001 From: g1879 Date: Wed, 3 Jan 2024 17:40:35 +0800 Subject: [PATCH 176/182] =?UTF-8?q?=E5=BE=AE=E8=B0=83get()=E9=80=BB?= =?UTF-8?q?=E8=BE=91=EF=BC=9B=E4=B8=8D=E6=8C=87=E5=AE=9A=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=A4=B9=E8=B7=AF=E5=BE=84=E6=97=B6=E4=BC=98?= =?UTF-8?q?=E5=85=88=E9=80=89=E6=8B=A9tmp=5Fpath?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_functions/browser.py | 3 ++- DrissionPage/_pages/chromium_base.py | 10 ++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/DrissionPage/_functions/browser.py b/DrissionPage/_functions/browser.py index 33f0df6..99e1211 100644 --- a/DrissionPage/_functions/browser.py +++ b/DrissionPage/_functions/browser.py @@ -87,7 +87,8 @@ def get_launch_args(opt): if not has_user_path and not opt.system_user_path: port = opt.address.split(':')[-1] if opt.address else '0' - path = Path(gettempdir()) / 'DrissionPage' / f'userData_{port}' + p = Path(opt.tmp_path) if opt.tmp_path else Path(gettempdir()) / 'DrissionPage' + path = p / f'userData_{port}' path.mkdir(parents=True, exist_ok=True) opt.set_user_data_path(path) result.add(f'--user-data-dir={path}') diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 87d0c70..3b66a76 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -6,7 +6,7 @@ from json import loads, JSONDecodeError from os.path import sep from pathlib import Path -from re import findall, match +from re import findall from threading import Thread from time import perf_counter, sleep from urllib.parse import quote @@ -894,13 +894,7 @@ class ChromiumBase(BasePage): :param interval: 重试间隔 :return: 重试次数和间隔组成的tuple """ - url = quote(url, safe='-_.~!*\'"();:@&=+$,/\\?#[]%') - if not url: - self._url = 'chrome://newtab/' - elif not match(r'.*?://', url): - self._url = f'http://{url}' - else: - self._url = url + self._url = quote(url, safe='-_.~!*\'"();:@&=+$,/\\?#[]%') or 'chrome://newtab/' retry = retry if retry is not None else self.retry_times interval = interval if interval is not None else self.retry_interval return retry, interval From f13681e0df6af9de8669dbc398f5d58cd382507f Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 4 Jan 2024 00:28:46 +0800 Subject: [PATCH 177/182] =?UTF-8?q?ChromiumPage=E5=A2=9E=E5=8A=A0close()?= =?UTF-8?q?=EF=BC=8C=E5=88=A0=E9=99=A4close=5Fother=5Ftabs()=EF=BC=9B?= =?UTF-8?q?=E5=BE=AE=E8=B0=83quit()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_base/browser.py | 2 -- DrissionPage/_pages/chromium_page.py | 19 ++++++++++++------- DrissionPage/_pages/chromium_page.pyi | 5 ++--- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index ab17493..7d2e61f 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -169,9 +169,7 @@ class Browser(object): """ try: self.run_cdp('Browser.close') - self.driver.stop() except PageDisconnectedError: - self.driver.stop() return if force: diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index ff33dfd..c4a0a93 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -214,6 +214,10 @@ class ChromiumPage(ChromiumBase): return self.browser.run_cdp('Target.createTarget', **kwargs)['targetId'] + def close(self): + """关闭Page管理的标签页""" + self.browser.close_tab(self.tab_id) + def close_tabs(self, tabs_or_ids=None, others=False): """关闭传入的标签页，默认关闭当前页。可传入多个 :param tabs_or_ids: 要关闭的标签页对象或id，可传入列表或元组，为None时关闭当前页 @@ -247,13 +251,6 @@ class ChromiumPage(ChromiumBase): while self.tabs_count != end_len and perf_counter() < end_time: sleep(.1) - def close_other_tabs(self, tabs_or_ids=None): - """关闭传入的标签页以外标签页，默认保留当前页。可传入多个 - :param tabs_or_ids: 要保留的标签页对象或id，可传入列表或元组，为None时保存当前页 - :return: None - """ - self.close_tabs(tabs_or_ids, True) - def quit(self, timeout=5, force=True): """关闭浏览器 :param timeout: 等待浏览器关闭超时时间（秒） @@ -265,6 +262,14 @@ class ChromiumPage(ChromiumBase): def __repr__(self): return f'' + # ----------即将废弃----------- + def close_other_tabs(self, tabs_or_ids=None): + """关闭传入的标签页以外标签页，默认保留当前页。可传入多个 + :param tabs_or_ids: 要保留的标签页对象或id，可传入列表或元组，为None时保存当前页 + :return: None + """ + self.close_tabs(tabs_or_ids, True) + def get_rename(original, rename): if '.' in rename: diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index ac29e10..6617fde 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -67,12 +67,11 @@ class ChromiumPage(ChromiumBase): def _new_tab(self, new_window: bool = False, background: bool = False, new_context: bool = False) -> str: ... + def close(self) -> None: ... + def close_tabs(self, tabs_or_ids: Union[str, ChromiumTab, List[Union[str, ChromiumTab]], Tuple[Union[str, ChromiumTab]]] = None, others: bool = False) -> None: ... - def close_other_tabs(self, tabs_or_ids: Union[ - str, ChromiumTab, List[Union[str, ChromiumTab]], Tuple[Union[str, ChromiumTab]]] = None) -> None: ... - def quit(self, timeout: float = 5, force: bool = True) -> None: ... From dcbf9700752143cb34fd6b861268820e2d688e70 Mon Sep 17 00:00:00 2001 From: g1879 Date: Thu, 4 Jan 2024 23:14:24 +0800 Subject: [PATCH 178/182] =?UTF-8?q?check()=E5=A2=9E=E5=8A=A0by=5Fjs?= =?UTF-8?q?=E5=8F=82=E6=95=B0=EF=BC=9Binput()=E5=92=8Cclear()by=5Fjs?= =?UTF-8?q?=E6=97=B6=E8=A7=A6=E5=8F=91change=EF=BC=9B=E5=A2=9E=E5=8A=A0set?= =?UTF-8?q?.blocked=5Furls()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_elements/chromium_element.py | 21 ++++++++++++++++++--- DrissionPage/_elements/chromium_element.pyi | 2 +- DrissionPage/_units/setter.py | 12 ++++++++++++ DrissionPage/_units/setter.pyi | 2 ++ 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 790801f..1c94c37 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -201,13 +201,26 @@ class ChromiumElement(DrissionElement): return self._select - def check(self, uncheck=False): + def check(self, uncheck=False, by_js=False): """选中或取消选中当前元素 :param uncheck: 是否取消选中 + :param by_js: 是否用js执行 :return: None """ - js = 'this.checked=false' if uncheck else 'this.checked=true' - self.run_js(js) + is_checked = self.states.is_checked + if by_js: + js = None + if is_checked and uncheck: + js = 'this.checked=false' + elif not is_checked and not uncheck: + js = 'this.checked=true' + if js: + self.run_js(js) + self.run_js('this.dispatchEvent(new Event("change", {bubbles: true}));') + + else: + if (is_checked and uncheck) or (not is_checked and not uncheck): + self.click() def parent(self, level_or_loc=1, index=1): """返回上面某一级父元素，可指定层数或用查询语法定位 @@ -598,6 +611,7 @@ class ChromiumElement(DrissionElement): if isinstance(vals, (list, tuple)): vals = ''.join([str(i) for i in vals]) self.set.prop('value', str(vals)) + self.run_js('this.dispatchEvent(new Event("change", {bubbles: true}));') return if clear and vals not in ('\n', '\ue007'): @@ -614,6 +628,7 @@ class ChromiumElement(DrissionElement): """ if by_js: self.run_js("this.value='';") + self.run_js('this.dispatchEvent(new Event("change", {bubbles: true}));') return self._input_focus() diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 4e3f886..9eab619 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -159,7 +159,7 @@ class ChromiumElement(DrissionElement): @property def select(self) -> SelectElement: ... - def check(self, uncheck: bool = False) -> None: ... + def check(self, uncheck: bool = False, by_js: bool = False) -> None: ... def attr(self, attr: str) -> Union[str, None]: ... diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index 8f8b1e1..8e9ad19 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -145,6 +145,18 @@ class ChromiumBaseSetter(BasePageSetter): """ self._page._alert.auto = accept if on_off else None + def blocked_urls(self, urls): + """设置要忽略的url，传入None时清空已设置的内容。 + :param urls: + :return: None + """ + if not urls: + urls = [] + if not isinstance(urls, (list, tuple)): + raise TypeError('urls需传入list或tuple类型。') + self._page.run_cdp('Network.enable') + self._page.run_cdp('Network.setBlockedURLs', urls=urls) + # --------------即将废弃--------------- @property diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index 0ac2f98..f8e5de0 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -62,6 +62,8 @@ class ChromiumBaseSetter(BasePageSetter): def upload_files(self, files: Union[str, list, tuple]) -> None: ... + def blocked_urls(self, urls: Optional[list, tuple]) -> None: ... + class TabSetter(ChromiumBaseSetter): def __init__(self, page): ... From e56995dcf0449a52d1ca4140a22f3f49902be584 Mon Sep 17 00:00:00 2001 From: g1879 Date: Fri, 5 Jan 2024 00:25:20 +0800 Subject: [PATCH 179/182] =?UTF-8?q?=E5=BE=AE=E8=B0=83=5Fget=5Fdocument()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/_pages/chromium_base.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 3b66a76..7581c89 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -151,29 +151,31 @@ class ChromiumBase(BasePage): """ if self._is_reading: return - timeout = timeout if timeout >= .5 else .5 self._is_reading = True + timeout = timeout if timeout >= .5 else .5 end_time = perf_counter() + timeout while perf_counter() < end_time: try: b_id = self.run_cdp('DOM.getDocument', _timeout=timeout)['root']['backendNodeId'] timeout = end_time - perf_counter() - timeout = .5 if timeout < 0 else timeout - self._root_id = self.run_cdp('DOM.resolveNode', - backendNodeId=b_id, _timeout=timeout)['object']['objectId'] - r = self.run_cdp('Page.getFrameTree') - for i in findall(r"'id': '(.*?)'", str(r)): - self.browser._frames[i] = self.tab_id + timeout = .5 if timeout <= 0 else timeout + self._root_id = self.run_cdp('DOM.resolveNode', backendNodeId=b_id, + _timeout=timeout)['object']['objectId'] result = True break except: timeout = end_time - perf_counter() - timeout = .5 if timeout < 0 else timeout + timeout = .5 if timeout <= 0 else timeout else: result = False + if result: + r = self.run_cdp('Page.getFrameTree') + for i in findall(r"'id': '(.*?)'", str(r)): + self.browser._frames[i] = self.tab_id + self._is_loading = False self._is_reading = False return result From bff8d6ba73793e6fae96dff5acc25e6eabe8aa57 Mon Sep 17 00:00:00 2001 From: g1879 Date: Sun, 7 Jan 2024 21:27:33 +0800 Subject: [PATCH 180/182] =?UTF-8?q?4.0.0b35(+)=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E6=9F=A5=E6=89=BE=E6=B5=8F=E8=A7=88=E5=99=A8=E6=96=B9=E6=B3=95?= =?UTF-8?q?=EF=BC=9B=20=E7=9B=91=E5=90=AC=E5=99=A8=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E8=B5=84=E6=BA=90=E7=B1=BB=E5=9E=8B=E7=AD=9B=E9=80=89=EF=BC=9B?= =?UTF-8?q?=20=E7=9B=91=E5=90=AC=E5=99=A8=E5=A2=9E=E5=8A=A0fail=5Finfo?= =?UTF-8?q?=E5=92=8Cis=5Ffailed=E5=B1=9E=E6=80=A7=EF=BC=9B=20=E8=B0=83?= =?UTF-8?q?=E6=95=B4set=5Ftargets()=E5=92=8Cstart()=E5=8F=82=E6=95=B0?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=80=BC=EF=BC=9B=20blocked=5Furls()?= =?UTF-8?q?=E5=8F=AF=E6=8E=A5=E6=94=B6str?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 2 +- DrissionPage/_elements/chromium_element.py | 3 +- DrissionPage/_functions/browser.py | 105 +++++++++------------ DrissionPage/_functions/browser.pyi | 6 +- DrissionPage/_functions/tools.py | 68 ------------- DrissionPage/_functions/tools.pyi | 9 -- DrissionPage/_pages/chromium_base.py | 4 +- DrissionPage/_units/downloader.py | 2 +- DrissionPage/_units/listener.py | 102 ++++++++++++++------ DrissionPage/_units/listener.pyi | 53 ++++++++--- DrissionPage/_units/setter.py | 8 +- DrissionPage/_units/setter.pyi | 2 +- requirements.txt | 2 +- setup.py | 4 +- 14 files changed, 174 insertions(+), 196 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 7ab5984..5220f8d 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -13,4 +13,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b34' +__version__ = '4.0.0b35' diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 1c94c37..8045070 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -8,13 +8,14 @@ from pathlib import Path from re import search from time import perf_counter, sleep +from DataRecorder.tools import get_usable_path + from .none_element import NoneElement from .session_element import make_session_ele from .._base.base import DrissionElement, BaseElement from .._functions.keys import input_text_or_keys from .._functions.locator import get_loc from .._functions.settings import Settings -from .._functions.tools import get_usable_path from .._functions.web import make_absolute_link, get_ele_txt, format_html, is_js_func, offset_scroll from .._units.clicker import Clicker from .._units.rect import ElementRect diff --git a/DrissionPage/_functions/browser.py b/DrissionPage/_functions/browser.py index 99e1211..dfa3b3e 100644 --- a/DrissionPage/_functions/browser.py +++ b/DrissionPage/_functions/browser.py @@ -46,10 +46,10 @@ def connect_browser(option): # 传入的路径找不到，主动在ini文件、注册表、系统变量中找 except FileNotFoundError: - chrome_path = get_chrome_path(show_msg=False) + chrome_path = get_chrome_path() if not chrome_path: - raise FileNotFoundError('无法找到chrome路径，请手动配置。') + raise FileNotFoundError('无法找到浏览器可执行文件路径，请手动配置。') _run_browser(port, chrome_path, args) @@ -281,34 +281,26 @@ def _remove_arg_from_dict(target_dict: dict, arg: str) -> None: pass -def get_chrome_path(ini_path=None, show_msg=True, from_ini=True, - from_regedit=True, from_system_path=True): - """从ini文件或系统变量中获取chrome.exe的路径 - :param ini_path: ini文件路径 - :param show_msg: 是否打印信息 - :param from_ini: 是否从ini文件获取 - :param from_regedit: 是否从注册表获取 - :param from_system_path: 是否从系统路径获取 - :return: chrome.exe路径 - """ +def get_chrome_path(): + """从ini文件或系统变量中获取chrome可执行文件的路径""" # -----------从ini文件中获取-------------- - if ini_path and from_ini: - try: - path = OptionsManager(ini_path).chromium_options['browser_path'] - except KeyError: - path = None - else: - path = None - + path = OptionsManager().chromium_options.get('browser_path', None) if path and Path(path).is_file(): - if show_msg: - print('ini文件中', end='') return str(path) + # -----------使用which获取----------- + from shutil import which + path = (which('chrome') or which('chromium') or which('google-chrome') or which('google-chrome-stable') + or which('google-chrome-unstable') or which('google-chrome-beta')) + if path: + return path + + # -----------从MAC和Linux默认路径获取----------- from platform import system sys = system().lower() if sys in ('macos', 'darwin'): - return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' + p = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' + return p if Path(p).exists() else None elif sys == 'linux': paths = ('/usr/bin/google-chrome', '/opt/google/chrome/google-chrome', @@ -322,48 +314,39 @@ def get_chrome_path(ini_path=None, show_msg=True, from_ini=True, return None # -----------从注册表中获取-------------- - if from_regedit: - import winreg - try: - key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, - r'SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\chrome.exe', - reserved=0, access=winreg.KEY_READ) - k = winreg.EnumValue(key, 0) - winreg.CloseKey(key) + import winreg + try: + key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, + r'SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\chrome.exe', + reserved=0, access=winreg.KEY_READ) + k = winreg.EnumValue(key, 0) + winreg.CloseKey(key) - if show_msg: - print('注册表中', end='') + return k[1] - return k[1] - - except FileNotFoundError: - pass + except FileNotFoundError: + pass # -----------从系统变量中获取-------------- - if from_system_path: + try: + paths = popen('set path').read().lower() + except: + return None + r = search(r'[^;]*chrome[^;]*', paths) + + if r: + path = Path(r.group(0)) if 'chrome.exe' in r.group(0) else Path(r.group(0)) / 'chrome.exe' + + if path.exists(): + return str(path) + + paths = paths.split(';') + + for path in paths: + path = Path(path) / 'chrome.exe' + try: - paths = popen('set path').read().lower() - except: - return None - r = search(r'[^;]*chrome[^;]*', paths) - - if r: - path = Path(r.group(0)) if 'chrome.exe' in r.group(0) else Path(r.group(0)) / 'chrome.exe' - if path.exists(): - if show_msg: - print('系统变量中', end='') return str(path) - - paths = paths.split(';') - - for path in paths: - path = Path(path) / 'chrome.exe' - - try: - if path.exists(): - if show_msg: - print('系统变量中', end='') - return str(path) - except OSError: - pass + except OSError: + pass diff --git a/DrissionPage/_functions/browser.pyi b/DrissionPage/_functions/browser.pyi index b32952b..a1e6fa9 100644 --- a/DrissionPage/_functions/browser.pyi +++ b/DrissionPage/_functions/browser.pyi @@ -23,8 +23,4 @@ def set_flags(opt: ChromiumOptions) -> None: ... def test_connect(ip: str, port: Union[int, str], timeout: float = 30) -> None: ... -def get_chrome_path(ini_path: str = None, - show_msg: bool = True, - from_ini: bool = True, - from_regedit: bool = True, - from_system_path: bool = True, ) -> Union[str, None]: ... +def get_chrome_path() -> Union[str, None]: ... diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index dcdae15..723327a 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -5,7 +5,6 @@ """ from pathlib import Path from platform import system -from re import search, sub from shutil import rmtree from time import perf_counter, sleep @@ -16,73 +15,6 @@ from ..errors import (ContextLostError, ElementLostError, CDPError, PageDisconne AlertExistsError, WrongURLError, StorageError, CookieFormatError, JavaScriptError) -def get_usable_path(path, is_file=True, parents=True): - """检查文件或文件夹是否有重名，并返回可以使用的路径 - :param path: 文件或文件夹路径 - :param is_file: 目标是文件还是文件夹 - :param parents: 是否创建目标路径 - :return: 可用的路径，Path对象 - """ - path = Path(path) - parent = path.parent - if parents: - parent.mkdir(parents=True, exist_ok=True) - path = parent / make_valid_name(path.name) - name = path.stem if path.is_file() else path.name - ext = path.suffix if path.is_file() else '' - - first_time = True - - while path.exists() and path.is_file() == is_file: - r = search(r'(.*)_(\d+)$', name) - - if not r or (r and first_time): - src_name, num = name, '1' - else: - src_name, num = r.group(1), int(r.group(2)) + 1 - - name = f'{src_name}_{num}' - path = parent / f'{name}{ext}' - first_time = None - - return path - - -def make_valid_name(full_name): - """获取有效的文件名 - :param full_name: 文件名 - :return: 可用的文件名 - """ - # ----------------去除前后空格---------------- - full_name = full_name.strip() - - # ----------------使总长度不大于255个字符（一个汉字是2个字符）---------------- - r = search(r'(.*)(\.[^.]+$)', full_name) # 拆分文件名和后缀名 - if r: - name, ext = r.group(1), r.group(2) - ext_long = len(ext) - else: - name, ext = full_name, '' - ext_long = 0 - - while get_long(name) > 255 - ext_long: - name = name[:-1] - - full_name = f'{name}{ext}' - - # ----------------去除不允许存在的字符---------------- - return sub(r'[<>/\\|:*?\n]', '', full_name) - - -def get_long(txt): - """返回字符串中字符个数（一个汉字是2个字符） - :param txt: 字符串 - :return: 字符个数 - """ - txt_len = len(txt) - return int((len(txt.encode('utf-8')) - txt_len) / 2 + txt_len) - - def port_is_using(ip, port): """检查端口是否被占用 :param ip: 浏览器地址 diff --git a/DrissionPage/_functions/tools.pyi b/DrissionPage/_functions/tools.pyi index b516919..2e3aa73 100644 --- a/DrissionPage/_functions/tools.pyi +++ b/DrissionPage/_functions/tools.pyi @@ -11,15 +11,6 @@ from types import FunctionType from .._pages.chromium_page import ChromiumPage -def get_usable_path(path: Union[str, Path], is_file: bool = True, parents: bool = True) -> Path: ... - - -def make_valid_name(full_name: str) -> str: ... - - -def get_long(txt) -> int: ... - - def port_is_using(ip: str, port: Union[str, int]) -> bool: ... diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 7581c89..7088998 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -11,13 +11,15 @@ from threading import Thread from time import perf_counter, sleep from urllib.parse import quote +from DataRecorder.tools import make_valid_name + from .._base.base import BasePage from .._elements.chromium_element import ChromiumElement, run_js, make_chromium_eles from .._elements.none_element import NoneElement from .._elements.session_element import make_session_ele from .._functions.locator import get_loc, is_loc from .._functions.settings import Settings -from .._functions.tools import raise_error, make_valid_name +from .._functions.tools import raise_error from .._functions.web import location_in_viewport from .._units.actions import Actions from .._units.listener import Listener diff --git a/DrissionPage/_units/downloader.py b/DrissionPage/_units/downloader.py index f3bb017..9d84e8b 100644 --- a/DrissionPage/_units/downloader.py +++ b/DrissionPage/_units/downloader.py @@ -8,7 +8,7 @@ from pathlib import Path from shutil import move from time import sleep, perf_counter -from .._functions.tools import get_usable_path +from DataRecorder.tools import get_usable_path class DownloadManager(object): diff --git a/DrissionPage/_units/listener.py b/DrissionPage/_units/listener.py index c8a8ffb..343a066 100644 --- a/DrissionPage/_units/listener.py +++ b/DrissionPage/_units/listener.py @@ -29,56 +29,76 @@ class Listener(object): self._driver = None self._running_requests = 0 - self._caught = None # 临存捕捉到的数据 - self._request_ids = None # 暂存须要拦截的请求id + self._caught = None + self._request_ids = None self._extra_info_ids = None self.listening = False - self._targets = None # 默认监听所有 - self.tab_id = None # 当前tab的id + self.tab_id = None + self._targets = True self._is_regex = False - self._method = None + self._method = ('GET', 'POST') + self._res_type = True @property def targets(self): """返回监听目标""" return self._targets - def set_targets(self, targets=True, is_regex=False, method=('GET', 'POST')): + def set_targets(self, targets=True, is_regex=False, method=('GET', 'POST'), res_type=True): """指定要等待的数据包 :param targets: 要匹配的数据包url特征，可用list等传入多个，为True时获取所有 :param is_regex: 设置的target是否正则表达式 - :param method: 设置监听的请求类型，可指定多个，为None时监听全部 + :param method: 设置监听的请求类型，可指定多个，为True时监听全部 + :param res_type: 设置监听的资源类型，可指定多个，为True时监听全部，可指定的值有： + Document, Stylesheet, Image, Media, Font, Script, TextTrack, XHR, Fetch, Prefetch, EventSource, WebSocket, + Manifest, SignedExchange, Ping, CSPViolationReport, Preflight, Other :return: None """ if targets is not None: if not isinstance(targets, (str, list, tuple, set)) and targets is not True: raise TypeError('targets只能是str、list、tuple、set、True。') if targets is True: - targets = '' + self._targets = True + else: + self._targets = {targets} if isinstance(targets, str) else set(targets) - self._targets = {targets} if isinstance(targets, str) else set(targets) - - self._is_regex = is_regex + if is_regex is not None: + self._is_regex = is_regex if method is not None: if isinstance(method, str): self._method = {method.upper()} elif isinstance(method, (list, tuple, set)): self._method = set(i.upper() for i in method) + elif method is True: + self._method = True else: - raise TypeError('method参数只能是str、list、tuple、set类型。') + raise TypeError('method参数只能是str、list、tuple、set、True类型。') - def start(self, targets=None, is_regex=False, method=('GET', 'POST')): + if res_type is not None: + if isinstance(res_type, str): + self._res_type = {res_type.upper()} + elif isinstance(res_type, (list, tuple, set)): + self._res_type = set(i.upper() for i in res_type) + elif res_type is True: + self._res_type = True + else: + raise TypeError('res_type参数只能是str、list、tuple、set、True类型。') + + def start(self, targets=None, is_regex=None, method=None, res_type=None): """拦截目标请求，每次拦截前清空结果 :param targets: 要匹配的数据包url特征，可用list等传入多个，为True时获取所有 - :param is_regex: 设置的target是否正则表达式 - :param method: 设置监听的请求类型，可指定多个，为None时监听全部 + :param is_regex: 设置的target是否正则表达式，为None时保持原来设置 + :param method: 设置监听的请求类型，可指定多个，默认('GET', 'POST')，为True时监听全部，为None时保持原来设置 + :param res_type: 设置监听的资源类型，可指定多个，默认为True时监听全部，为None时保持原来设置，可指定的值有： + Document, Stylesheet, Image, Media, Font, Script, TextTrack, XHR, Fetch, Prefetch, EventSource, WebSocket, + Manifest, SignedExchange, Ping, CSPViolationReport, Preflight, Other :return: None """ - if targets or method: - self.set_targets(targets, is_regex, method) + if targets or is_regex is not None or method or res_type: + self.set_targets(targets, is_regex, method, res_type) self.clear() if self.listening: @@ -240,10 +260,11 @@ class Listener(object): """接收到请求时的回调函数""" self._running_requests += 1 p = None - if not self._targets: - if not self._method or kwargs['request']['method'] in self._method: + if self._targets is True: + if ((self._method is True or kwargs['request']['method'] in self._method) + and (self._res_type is True or kwargs.get('type', '').upper() in self._res_type)): rid = kwargs['requestId'] - p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, None)) + p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, True)) p._raw_request = kwargs if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None): p._raw_post_data = self._driver.run('Network.getRequestPostData', @@ -252,9 +273,10 @@ class Listener(object): else: rid = kwargs['requestId'] for target in self._targets: - if ((self._is_regex and search(target, kwargs['request']['url'])) or - (not self._is_regex and target in kwargs['request']['url'])) and ( - not self._method or kwargs['request']['method'] in self._method): + if (((self._is_regex and search(target, kwargs['request']['url'])) + or (not self._is_regex and target in kwargs['request']['url'])) + and (self._method is True or kwargs['request']['method'] in self._method) + and (self._res_type is True or kwargs.get('type', '').upper() in self._res_type)): p = self._request_ids.setdefault(rid, DataPacket(self._page.tab_id, target)) p._raw_request = kwargs break @@ -329,8 +351,9 @@ class Listener(object): r_id = kwargs['requestId'] dp = self._request_ids.get(r_id, None) if dp: - dp.errorText = kwargs['errorText'] + dp._raw_fail_info = kwargs dp._resource_type = kwargs['type'] + dp.is_failed = True r = self._extra_info_ids.get(kwargs['requestId'], None) if r: @@ -374,23 +397,25 @@ class DataPacket(object): """ self.tab_id = tab_id self.target = target + self.is_failed = False self._raw_request = None self._raw_post_data = None - self._raw_response = None self._raw_body = None - self._base64_body = False - self._requestExtraInfo = None - self._responseExtraInfo = None + self._raw_fail_info = None self._request = None self._response = None - self.errorText = None + self._fail_info = None + + self._base64_body = False + self._requestExtraInfo = None + self._responseExtraInfo = None self._resource_type = None def __repr__(self): - t = f'"{self.target}"' if self.target is not None else None + t = f'"{self.target}"' if self.target is not True else True return f'' @property @@ -429,6 +454,12 @@ class DataPacket(object): self._response = Response(self, self._raw_response, self._raw_body, self._base64_body) return self._response + @property + def fail_info(self): + if self._fail_info is None: + self._fail_info = FailInfo(self, self._raw_fail_info) + return self._fail_info + def wait_extra_info(self, timeout=None): """等待额外的信息加载完成 :param timeout: 超时时间，None为无限等待 @@ -498,7 +529,7 @@ class Response(object): self._headers = None def __getattr__(self, item): - return self._response.get(item, None) + return self._response.get(item, None) if self._response else None @property def headers(self): @@ -551,3 +582,12 @@ class RequestExtraInfo(ExtraInfo): class ResponseExtraInfo(ExtraInfo): pass + + +class FailInfo(object): + def __init__(self, data_packet, fail_info): + self._data_packet = data_packet + self._fail_info = fail_info + + def __getattr__(self, item): + return self._fail_info.get(item, None) if self._fail_info else None diff --git a/DrissionPage/_units/listener.pyi b/DrissionPage/_units/listener.pyi index dccb3ec..b2511df 100644 --- a/DrissionPage/_units/listener.pyi +++ b/DrissionPage/_units/listener.pyi @@ -4,7 +4,7 @@ @Contact : g1879@qq.com """ from queue import Queue -from typing import Union, Dict, List, Iterable, Tuple, Optional +from typing import Union, Dict, List, Iterable, Optional, Literal from requests.structures import CaseInsensitiveDict @@ -12,6 +12,9 @@ from .._base.driver import Driver from .._pages.chromium_base import ChromiumBase from .._pages.chromium_frame import ChromiumFrame +__RES_TYPE__ = Literal['Document', 'Stylesheet', 'Image', 'Media', 'Font', 'Script', 'TextTrack', 'XHR', 'Fetch', +'Prefetch', 'EventSource', 'WebSocket', 'Manifest', 'SignedExchange', 'Ping', 'CSPViolationReport', 'Preflight', 'Other'] + class Listener(object): def __init__(self, page: ChromiumBase): @@ -20,6 +23,7 @@ class Listener(object): self._target_id: str = ... self._targets: Union[str, dict] = ... self._method: set = ... + self._res_type: set = ... self._caught: Queue = ... self._is_regex: bool = ... self._driver: Driver = ... @@ -31,8 +35,17 @@ class Listener(object): @property def targets(self) -> Optional[set]: ... - def set_targets(self, targets: Union[str, list, tuple, set, None] = None, is_regex: bool = False, - method: Union[str, list, tuple, set] = None) -> None: ... + def set_targets(self, + targets: Optional[str, list, tuple, set, bool] = True, + is_regex: Optional[bool] = False, + method: Optional[str, list, tuple, set, bool] = ('GET', 'POST'), + res_type: Optional[__RES_TYPE__, list, tuple, set, bool] = True) -> None: ... + + def start(self, + targets: Optional[str, list, tuple, set, bool] = None, + is_regex: Optional[bool] = None, + method: Optional[str, list, tuple, set, bool] = None, + res_type: Optional[__RES_TYPE__, list, tuple, set, bool] = None) -> None: ... def stop(self) -> None: ... @@ -40,7 +53,10 @@ class Listener(object): def resume(self) -> None: ... - def wait(self, count: int = 1, timeout: float = None, fit_count: bool = True, + def wait(self, + count: int = 1, + timeout: float = None, + fit_count: bool = True, raise_err: bool = None) -> Union[List[DataPacket], DataPacket, None]: ... @property @@ -52,10 +68,6 @@ class Listener(object): def _to_target(self, target_id: str, address: str, page: ChromiumBase) -> None: ... - def start(self, targets: Union[str, List[str], Tuple, bool, None] = None, is_regex: bool = False, - method: Union[str, list, tuple, set] = None) \ - -> Union[DataPacket, Dict[str, List[DataPacket]], False]: ... - def _requestWillBeSent(self, **kwargs) -> None: ... def _requestWillBeSentExtraInfo(self, **kwargs) -> None: ... @@ -68,7 +80,9 @@ class Listener(object): def _loading_failed(self, **kwargs) -> None: ... - def steps(self, count: int = None, timeout: float = None, + def steps(self, + count: int = None, + timeout: float = None, gap=1) -> Iterable[Union[DataPacket, List[DataPacket]]]: ... def _set_callback(self) -> None: ... @@ -83,17 +97,19 @@ class FrameListener(Listener): class DataPacket(object): """返回的数据包管理类""" - def __init__(self, tab_id: str, target: Optional[str]): + def __init__(self, tab_id: str, target: [str, bool]): self.tab_id: str = ... self.target: str = ... + self.is_failed: bool = ... self._raw_request: Optional[dict] = ... self._raw_response: Optional[dict] = ... self._raw_post_data: str = ... self._raw_body: str = ... + self._raw_fail_info: Optional[dict] = ... self._base64_body: bool = ... self._request: Request = ... self._response: Response = ... - self.errorText: str = ... + self._fail_info: Optional[FailInfo] = ... self._resource_type: str = ... self._requestExtraInfo: Optional[dict] = ... self._responseExtraInfo: Optional[dict] = ... @@ -122,6 +138,9 @@ class DataPacket(object): @property def response(self) -> Response: ... + @property + def fail_info(self) -> Optional[FailInfo]: ... + def wait_extra_info(self, timeout: float = None) -> bool: ... @@ -228,3 +247,15 @@ class ResponseExtraInfo(ExtraInfo): headersText: str = ... cookiePartitionKey: str = ... cookiePartitionKeyOpaque: bool = ... + + +class FailInfo(object): + _data_packet: DataPacket + _fail_info: dict + _fail_info: float + errorText: str + canceled: bool + blockedReason: Optional[str] + corsErrorStatus: Optional[str] + + def __init__(self, data_packet: DataPacket, fail_info: dict): ... diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index 8e9ad19..f6d0f87 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -146,14 +146,16 @@ class ChromiumBaseSetter(BasePageSetter): self._page._alert.auto = accept if on_off else None def blocked_urls(self, urls): - """设置要忽略的url，传入None时清空已设置的内容。 - :param urls: + """设置要忽略的url + :param urls: 要忽略的url，可用*通配符，可输入多个，传入None时清空已设置的内容 :return: None """ if not urls: urls = [] + elif isinstance(urls, str): + urls = (urls,) if not isinstance(urls, (list, tuple)): - raise TypeError('urls需传入list或tuple类型。') + raise TypeError('urls需传入str、list或tuple类型。') self._page.run_cdp('Network.enable') self._page.run_cdp('Network.setBlockedURLs', urls=urls) diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index f8e5de0..c68d848 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -62,7 +62,7 @@ class ChromiumBaseSetter(BasePageSetter): def upload_files(self, files: Union[str, list, tuple]) -> None: ... - def blocked_urls(self, urls: Optional[list, tuple]) -> None: ... + def blocked_urls(self, urls: Optional[list, tuple, str]) -> None: ... class TabSetter(ChromiumBaseSetter): diff --git a/requirements.txt b/requirements.txt index e02789b..8d24602 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ requests lxml cssselect -DownloadKit>=2.0.0b3 +DownloadKit>=2.0.0b5 websocket-client>=1.7.0 click tldextract diff --git a/setup.py b/setup.py index e304e1f..8f5dd8f 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b34", + version="4.0.0b35", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", @@ -22,7 +22,7 @@ setup( 'lxml', 'requests', 'cssselect', - 'DownloadKit>=2.0.0b3', + 'DownloadKit>=2.0.0b5', 'websocket-client>=1.7.0', 'click', 'tldextract', From c3b58bc90d9b1639e6b54a367e9a0ffb7d27a586 Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 8 Jan 2024 18:04:59 +0800 Subject: [PATCH 181/182] =?UTF-8?q?4.0.0get=5Fframe()=E5=BA=8F=E5=8F=B7?= =?UTF-8?q?=E6=94=B9=E6=88=90=E4=BB=8E0=E5=BC=80=E5=A7=8B=EF=BC=9B?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2save()=E5=A2=9E=E5=8A=A0as=5Fpdf=E5=8F=82?= =?UTF-8?q?=E6=95=B0=EF=BC=9B=E4=BF=AE=E5=A4=8D=E4=B8=80=E4=BA=9B=E5=B0=8F?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DrissionPage/__init__.py | 9 +++-- DrissionPage/_base/base.py | 6 ++- DrissionPage/_base/base.pyi | 6 ++- DrissionPage/_base/browser.py | 17 +++++--- DrissionPage/_base/browser.pyi | 6 ++- DrissionPage/_base/driver.py | 6 ++- DrissionPage/_base/driver.pyi | 6 ++- DrissionPage/_configs/chromium_options.py | 6 ++- DrissionPage/_configs/chromium_options.pyi | 6 ++- DrissionPage/_configs/options_manage.py | 6 ++- DrissionPage/_configs/options_manage.pyi | 6 ++- DrissionPage/_configs/session_options.py | 6 ++- DrissionPage/_configs/session_options.pyi | 6 ++- DrissionPage/_elements/chromium_element.py | 6 ++- DrissionPage/_elements/chromium_element.pyi | 6 ++- DrissionPage/_elements/none_element.py | 6 ++- DrissionPage/_elements/session_element.py | 6 ++- DrissionPage/_elements/session_element.pyi | 6 ++- DrissionPage/_functions/browser.py | 6 ++- DrissionPage/_functions/browser.pyi | 6 ++- DrissionPage/_functions/by.py | 8 ++++ DrissionPage/_functions/cli.py | 6 ++- DrissionPage/_functions/keys.py | 6 ++- DrissionPage/_functions/keys.pyi | 6 ++- DrissionPage/_functions/locator.py | 6 ++- DrissionPage/_functions/locator.pyi | 6 ++- DrissionPage/_functions/settings.py | 6 ++- DrissionPage/_functions/tools.py | 6 ++- DrissionPage/_functions/tools.pyi | 6 ++- DrissionPage/_functions/web.py | 6 ++- DrissionPage/_functions/web.pyi | 6 ++- DrissionPage/_pages/chromium_base.py | 45 +++++++++++++++++---- DrissionPage/_pages/chromium_base.pyi | 15 +++++-- DrissionPage/_pages/chromium_frame.py | 6 ++- DrissionPage/_pages/chromium_frame.pyi | 6 ++- DrissionPage/_pages/chromium_page.py | 18 +++++---- DrissionPage/_pages/chromium_page.pyi | 27 +++++++++++-- DrissionPage/_pages/chromium_tab.py | 6 ++- DrissionPage/_pages/chromium_tab.pyi | 6 ++- DrissionPage/_pages/session_page.py | 6 ++- DrissionPage/_pages/session_page.pyi | 6 ++- DrissionPage/_pages/web_page.py | 6 ++- DrissionPage/_pages/web_page.pyi | 6 ++- DrissionPage/_units/actions.py | 6 ++- DrissionPage/_units/actions.pyi | 6 ++- DrissionPage/_units/clicker.py | 6 ++- DrissionPage/_units/clicker.pyi | 6 ++- DrissionPage/_units/cookies_setter.py | 6 ++- DrissionPage/_units/cookies_setter.pyi | 6 +++ DrissionPage/_units/downloader.py | 6 ++- DrissionPage/_units/downloader.pyi | 6 ++- DrissionPage/_units/listener.py | 6 ++- DrissionPage/_units/listener.pyi | 6 ++- DrissionPage/_units/rect.py | 6 ++- DrissionPage/_units/rect.pyi | 6 ++- DrissionPage/_units/screencast.py | 6 ++- DrissionPage/_units/screencast.pyi | 6 ++- DrissionPage/_units/scroller.py | 6 ++- DrissionPage/_units/scroller.pyi | 6 +++ DrissionPage/_units/selector.py | 6 ++- DrissionPage/_units/selector.pyi | 6 ++- DrissionPage/_units/setter.py | 6 ++- DrissionPage/_units/setter.pyi | 6 ++- DrissionPage/_units/states.py | 6 ++- DrissionPage/_units/states.pyi | 6 ++- DrissionPage/_units/waiter.py | 6 +++ DrissionPage/_units/waiter.pyi | 6 ++- DrissionPage/common.py | 6 ++- DrissionPage/errors.py | 6 ++- README.md | 4 +- requirements.txt | 2 +- setup.py | 4 +- 72 files changed, 369 insertions(+), 152 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 5220f8d..5eb95ff 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -1,9 +1,10 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ -# 常用页面类 from ._pages.chromium_page import ChromiumPage from ._pages.session_page import SessionPage from ._pages.web_page import WebPage @@ -13,4 +14,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0b35' +__version__ = '4.0.0' diff --git a/DrissionPage/_base/base.py b/DrissionPage/_base/base.py index 8f2598d..0f16588 100644 --- a/DrissionPage/_base/base.py +++ b/DrissionPage/_base/base.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from abc import abstractmethod from re import sub diff --git a/DrissionPage/_base/base.pyi b/DrissionPage/_base/base.pyi index 13175e1..b4964b0 100644 --- a/DrissionPage/_base/base.pyi +++ b/DrissionPage/_base/base.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from abc import abstractmethod from typing import Union, Tuple, List, Any diff --git a/DrissionPage/_base/browser.py b/DrissionPage/_base/browser.py index 7d2e61f..0e71e1d 100644 --- a/DrissionPage/_base/browser.py +++ b/DrissionPage/_base/browser.py @@ -1,12 +1,16 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from shutil import rmtree from time import sleep, perf_counter +from websocket import WebSocketBadStatusException + from .driver import BrowserDriver, Driver from .._functions.tools import stop_process_on_port, raise_error from .._units.downloader import DownloadManager @@ -70,8 +74,11 @@ class Browser(object): """标签页创建时执行""" if (kwargs['targetInfo']['type'] in ('page', 'webview') and not kwargs['targetInfo']['url'].startswith('devtools://')): - self._drivers[kwargs['targetInfo']['targetId']] = Driver(kwargs['targetInfo']['targetId'], - 'page', self.address) + try: + self._drivers[kwargs['targetInfo']['targetId']] = Driver(kwargs['targetInfo']['targetId'], + 'page', self.address) + except WebSocketBadStatusException: + pass def _onTargetDestroyed(self, **kwargs): """标签页关闭时执行""" @@ -205,5 +212,5 @@ class Browser(object): try: rmtree(path) break - except (PermissionError, FileNotFoundError): + except (PermissionError, FileNotFoundError, OSError): pass diff --git a/DrissionPage/_base/browser.pyi b/DrissionPage/_base/browser.pyi index b5d0161..e29afcb 100644 --- a/DrissionPage/_base/browser.pyi +++ b/DrissionPage/_base/browser.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from typing import List, Optional, Union diff --git a/DrissionPage/_base/driver.py b/DrissionPage/_base/driver.py index 809c747..d5ce858 100644 --- a/DrissionPage/_base/driver.py +++ b/DrissionPage/_base/driver.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from json import dumps, loads, JSONDecodeError from queue import Queue, Empty diff --git a/DrissionPage/_base/driver.pyi b/DrissionPage/_base/driver.pyi index cdabcb2..ae86582 100644 --- a/DrissionPage/_base/driver.pyi +++ b/DrissionPage/_base/driver.pyi @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from queue import Queue from threading import Thread, Event diff --git a/DrissionPage/_configs/chromium_options.py b/DrissionPage/_configs/chromium_options.py index 342008e..11b56f2 100644 --- a/DrissionPage/_configs/chromium_options.py +++ b/DrissionPage/_configs/chromium_options.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from re import search diff --git a/DrissionPage/_configs/chromium_options.pyi b/DrissionPage/_configs/chromium_options.pyi index 8fd4ab9..ce2a748 100644 --- a/DrissionPage/_configs/chromium_options.pyi +++ b/DrissionPage/_configs/chromium_options.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from threading import Lock diff --git a/DrissionPage/_configs/options_manage.py b/DrissionPage/_configs/options_manage.py index a524c08..703adbd 100644 --- a/DrissionPage/_configs/options_manage.py +++ b/DrissionPage/_configs/options_manage.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from configparser import RawConfigParser, NoSectionError, NoOptionError from pathlib import Path diff --git a/DrissionPage/_configs/options_manage.pyi b/DrissionPage/_configs/options_manage.pyi index bd431f5..805ae92 100644 --- a/DrissionPage/_configs/options_manage.pyi +++ b/DrissionPage/_configs/options_manage.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from configparser import RawConfigParser from typing import Any diff --git a/DrissionPage/_configs/session_options.py b/DrissionPage/_configs/session_options.py index f68d6b0..532098a 100644 --- a/DrissionPage/_configs/session_options.py +++ b/DrissionPage/_configs/session_options.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path diff --git a/DrissionPage/_configs/session_options.pyi b/DrissionPage/_configs/session_options.pyi index cbd2931..cc1b4e9 100644 --- a/DrissionPage/_configs/session_options.pyi +++ b/DrissionPage/_configs/session_options.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from typing import Any, Union, Tuple, Optional diff --git a/DrissionPage/_elements/chromium_element.py b/DrissionPage/_elements/chromium_element.py index 8045070..9dff0a2 100644 --- a/DrissionPage/_elements/chromium_element.py +++ b/DrissionPage/_elements/chromium_element.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from os.path import basename, sep from pathlib import Path diff --git a/DrissionPage/_elements/chromium_element.pyi b/DrissionPage/_elements/chromium_element.pyi index 9eab619..fa341ca 100644 --- a/DrissionPage/_elements/chromium_element.pyi +++ b/DrissionPage/_elements/chromium_element.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from typing import Union, Tuple, List, Any, Literal diff --git a/DrissionPage/_elements/none_element.py b/DrissionPage/_elements/none_element.py index 25732e5..fac5415 100644 --- a/DrissionPage/_elements/none_element.py +++ b/DrissionPage/_elements/none_element.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from ..errors import ElementNotFoundError diff --git a/DrissionPage/_elements/session_element.py b/DrissionPage/_elements/session_element.py index 82820a9..c829ed6 100644 --- a/DrissionPage/_elements/session_element.py +++ b/DrissionPage/_elements/session_element.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from html import unescape from re import match, sub, DOTALL diff --git a/DrissionPage/_elements/session_element.pyi b/DrissionPage/_elements/session_element.pyi index 32452c6..5c82e6f 100644 --- a/DrissionPage/_elements/session_element.pyi +++ b/DrissionPage/_elements/session_element.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from typing import Union, List, Tuple, Optional diff --git a/DrissionPage/_functions/browser.py b/DrissionPage/_functions/browser.py index dfa3b3e..113a7f9 100644 --- a/DrissionPage/_functions/browser.py +++ b/DrissionPage/_functions/browser.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from json import load, dump, JSONDecodeError from os import popen diff --git a/DrissionPage/_functions/browser.pyi b/DrissionPage/_functions/browser.pyi index a1e6fa9..6285a34 100644 --- a/DrissionPage/_functions/browser.pyi +++ b/DrissionPage/_functions/browser.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from typing import Union diff --git a/DrissionPage/_functions/by.py b/DrissionPage/_functions/by.py index 899a183..e494189 100644 --- a/DrissionPage/_functions/by.py +++ b/DrissionPage/_functions/by.py @@ -1,4 +1,12 @@ # -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. +""" + + class By: ID = 'id' XPATH = 'xpath' diff --git a/DrissionPage/_functions/cli.py b/DrissionPage/_functions/cli.py index a06b228..0141c60 100644 --- a/DrissionPage/_functions/cli.py +++ b/DrissionPage/_functions/cli.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from click import command, option diff --git a/DrissionPage/_functions/keys.py b/DrissionPage/_functions/keys.py index 8c0a243..e4bebea 100644 --- a/DrissionPage/_functions/keys.py +++ b/DrissionPage/_functions/keys.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from ..errors import AlertExistsError diff --git a/DrissionPage/_functions/keys.pyi b/DrissionPage/_functions/keys.pyi index de2ca51..896bc42 100644 --- a/DrissionPage/_functions/keys.pyi +++ b/DrissionPage/_functions/keys.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from typing import Tuple, Dict, Union, Any diff --git a/DrissionPage/_functions/locator.py b/DrissionPage/_functions/locator.py index 13ee7d1..ac65759 100644 --- a/DrissionPage/_functions/locator.py +++ b/DrissionPage/_functions/locator.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from re import split from .by import By diff --git a/DrissionPage/_functions/locator.pyi b/DrissionPage/_functions/locator.pyi index b890c03..2f79a69 100644 --- a/DrissionPage/_functions/locator.pyi +++ b/DrissionPage/_functions/locator.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from typing import Union diff --git a/DrissionPage/_functions/settings.py b/DrissionPage/_functions/settings.py index 0107271..225190c 100644 --- a/DrissionPage/_functions/settings.py +++ b/DrissionPage/_functions/settings.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ diff --git a/DrissionPage/_functions/tools.py b/DrissionPage/_functions/tools.py index 723327a..b785ed1 100644 --- a/DrissionPage/_functions/tools.py +++ b/DrissionPage/_functions/tools.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from platform import system diff --git a/DrissionPage/_functions/tools.pyi b/DrissionPage/_functions/tools.pyi index 2e3aa73..057a43f 100644 --- a/DrissionPage/_functions/tools.pyi +++ b/DrissionPage/_functions/tools.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from os import popen from pathlib import Path diff --git a/DrissionPage/_functions/web.py b/DrissionPage/_functions/web.py index 4e4cee7..9479c05 100644 --- a/DrissionPage/_functions/web.py +++ b/DrissionPage/_functions/web.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from datetime import datetime from html import unescape diff --git a/DrissionPage/_functions/web.pyi b/DrissionPage/_functions/web.pyi index 128f48f..240bdab 100644 --- a/DrissionPage/_functions/web.pyi +++ b/DrissionPage/_functions/web.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from http.cookiejar import Cookie from typing import Union diff --git a/DrissionPage/_pages/chromium_base.py b/DrissionPage/_pages/chromium_base.py index 7088998..dbaf4f8 100644 --- a/DrissionPage/_pages/chromium_base.py +++ b/DrissionPage/_pages/chromium_base.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from json import loads, JSONDecodeError from os.path import sep @@ -651,7 +653,7 @@ class ChromiumBase(BasePage): self.run_cdp('DOM.removeNode', nodeId=ele._node_id) def get_frame(self, loc_ind_ele, timeout=None): - """获取页面中一个frame对象，可传入定位符、iframe序号、ChromiumFrame对象，序号从1开始 + """获取页面中一个frame对象，可传入定位符、iframe序号、ChromiumFrame对象，序号从0开始 :param loc_ind_ele: 定位符、iframe序号、ChromiumFrame对象 :param timeout: 查找元素超时时间（秒） :return: ChromiumFrame对象 @@ -674,9 +676,9 @@ class ChromiumBase(BasePage): r = ele elif isinstance(loc_ind_ele, int): - if loc_ind_ele < 1: - raise ValueError('序号必须大于0。') - xpath = f'xpath:(//*[name()="frame" or name()="iframe"])[{loc_ind_ele}]' + if loc_ind_ele < 0: + raise ValueError('序号必须大于等于0。') + xpath = f'xpath:(//*[name()="frame" or name()="iframe"])[{loc_ind_ele + 1}]' r = self._ele(xpath, timeout=timeout) elif str(type(loc_ind_ele)).endswith(".ChromiumFrame'>"): @@ -1135,7 +1137,7 @@ def get_mhtml(page, path=None, name=None): """把当前页面保存为mhtml文件，如果path和name参数都为None，只返回mhtml文本 :param page: 要保存的页面对象 :param path: 保存路径，为None且name不为None时保存在当前路径 - :param name: 文件名，为None且path不为None时用title属性值 + :param name: 文件名，为None且path不为None时用title属性值 :return: mhtml文本 """ r = page.run_cdp('Page.captureSnapshot')['data'] @@ -1147,3 +1149,32 @@ def get_mhtml(page, path=None, name=None): with open(f'{path}{sep}{name}.mhtml', 'w', encoding='utf-8') as f: f.write(r) return r + + +def get_pdf(page, path=None, name=None, kwargs=None): + """把当前页面保存为pdf文件，如果path和name参数都为None，只返回字节 + :param page: 要保存的页面对象 + :param path: 保存路径，为None且name不为None时保存在当前路径 + :param name: 文件名，为None且path不为None时用title属性值 + :param kwargs: pdf生成参数 + :return: pdf文本 + """ + if not kwargs: + kwargs = {} + kwargs['transferMode'] = 'ReturnAsBase64' + if 'printBackground' not in kwargs: + kwargs['printBackground'] = True + try: + r = page.run_cdp('Page.printToPDF', **kwargs)['data'] + except: + raise RuntimeError('保存失败，可能浏览器版本不支持。') + from base64 import b64decode + r = b64decode(r) + if path is None and name is None: + return r + path = path or '.' + Path(path).mkdir(parents=True, exist_ok=True) + name = make_valid_name(name or page.title) + with open(f'{path}{sep}{name}.pdf', 'wb') as f: + f.write(r) + return r diff --git a/DrissionPage/_pages/chromium_base.pyi b/DrissionPage/_pages/chromium_base.pyi index a24ddd8..d4993b1 100644 --- a/DrissionPage/_pages/chromium_base.pyi +++ b/DrissionPage/_pages/chromium_base.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from typing import Union, Tuple, List, Any, Optional, Literal @@ -270,4 +272,11 @@ class Alert(object): self.auto: Optional[bool] = ... -def get_mhtml(page: Union[ChromiumPage, ChromiumTab], path: Union[str, Path] = None, name: str = None) -> str: ... +def get_mhtml(page: Union[ChromiumPage, ChromiumTab], + path: Union[str, Path] = None, + name: str = None) -> str: ... + + +def get_pdf(page: Union[ChromiumPage, ChromiumTab], + path: Union[str, Path] = None, + name: str = None, kwargs: dict=None) -> bytes: ... diff --git a/DrissionPage/_pages/chromium_frame.py b/DrissionPage/_pages/chromium_frame.py index 290b9a5..a7aac92 100644 --- a/DrissionPage/_pages/chromium_frame.py +++ b/DrissionPage/_pages/chromium_frame.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from copy import copy from re import search, findall, DOTALL diff --git a/DrissionPage/_pages/chromium_frame.pyi b/DrissionPage/_pages/chromium_frame.pyi index a2149bb..74c3314 100644 --- a/DrissionPage/_pages/chromium_frame.pyi +++ b/DrissionPage/_pages/chromium_frame.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from typing import Union, Tuple, List, Any diff --git a/DrissionPage/_pages/chromium_page.py b/DrissionPage/_pages/chromium_page.py index c4a0a93..820d085 100644 --- a/DrissionPage/_pages/chromium_page.py +++ b/DrissionPage/_pages/chromium_page.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from time import sleep, perf_counter @@ -11,7 +13,7 @@ from requests import get from .._base.browser import Browser from .._functions.browser import connect_browser from .._configs.chromium_options import ChromiumOptions, PortFinder -from .._pages.chromium_base import ChromiumBase, get_mhtml, Timeout +from .._pages.chromium_base import ChromiumBase, get_mhtml, get_pdf, Timeout from .._pages.chromium_tab import ChromiumTab from .._units.setter import ChromiumPageSetter from .._units.waiter import PageWaiter @@ -146,13 +148,15 @@ class ChromiumPage(ChromiumBase): """返回浏览器进程id""" return self.browser.process_id - def save(self, path=None, name=None): - """把当前页面保存为mhtml文件，如果path和name参数都为None，只返回mhtml文本 + def save(self, path=None, name=None, as_pdf=False, **kwargs): + """把当前页面保存为文件，如果path和name参数都为None，只返回文本 :param path: 保存路径，为None且name不为None时保存在当前路径 :param name: 文件名，为None且path不为None时用title属性值 - :return: mhtml文本 + :param as_pdf: 为Ture保存为pdf，否则为mhtml且忽略kwargs参数 + :param kwargs: pdf生成参数 + :return: as_pdf为True时返回bytes，否则返回文件文本 """ - return get_mhtml(self, path, name) + return get_pdf(self, path, name, kwargs)if as_pdf else get_mhtml(self, path, name) def get_tab(self, id_or_num=None): """获取一个标签页对象 diff --git a/DrissionPage/_pages/chromium_page.pyi b/DrissionPage/_pages/chromium_page.pyi index 6617fde..407cd1a 100644 --- a/DrissionPage/_pages/chromium_page.pyi +++ b/DrissionPage/_pages/chromium_page.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from typing import Union, Tuple, List, Optional @@ -55,7 +57,26 @@ class ChromiumPage(ChromiumBase): @property def set(self) -> ChromiumPageSetter: ... - def save(self, path: Union[str, Path] = None, name: str = None) -> str: ... + def save(self, + path: Union[str, Path] = None, + name: str = None, + as_pdf: bool = False, + landscape: bool = ..., + displayHeaderFooter: bool = ..., + printBackground: bool = ..., + scale: float = ..., + paperWidth: float = ..., + paperHeight: float = ..., + marginTop: float = ..., + marginBottom: float = ..., + marginLeft: float = ..., + marginRight: float = ..., + pageRanges: str = ..., + headerTemplate: str = ..., + footerTemplate: str = ..., + preferCSSPageSize: bool = ..., + generateTaggedPDF: bool = ..., + generateDocumentOutline: bool = ...) -> Union[bytes, str]: ... def get_tab(self, tab_id: Union[str, ChromiumTab, int] = None) -> ChromiumTab: ... diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 06ae0fa..9eaf97e 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from copy import copy diff --git a/DrissionPage/_pages/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi index f80d0a6..7735312 100644 --- a/DrissionPage/_pages/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from typing import Union, Tuple, Any, List, Optional diff --git a/DrissionPage/_pages/session_page.py b/DrissionPage/_pages/session_page.py index 0f3ddce..4faa2c3 100644 --- a/DrissionPage/_pages/session_page.py +++ b/DrissionPage/_pages/session_page.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from re import search, DOTALL diff --git a/DrissionPage/_pages/session_page.pyi b/DrissionPage/_pages/session_page.pyi index dd4da4e..a6379ca 100644 --- a/DrissionPage/_pages/session_page.pyi +++ b/DrissionPage/_pages/session_page.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from typing import Any, Union, Tuple, List, Optional diff --git a/DrissionPage/_pages/web_page.py b/DrissionPage/_pages/web_page.py index 7fec00d..fd4e079 100644 --- a/DrissionPage/_pages/web_page.py +++ b/DrissionPage/_pages/web_page.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from .chromium_page import ChromiumPage from .chromium_tab import WebPageTab diff --git a/DrissionPage/_pages/web_page.pyi b/DrissionPage/_pages/web_page.pyi index 2b1671d..20bcdf0 100644 --- a/DrissionPage/_pages/web_page.pyi +++ b/DrissionPage/_pages/web_page.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from typing import Union, Tuple, List, Any diff --git a/DrissionPage/_units/actions.py b/DrissionPage/_units/actions.py index 7f47b11..0a41be9 100644 --- a/DrissionPage/_units/actions.py +++ b/DrissionPage/_units/actions.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from time import sleep, perf_counter diff --git a/DrissionPage/_units/actions.pyi b/DrissionPage/_units/actions.pyi index 8a56363..55f863c 100644 --- a/DrissionPage/_units/actions.pyi +++ b/DrissionPage/_units/actions.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from typing import Union, Tuple, Any, Literal diff --git a/DrissionPage/_units/clicker.py b/DrissionPage/_units/clicker.py index 6565e50..1c397ac 100644 --- a/DrissionPage/_units/clicker.py +++ b/DrissionPage/_units/clicker.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from time import perf_counter, sleep diff --git a/DrissionPage/_units/clicker.pyi b/DrissionPage/_units/clicker.pyi index 34286a9..895f762 100644 --- a/DrissionPage/_units/clicker.pyi +++ b/DrissionPage/_units/clicker.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from typing import Union, Optional diff --git a/DrissionPage/_units/cookies_setter.py b/DrissionPage/_units/cookies_setter.py index 3eda66a..63bcc2f 100644 --- a/DrissionPage/_units/cookies_setter.py +++ b/DrissionPage/_units/cookies_setter.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from http.cookiejar import Cookie diff --git a/DrissionPage/_units/cookies_setter.pyi b/DrissionPage/_units/cookies_setter.pyi index f1b5877..3c6f37a 100644 --- a/DrissionPage/_units/cookies_setter.pyi +++ b/DrissionPage/_units/cookies_setter.pyi @@ -1,4 +1,10 @@ # -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. +""" from http.cookiejar import Cookie from typing import Union diff --git a/DrissionPage/_units/downloader.py b/DrissionPage/_units/downloader.py index 9d84e8b..2d0cdb4 100644 --- a/DrissionPage/_units/downloader.py +++ b/DrissionPage/_units/downloader.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from os.path import sep from pathlib import Path diff --git a/DrissionPage/_units/downloader.pyi b/DrissionPage/_units/downloader.pyi index 1617d56..37d8ae7 100644 --- a/DrissionPage/_units/downloader.pyi +++ b/DrissionPage/_units/downloader.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from typing import Dict, Optional, Union, Literal diff --git a/DrissionPage/_units/listener.py b/DrissionPage/_units/listener.py index 343a066..9063b53 100644 --- a/DrissionPage/_units/listener.py +++ b/DrissionPage/_units/listener.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from base64 import b64decode from json import JSONDecodeError, loads diff --git a/DrissionPage/_units/listener.pyi b/DrissionPage/_units/listener.pyi index b2511df..9daef5d 100644 --- a/DrissionPage/_units/listener.pyi +++ b/DrissionPage/_units/listener.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from queue import Queue from typing import Union, Dict, List, Iterable, Optional, Literal diff --git a/DrissionPage/_units/rect.py b/DrissionPage/_units/rect.py index ac1a8a9..3011ec6 100644 --- a/DrissionPage/_units/rect.py +++ b/DrissionPage/_units/rect.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ diff --git a/DrissionPage/_units/rect.pyi b/DrissionPage/_units/rect.pyi index deb346f..2e442ce 100644 --- a/DrissionPage/_units/rect.pyi +++ b/DrissionPage/_units/rect.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from typing import Tuple, Union, List diff --git a/DrissionPage/_units/screencast.py b/DrissionPage/_units/screencast.py index b752f27..958dbfc 100644 --- a/DrissionPage/_units/screencast.py +++ b/DrissionPage/_units/screencast.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from base64 import b64decode from os.path import sep diff --git a/DrissionPage/_units/screencast.pyi b/DrissionPage/_units/screencast.pyi index c185375..92d7d0f 100644 --- a/DrissionPage/_units/screencast.pyi +++ b/DrissionPage/_units/screencast.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from typing import Union diff --git a/DrissionPage/_units/scroller.py b/DrissionPage/_units/scroller.py index b1b987e..545b3f7 100644 --- a/DrissionPage/_units/scroller.py +++ b/DrissionPage/_units/scroller.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from time import sleep, perf_counter diff --git a/DrissionPage/_units/scroller.pyi b/DrissionPage/_units/scroller.pyi index 510543d..8233948 100644 --- a/DrissionPage/_units/scroller.pyi +++ b/DrissionPage/_units/scroller.pyi @@ -1,4 +1,10 @@ # -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. +""" from typing import Union from .._elements.chromium_element import ChromiumElement diff --git a/DrissionPage/_units/selector.py b/DrissionPage/_units/selector.py index e7a84d3..1b94ed7 100644 --- a/DrissionPage/_units/selector.py +++ b/DrissionPage/_units/selector.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from time import perf_counter diff --git a/DrissionPage/_units/selector.pyi b/DrissionPage/_units/selector.pyi index f0d2fcc..8e74554 100644 --- a/DrissionPage/_units/selector.pyi +++ b/DrissionPage/_units/selector.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from typing import Union, Tuple, List diff --git a/DrissionPage/_units/setter.py b/DrissionPage/_units/setter.py index f6d0f87..7928a93 100644 --- a/DrissionPage/_units/setter.py +++ b/DrissionPage/_units/setter.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path diff --git a/DrissionPage/_units/setter.pyi b/DrissionPage/_units/setter.pyi index c68d848..a5ece0b 100644 --- a/DrissionPage/_units/setter.pyi +++ b/DrissionPage/_units/setter.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from pathlib import Path from typing import Union, Tuple, Literal, Any, Optional diff --git a/DrissionPage/_units/states.py b/DrissionPage/_units/states.py index 9736a33..bdf99cc 100644 --- a/DrissionPage/_units/states.py +++ b/DrissionPage/_units/states.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from .._functions.web import location_in_viewport from ..errors import CDPError, NoRectError, PageDisconnectedError, ElementLostError diff --git a/DrissionPage/_units/states.pyi b/DrissionPage/_units/states.pyi index dcd57e5..067c0bd 100644 --- a/DrissionPage/_units/states.pyi +++ b/DrissionPage/_units/states.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from typing import Union, Tuple, List, Optional, Literal diff --git a/DrissionPage/_units/waiter.py b/DrissionPage/_units/waiter.py index 84a6676..1569b0e 100644 --- a/DrissionPage/_units/waiter.py +++ b/DrissionPage/_units/waiter.py @@ -1,4 +1,10 @@ # -*- coding:utf-8 -*- +""" +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. +""" from time import sleep, perf_counter from .._functions.settings import Settings diff --git a/DrissionPage/_units/waiter.pyi b/DrissionPage/_units/waiter.pyi index 9e0ada1..a0c8603 100644 --- a/DrissionPage/_units/waiter.pyi +++ b/DrissionPage/_units/waiter.pyi @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from typing import Union diff --git a/DrissionPage/common.py b/DrissionPage/common.py index 44ab030..df2b6aa 100644 --- a/DrissionPage/common.py +++ b/DrissionPage/common.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ from ._elements.session_element import make_session_ele from ._functions.by import By diff --git a/DrissionPage/errors.py b/DrissionPage/errors.py index 155453e..9ed94df 100644 --- a/DrissionPage/errors.py +++ b/DrissionPage/errors.py @@ -1,7 +1,9 @@ # -*- coding:utf-8 -*- """ -@Author : g1879 -@Contact : g1879@qq.com +@Author : g1879 +@Contact : g1879@qq.com +@Copyright: (c) 2024 by g1879, Inc. All Rights Reserved. +@License : BSD 3-Clause. """ diff --git a/README.md b/README.md index d775971..9dfc9ec 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ DrissionPage 是一个基于 python 的网页自动化工具。 支持系统：Windows、Linux、Mac -python 版本：3.6 及以上 +python 版本：3.8 及以上 支持浏览器：Chromium 内核浏览器(如 Chrome 和 Edge)，electron 应用 @@ -108,7 +108,7 @@ python 版本：3.6 及以上 # 🔖 版本历史 -[点击查看版本历史](https://g1879.gitee.io/drissionpagedocs/history/) +[点击查看版本历史](https://g1879.gitee.io/drissionpagedocs/history/introduction/) --- diff --git a/requirements.txt b/requirements.txt index 8d24602..8a430e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ requests lxml cssselect -DownloadKit>=2.0.0b5 +DownloadKit>=2.0.0 websocket-client>=1.7.0 click tldextract diff --git a/setup.py b/setup.py index 8f5dd8f..4cfbfa7 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0b35", + version="4.0.0", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.", @@ -22,7 +22,7 @@ setup( 'lxml', 'requests', 'cssselect', - 'DownloadKit>=2.0.0b5', + 'DownloadKit>=2.0.0', 'websocket-client>=1.7.0', 'click', 'tldextract', From 29de18c023c07f2e85d229e9ceca5eaba8d657ef Mon Sep 17 00:00:00 2001 From: g1879 Date: Mon, 8 Jan 2024 18:58:56 +0800 Subject: [PATCH 182/182] 4.0.1 --- DrissionPage/__init__.py | 2 +- DrissionPage/_pages/chromium_tab.py | 12 +++++++----- DrissionPage/_pages/chromium_tab.pyi | 21 ++++++++++++++++++++- setup.py | 2 +- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/DrissionPage/__init__.py b/DrissionPage/__init__.py index 5eb95ff..0f1467c 100644 --- a/DrissionPage/__init__.py +++ b/DrissionPage/__init__.py @@ -14,4 +14,4 @@ from ._configs.chromium_options import ChromiumOptions from ._configs.session_options import SessionOptions __all__ = ['ChromiumPage', 'ChromiumOptions', 'SessionOptions', 'SessionPage', 'WebPage', '__version__'] -__version__ = '4.0.0' +__version__ = '4.0.1' diff --git a/DrissionPage/_pages/chromium_tab.py b/DrissionPage/_pages/chromium_tab.py index 9eaf97e..cbab53e 100644 --- a/DrissionPage/_pages/chromium_tab.py +++ b/DrissionPage/_pages/chromium_tab.py @@ -10,7 +10,7 @@ from copy import copy from .._base.base import BasePage from .._configs.session_options import SessionOptions from .._functions.web import set_session_cookies, set_browser_cookies -from .._pages.chromium_base import ChromiumBase, get_mhtml +from .._pages.chromium_base import ChromiumBase, get_mhtml, get_pdf from .._pages.session_page import SessionPage from .._units.setter import TabSetter, WebPageTabSetter from .._units.waiter import TabWaiter @@ -60,13 +60,15 @@ class ChromiumTab(ChromiumBase): self._wait = TabWaiter(self) return self._wait - def save(self, path=None, name=None): - """把当前页面保存为mhtml文件，如果path和name参数都为None，只返回mhtml文本 + def save(self, path=None, name=None, as_pdf=False, **kwargs): + """把当前页面保存为文件，如果path和name参数都为None，只返回文本 :param path: 保存路径，为None且name不为None时保存在当前路径 :param name: 文件名，为None且path不为None时用title属性值 - :return: mhtml文本 + :param as_pdf: 为Ture保存为pdf，否则为mhtml且忽略kwargs参数 + :param kwargs: pdf生成参数 + :return: as_pdf为True时返回bytes，否则返回文件文本 """ - return get_mhtml(self, path, name) + return get_pdf(self, path, name, kwargs) if as_pdf else get_mhtml(self, path, name) def __repr__(self): return f'' diff --git a/DrissionPage/_pages/chromium_tab.pyi b/DrissionPage/_pages/chromium_tab.pyi index 7735312..de61132 100644 --- a/DrissionPage/_pages/chromium_tab.pyi +++ b/DrissionPage/_pages/chromium_tab.pyi @@ -44,7 +44,26 @@ class ChromiumTab(ChromiumBase): @property def wait(self) -> TabWaiter: ... - def save(self, path: Union[str, Path] = None, name: str = None) -> str: ... + def save(self, + path: Union[str, Path] = None, + name: str = None, + as_pdf: bool = False, + landscape: bool = ..., + displayHeaderFooter: bool = ..., + printBackground: bool = ..., + scale: float = ..., + paperWidth: float = ..., + paperHeight: float = ..., + marginTop: float = ..., + marginBottom: float = ..., + marginLeft: float = ..., + marginRight: float = ..., + pageRanges: str = ..., + headerTemplate: str = ..., + footerTemplate: str = ..., + preferCSSPageSize: bool = ..., + generateTaggedPDF: bool = ..., + generateDocumentOutline: bool = ...) -> Union[bytes, str]: ... class WebPageTab(SessionPage, ChromiumTab): diff --git a/setup.py b/setup.py index 4cfbfa7..4e39d05 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding='utf-8') as fh: setup( name="DrissionPage", - version="4.0.0", + version="4.0.1", author="g1879", author_email="g1879@qq.com", description="Python based web automation tool. It can control the browser and send and receive data packets.",