4.0.0b3抓包可包含扩展信息；url不对所有允许的字符转义

DrissionPage/_base/base.py

@@ -414,7 +414,7 @@ class BasePage(BaseParser):
         :param interval: 重试间隔
         :return: 重试次数和间隔组成的tuple
         """
-        self._url = quote(url, safe='/:&?=%;#@+![]')
+        self._url = quote(url, safe='-_.~!*\'();:@&=+$,/?#[]')
         retry = retry if retry is not None else self.retry_times
         interval = interval if interval is not None else self.retry_interval
         return retry, interval

DrissionPage/_base/chromium_driver.py

@@ -155,7 +155,7 @@ class ChromiumDriver(object):
         if self._stopped.is_set():
             return {'error': 'tab closed', 'type': 'tab_closed'}
 
-        timeout = kwargs.pop("_timeout", None)
+        timeout = kwargs.pop("_timeout", 30)
         result = self._send({"method": _method, "params": kwargs}, timeout=timeout)
         if result is None:
             return {'error': 'tab closed', 'type': 'tab_closed'}

DrissionPage/_pages/chromium_base.py

@@ -48,6 +48,7 @@ class ChromiumBase(BasePage):
         self._listener = None
         self._has_alert = False
         self._ready_state = None
+        self._doc_got = False  # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc
 
         self._download_path = str(Path('.').absolute())
 
@@ -145,28 +146,6 @@ class ChromiumBase(BasePage):
         self._is_loading = False
         self._is_reading = False
 
-    def _wait_loaded(self, timeout=None):
-        """等待页面加载完成，超时触发停止加载
-        :param timeout: 超时时间
-        :return: 是否成功，超时返回False
-        """
-        if self.page_load_strategy == 'none':
-            return True
-
-        timeout = timeout if timeout is not None else self.timeouts.page_load
-        end_time = perf_counter() + timeout
-        while perf_counter() < end_time:
-            if self._ready_state == 'complete':
-                return True
-            elif self.page_load_strategy == 'eager' and self._ready_state in ('interactive', 'complete'):
-                self.stop_loading()
-                return True
-
-            sleep(.1)
-
-        self.stop_loading()
-        return False
-
     def _onFrameDetached(self, **kwargs):
         try:
             self.browser._frames.pop(kwargs['frameId'])
@@ -210,16 +189,18 @@ class ChromiumBase(BasePage):
         self._ready_state = 'complete'
         if self._debug:
             print(f'LoadEventFired {kwargs}')
-        # self._get_document()
+        self._get_document()
+        self._doc_got = True
 
     def _onFrameStoppedLoading(self, **kwargs):
         """页面加载完成后执行"""
         self.browser._frames[kwargs['frameId']] = self.tab_id
-        if kwargs['frameId'] == self._frame_id:
+        if kwargs['frameId'] == self._frame_id and self._doc_got is False:
             self._ready_state = 'complete'
             if self._debug:
                 print(f'FrameStoppedLoading {kwargs}')
             self._get_document()
+            self._doc_got = False
 
     def _onFileChooserOpened(self, **kwargs):
         """文件选择框打开时执行"""
@@ -845,6 +826,28 @@ class ChromiumBase(BasePage):
         self._alert.response_text = None
         self._has_alert = True
 
+    def _wait_loaded(self, timeout=None):
+        """等待页面加载完成，超时触发停止加载
+        :param timeout: 超时时间
+        :return: 是否成功，超时返回False
+        """
+        if self.page_load_strategy == 'none':
+            return True
+
+        timeout = timeout if timeout is not None else self.timeouts.page_load
+        end_time = perf_counter() + timeout
+        while perf_counter() < end_time:
+            if self._ready_state == 'complete':
+                return True
+            elif self.page_load_strategy == 'eager' and self._ready_state in ('interactive',
+                                                                              'complete') and not self._is_loading:
+                return True
+
+            sleep(.1)
+
+        self.stop_loading()
+        return False
+
     def _d_connect(self, to_url, times=0, interval=1, show_errmsg=False, timeout=None):
         """尝试连接，重试若干次
         :param to_url: 要访问的url

DrissionPage/_pages/chromium_base.pyi

@@ -32,6 +32,7 @@ class ChromiumBase(BasePage):
         self._driver: ChromiumDriver = ...
         self._frame_id: str = ...
         self._is_reading: bool = ...
+        self._is_timeout: bool = ...
         self._timeouts: Timeout = ...
         self._first_run: bool = ...
         self._is_loading: bool = ...
@@ -48,6 +49,7 @@ class ChromiumBase(BasePage):
         self._listener: NetworkListener = ...
         self._alert: Alert = ...
         self._has_alert: bool = ...
+        self._doc_got: bool = ...
         self._ready_state: Optional[str] = ...
 
     def _connect_browser(self, tab_id: str = None) -> None: ...

DrissionPage/_units/network_listener.py

@@ -179,17 +179,20 @@ class NetworkListener(object):
     def _set_callback(self):
         """设置监听请求的回调函数"""
         self._driver.set_listener('Network.requestWillBeSent', self._requestWillBeSent)
+        self._driver.set_listener('Network.requestWillBeSentExtraInfo', self._requestWillBeSentExtraInfo)
         self._driver.set_listener('Network.responseReceived', self._response_received)
+        self._driver.set_listener('Network.responseReceivedExtraInfo', self._responseReceivedExtraInfo)
         self._driver.set_listener('Network.loadingFinished', self._loading_finished)
         self._driver.set_listener('Network.loadingFailed', self._loading_failed)
 
     def _requestWillBeSent(self, **kwargs):
         """接收到请求时的回调函数"""
         if not self._targets:
-            self._request_ids[kwargs['requestId']] = DataPacket(self._page.tab_id, None, kwargs)
+            packet = self._request_ids.setdefault(kwargs['requestId'], DataPacket(self._page.tab_id, None))
+            packet._raw_request = kwargs
             if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None):
-                self._request_ids[kwargs['requestId']]._raw_post_data = \
-                    self._driver.call_method('Network.getRequestPostData', requestId=kwargs['requestId'])['postData']
+                packet._raw_post_data = self._driver.call_method('Network.getRequestPostData',
+                                                                 requestId=kwargs['requestId'])['postData']
 
             return
 
@@ -197,20 +200,40 @@ class NetworkListener(object):
             if ((self._is_regex and search(target, kwargs['request']['url'])) or
                 (not self._is_regex and target in kwargs['request']['url'])) and (
                     not self._method or kwargs['request']['method'] in self._method):
-                self._request_ids[kwargs['requestId']] = DataPacket(self._page.tab_id, target, kwargs)
-
+                packet = self._request_ids.setdefault(kwargs['requestId'], DataPacket(self._page.tab_id, None))
+                packet._raw__request = kwargs
                 if kwargs['request'].get('hasPostData', None) and not kwargs['request'].get('postData', None):
-                    self._request_ids[kwargs['requestId']]._raw_post_data = \
-                        self._driver.call_method('Network.getRequestPostData', requestId=kwargs['requestId'])['postData']
+                    packet._raw_post_data = self._driver.call_method('Network.getRequestPostData',
+                                                                     requestId=kwargs['requestId'])['postData']
+
+                break
+
+    def _requestWillBeSentExtraInfo(self, **kwargs):
+        if not self._targets:
+            packet = self._request_ids.setdefault(kwargs['requestId'], DataPacket(self._page.tab_id, None))
+            packet._requestExtraInfo = kwargs
+            return
+
+        for target in self._targets:
+            if ((self._is_regex and search(target, kwargs['request']['url'])) or
+                (not self._is_regex and target in kwargs['request']['url'])) and (
+                    not self._method or kwargs['request']['method'] in self._method):
+                packet = self._request_ids.setdefault(kwargs['requestId'], DataPacket(self._page.tab_id, None))
+                packet._requestExtraInfo = kwargs
 
                 break
 
     def _response_received(self, **kwargs):
         """接收到返回信息时处理方法"""
-        request_id = kwargs['requestId']
-        if request_id in self._request_ids:
-            self._request_ids[request_id]._raw_response = kwargs['response']
-            self._request_ids[request_id]._resource_type = kwargs['type']
+        request = self._request_ids.get(kwargs['requestId'])
+        if request:
+            request._raw_response = kwargs['response']
+            request._resource_type = kwargs['type']
+
+    def _responseReceivedExtraInfo(self, **kwargs):
+        request = self._request_ids.get(kwargs['requestId'])
+        if request:
+            request._responseExtraInfo = kwargs
 
     def _loading_finished(self, **kwargs):
         """请求完成时处理方法"""
@@ -249,21 +272,22 @@ class NetworkListener(object):
 class DataPacket(object):
     """返回的数据包管理类"""
 
-    def __init__(self, tab_id, target, raw_request):
+    def __init__(self, tab_id, target):
         """
         :param tab_id: 产生这个数据包的tab的id
         :param target: 监听目标
-        :param raw_request: 原始request数据，从cdp获得
         """
         self.tab_id = tab_id
         self.target = target
 
-        self._raw_request = raw_request
+        self._raw_request = None
         self._raw_post_data = None
 
         self._raw_response = None
         self._raw_body = None
         self._base64_body = False
+        self._requestExtraInfo = None
+        self._responseExtraInfo = None
 
         self._request = None
         self._response = None
@@ -293,22 +317,23 @@ class DataPacket(object):
     @property
     def request(self):
         if self._request is None:
-            self._request = Request(self._raw_request['request'], self._raw_post_data)
+            self._request = Request(self._raw_request['request'], self._raw_post_data, self._requestExtraInfo)
         return self._request
 
     @property
     def response(self):
         if self._response is None:
-            self._response = Response(self._raw_response, self._raw_body, self._base64_body)
+            self._response = Response(self._raw_response, self._raw_body, self._base64_body, self._responseExtraInfo)
         return self._response
 
 
 class Request(object):
-    def __init__(self, raw_request, post_data):
+    def __init__(self, raw_request, post_data, extra_info):
         self._request = raw_request
         self._raw_post_data = post_data
         self._postData = None
         self._headers = None
+        self.extra_info = RequestExtraInfo(extra_info or {})
 
     def __getattr__(self, item):
         return self._request.get(item, None)
@@ -338,12 +363,13 @@ class Request(object):
 
 
 class Response(object):
-    def __init__(self, raw_response, raw_body, base64_body):
+    def __init__(self, raw_response, raw_body, base64_body, extra_info):
         self._response = raw_response
         self._raw_body = raw_body
         self._is_base64_body = base64_body
         self._body = None
         self._headers = None
+        self.extra_info = ResponseExtraInfo(extra_info or {})
 
     def __getattr__(self, item):
         return self._response.get(item, None)
@@ -374,3 +400,19 @@ class Response(object):
                     self._body = self._raw_body
 
         return self._body
+
+
+class ExtraInfo(object):
+    def __init__(self, extra_info):
+        self._extra_info = extra_info
+
+    def __getattr__(self, item):
+        return self._extra_info.get(item, None)
+
+
+class RequestExtraInfo(ExtraInfo):
+    pass
+
+
+class ResponseExtraInfo(ExtraInfo):
+    pass

DrissionPage/_units/network_listener.pyi

@@ -35,7 +35,8 @@ class NetworkListener(object):
 
     def go_on(self) -> None: ...
 
-    def wait(self, count: int = 1, timeout: float = None, fix_count: bool = True): ...
+    def wait(self, count: int = 1, timeout: float = None,
+             fix_count: bool = True) -> Union[List[DataPacket], DataPacket, None]: ...
 
     @property
     def results(self) -> Union[DataPacket, Dict[str, List[DataPacket]], False]: ...
@@ -48,8 +49,12 @@ class NetworkListener(object):
 
     def _requestWillBeSent(self, **kwargs) -> None: ...
 
+    def _requestWillBeSentExtraInfo(self, **kwargs) -> None: ...
+
     def _response_received(self, **kwargs) -> None: ...
 
+    def _responseReceivedExtraInfo(self, **kwargs) -> None: ...
+
     def _loading_finished(self, **kwargs) -> None: ...
 
     def _loading_failed(self, **kwargs) -> None: ...
@@ -63,11 +68,11 @@ class NetworkListener(object):
 class DataPacket(object):
     """返回的数据包管理类"""
 
-    def __init__(self, tab_id: str, target: Optional[str], raw_info: dict):
+    def __init__(self, tab_id: str, target: Optional[str]):
         self.tab_id: str = ...
         self.target: str = ...
-        self._raw_request: dict = ...
-        self._raw_response: dict = ...
+        self._raw_request: Optional[dict] = ...
+        self._raw_response: Optional[dict] = ...
         self._raw_post_data: str = ...
         self._raw_body: str = ...
         self._base64_body: bool = ...
@@ -75,6 +80,8 @@ class DataPacket(object):
         self._response: Response = ...
         self.errorText: str = ...
         self._resource_type: str = ...
+        self._requestExtraInfo: Optional[dict] = ...
+        self._responseExtraInfo: Optional[dict] = ...
 
     @property
     def url(self) -> str: ...
@@ -97,6 +104,7 @@ class DataPacket(object):
 
 class Request(object):
     url: str = ...
+    extra_info: Optional[RequestExtraInfo] = ...
     _headers: Union[CaseInsensitiveDict, None] = ...
     method: str = ...
 
@@ -110,7 +118,7 @@ class Request(object):
     trustTokenParams = ...
     isSameSite = ...
 
-    def __init__(self, raw_request: dict, post_data: str):
+    def __init__(self, raw_request: dict, post_data: str, extra_info: Optional[dict]):
         self._request: dict = ...
         self._raw_post_data: str = ...
         self._postData: str = ...
@@ -123,6 +131,7 @@ class Request(object):
 
 
 class Response(object):
+    extra_info: Optional[ResponseExtraInfo] = ...
     url = ...
     status = ...
     statusText = ...
@@ -147,7 +156,7 @@ class Response(object):
     securityState = ...
     securityDetails = ...
 
-    def __init__(self, raw_response: dict, raw_body: str, base64_body: bool):
+    def __init__(self, raw_response: dict, raw_body: str, base64_body: bool, extra_info: Optional[dict]):
         self._response: dict = ...
         self._raw_body: str = ...
         self._is_base64_body: bool = ...
@@ -162,3 +171,28 @@ class Response(object):
 
     @property
     def body(self) -> Union[str, dict, bool]: ...
+
+
+class ExtraInfo(object):
+    def __init__(self, extra_info: dict):
+        self._extra_info: dict = ...
+
+
+class RequestExtraInfo(ExtraInfo):
+    requestId: str = ...
+    associatedCookies: List[dict] = ...
+    headers: dict = ...
+    connectTiming: dict = ...
+    clientSecurityState: dict = ...
+    siteHasCookieInOtherPartition: bool = ...
+
+
+class ResponseExtraInfo(ExtraInfo):
+    requestId: str = ...
+    blockedCookies: List[dict] = ...
+    headers: dict = ...
+    resourceIPAddressSpace: str = ...
+    statusCode: int = ...
+    headersText: str = ...
+    cookiePartitionKey: str = ...
+    cookiePartitionKeyOpaque: bool = ...

DrissionPage/_units/setter.py

@@ -122,6 +122,11 @@ class TabSetter(ChromiumBaseSetter):
     def __init__(self, page):
         super().__init__(page)
 
+    @property
+    def window(self):
+        """返回用于设置浏览器窗口的对象"""
+        return WindowSetter(self._page)
+
     def download_path(self, path):
         """设置下载路径
         :param path: 下载路径
@@ -165,11 +170,6 @@ class ChromiumPageSetter(TabSetter):
         """
         self._page._main_tab = tab_id or self._page.tab_id
 
-    @property
-    def window(self):
-        """返回用于设置浏览器窗口的对象"""
-        return WindowSetter(self._page)
-
     def tab_to_front(self, tab_or_id=None):
         """激活标签页使其处于最前面
         :param tab_or_id: 标签页对象或id，为None表示当前标签页

DrissionPage/_units/setter.pyi

@@ -56,6 +56,9 @@ class ChromiumBaseSetter(object):
 class TabSetter(ChromiumBaseSetter):
     def __init__(self, page): ...
 
+    @property
+    def window(self) -> WindowSetter: ...
+
     def download_path(self, path: Union[str, Path]) -> None: ...
 
     def download_file_name(self, name: str) -> None: ...
@@ -70,9 +73,6 @@ class ChromiumPageSetter(TabSetter):
 
     def main_tab(self, tab_id: str = None) -> None: ...
 
-    @property
-    def window(self) -> WindowSetter: ...
-
     def tab_to_front(self, tab_or_id: Union[str, ChromiumTab] = None) -> None: ...
 
 
@@ -184,8 +184,8 @@ class PageScrollSetter(object):
 
 class WindowSetter(object):
 
-    def __init__(self, page: ChromiumPage):
-        self._page: ChromiumPage = ...
+    def __init__(self, page: ChromiumBase):
+        self._page: ChromiumBase = ...
         self._window_id: str = ...
 
     def maximized(self) -> None: ...

DrissionPage/_units/tab_rect.py

@@ -12,16 +12,27 @@ class ChromiumTabRect(object):
     @property
     def window_state(self):
         """返回窗口状态：normal、fullscreen、maximized、 minimized"""
-        return self._get_browser_rect()['windowState']
+        return self._get_window_rect()['windowState']
 
     @property
-    def browser_location(self):
-        """返回浏览器在屏幕上的坐标，左上角为(0, 0)"""
-        r = self._get_browser_rect()
+    def window_location(self):
+        """返回窗口在屏幕上的坐标，左上角为(0, 0)"""
+        r = self._get_window_rect()
         if r['windowState'] in ('maximized', 'fullscreen'):
             return 0, 0
         return r['left'] + 7, r['top']
 
+    @property
+    def window_size(self):
+        """返回窗口大小"""
+        r = self._get_window_rect()
+        if r['windowState'] == 'fullscreen':
+            return r['width'], r['height']
+        elif r['windowState'] == 'maximized':
+            return r['width'] - 16, r['height'] - 16
+        else:
+            return r['width'] - 16, r['height'] - 7
+
     @property
     def page_location(self):
         """返回页面左上角在屏幕中坐标，左上角为(0, 0)"""
@@ -32,22 +43,11 @@ class ChromiumTabRect(object):
     @property
     def viewport_location(self):
         """返回视口在屏幕中坐标，左上角为(0, 0)"""
-        w_bl, h_bl = self.browser_location
-        w_bs, h_bs = self.browser_size
+        w_bl, h_bl = self.window_location
+        w_bs, h_bs = self.window_size
         w_vs, h_vs = self.viewport_size_with_scrollbar
         return w_bl + w_bs - w_vs, h_bl + h_bs - h_vs
 
-    @property
-    def browser_size(self):
-        """返回浏览器大小"""
-        r = self._get_browser_rect()
-        if r['windowState'] == 'fullscreen':
-            return r['width'], r['height']
-        elif r['windowState'] == 'maximized':
-            return r['width'] - 16, r['height'] - 16
-        else:
-            return r['width'] - 16, r['height'] - 7
-
     @property
     def page_size(self):
         """返回页面总宽高，格式：(宽, 高)"""
@@ -71,6 +71,6 @@ class ChromiumTabRect(object):
         """获取页面范围信息"""
         return self._page.run_cdp_loaded('Page.getLayoutMetrics')
 
-    def _get_browser_rect(self):
-        """获取浏览器范围信息"""
+    def _get_window_rect(self):
+        """获取窗口范围信息"""
         return self._page.browser.get_window_bounds(self._page.tab_id)

DrissionPage/_units/tab_rect.pyi

@@ -17,7 +17,7 @@ class ChromiumTabRect(object):
     def window_state(self) -> str: ...
 
     @property
-    def browser_location(self) -> Tuple[int, int]: ...
+    def window_location(self) -> Tuple[int, int]: ...
 
     @property
     def page_location(self) -> Tuple[int, int]: ...
@@ -26,7 +26,7 @@ class ChromiumTabRect(object):
     def viewport_location(self) -> Tuple[int, int]: ...
 
     @property
-    def browser_size(self) -> Tuple[int, int]: ...
+    def window_size(self) -> Tuple[int, int]: ...
 
     @property
     def page_size(self) -> Tuple[int, int]: ...
@@ -39,4 +39,4 @@ class ChromiumTabRect(object):
 
     def _get_page_rect(self) -> dict: ...
 
-    def _get_browser_rect(self) -> dict: ...
+    def _get_window_rect(self) -> dict: ...