diff --git a/DrissionPage/chromium_tab.py b/DrissionPage/chromium_tab.py index 5a6e685..dacf051 100644 --- a/DrissionPage/chromium_tab.py +++ b/DrissionPage/chromium_tab.py @@ -20,5 +20,5 @@ class ChromiumTab(ChromiumBase): def _set_options(self): self.set_timeouts(page_load=self.page.timeouts.page_load, script=self.page.timeouts.script, - implicit=self.page.timeouts.implicit if self.timeout is None else self.timeout) + implicit=self.page.timeouts.implicit) self._page_load_strategy = self.page.page_load_strategy diff --git a/DrissionPage/config.py b/DrissionPage/config.py index f75ed0a..5efdd6f 100644 --- a/DrissionPage/config.py +++ b/DrissionPage/config.py @@ -65,12 +65,21 @@ class OptionsManager(object): :param section: 段名 :param item: 项名 :param value: 项值 - :return: 当前对象 + :return: None """ self._conf.set(section, item, str(value)) self.__setattr__(f'_{section}', None) return self + def remove_item(self, section, item): + """删除配置值 \n + :param section: 段名 + :param item: 项名 + :return: None + """ + self._conf.remove_option(section, item) + return self + def save(self, path=None): """保存配置文件 \n :param path: ini文件的路径，传入 'default' 保存到默认ini文件 @@ -124,26 +133,22 @@ class SessionOptions(object): self._max_redirects = None self._timeout = 10 + self._del_set = set() # 记录要从ini文件删除的参数 + if read_file: self.ini_path = ini_path or str(Path(__file__).parent / 'configs.ini') om = OptionsManager(self.ini_path) options_dict = om.session_options if options_dict.get('headers', None) is not None: - self._headers = {key.lower(): options_dict['headers'][key] for key in options_dict['headers']} + self.set_headers(options_dict['headers']) if options_dict.get('cookies', None) is not None: - self._cookies = options_dict['cookies'] + self.set_cookies(options_dict['cookies']) if options_dict.get('auth', None) is not None: self._auth = options_dict['auth'] - if options_dict.get('proxies', None) is not None: - self._proxies = options_dict['proxies'] - - if options_dict.get('hooks', None) is not None: - self._hooks = options_dict['hooks'] - if options_dict.get('params', None) is not None: self._params = options_dict['params'] @@ -153,9 +158,6 @@ class SessionOptions(object): if options_dict.get('cert', None) is not None: self._cert = options_dict['cert'] - # if options_dict.get('adapters', None) is not None: - # self._adapters = options_dict['adapters'] - if options_dict.get('stream', None) is not None: self._stream = options_dict['stream'] @@ -165,169 +167,29 @@ class SessionOptions(object): if options_dict.get('max_redirects', None) is not None: self._max_redirects = options_dict['max_redirects'] - self._timeout = options_dict.get('timeout', 10) - + self.set_proxies(om.proxies.get('http', None), om.proxies.get('https', None)) + self._timeout = om.timeouts.get('implicit', 10) self._download_path = om.paths.get('download_path', None) - @property - def timeout(self): - """返回timeout属性信息""" - return self._timeout - + # ===========须独立处理的项开始============ @property def download_path(self): """返回默认下载路径属性信息""" return self._download_path - @property - def headers(self): - """返回headers设置信息""" - if self._headers is None: - self._headers = {} - return self._headers - - @property - def cookies(self): - """返回cookies设置信息""" - if self._cookies is None: - self._cookies = [] - return self._cookies - - @property - def auth(self): - """返回auth设置信息""" - return self._auth - - @property - def proxies(self): - """返回proxies设置信息""" - if self._proxies is None: - self._proxies = {} - return self._proxies - - @property - def hooks(self): - """返回hooks设置信息""" - if self._hooks is None: - self._hooks = {} - return self._hooks - - @property - def params(self): - """返回params设置信息""" - if self._params is None: - self._params = {} - return self._params - - @property - def verify(self): - """返回verify设置信息""" - return self._verify - - @property - def cert(self): - """返回cert设置信息""" - return self._cert - - @property - def adapters(self): - """返回adapters设置信息""" - return self._adapters - - @property - def stream(self): - """返回stream设置信息""" - return self._stream - - @property - def trust_env(self): - """返回trust_env设置信息""" - return self._trust_env - - @property - def max_redirects(self): - """返回max_redirects设置信息""" - return self._max_redirects - - @cookies.setter - def cookies(self, cookies): - """设置cookies参数 \n - :param cookies: 参数值 - :return: None + def set_paths(self, download_path=None): + """设置默认下载路径 \n + :param download_path: 下载路径 + :return: 返回当前对象 """ - self._cookies = cookies + if download_path is not None: + self._download_path = str(download_path) + return self - @auth.setter - def auth(self, auth): - """设置auth参数 \n - :param auth: 参数值 - :return: None - """ - self._auth = auth - - @hooks.setter - def hooks(self, hooks): - """设置hooks参数 \n - :param hooks: 参数值 - :return: None - """ - self._hooks = hooks - - @params.setter - def params(self, params): - """设置params参数 \n - :param params: 参数值 - :return: None - """ - self._params = params - - @verify.setter - def verify(self, verify): - """设置verify参数 \n - :param verify: 参数值 - :return: None - """ - self._verify = verify - - @cert.setter - def cert(self, cert): - """设置cert参数 \n - :param cert: 参数值 - :return: None - """ - self._cert = cert - - @adapters.setter - def adapters(self, adapters): - """设置 \n - :param adapters: 参数值 - :return: None - """ - self._adapters = adapters - - @stream.setter - def stream(self, stream): - """设置stream参数 \n - :param stream: 参数值 - :return: None - """ - self._stream = stream - - @trust_env.setter - def trust_env(self, trust_env): - """设置trust_env参数 \n - :param trust_env: 参数值 - :return: None - """ - self._trust_env = trust_env - - @max_redirects.setter - def max_redirects(self, max_redirects): - """设置max_redirects参数 \n - :param max_redirects: 参数值 - :return: None - """ - self._max_redirects = max_redirects + @property + def timeout(self): + """返回timeout属性信息""" + return self._timeout def set_timeout(self, second): """设置超时信息 @@ -337,12 +199,42 @@ class SessionOptions(object): self._timeout = second return self - def set_headers(self, headers): - """设置headers参数 \n - :param headers: 参数值 + @property + def proxies(self): + """返回proxies设置信息""" + if self._proxies is None: + self._proxies = {} + return self._proxies + + def set_proxies(self, http, https=None): + """设置proxies参数 \n + :param http: http代理地址 + :param https: https代理地址 :return: 返回当前对象 """ - self._headers = {key.lower(): headers[key] for key in headers} + proxies = None if http == https is None else {'http': http, 'https': https or http} + self._sets('proxies', proxies) + return self + + # ===========须独立处理的项结束============ + + @property + def headers(self): + """返回headers设置信息""" + if self._headers is None: + self._headers = {} + return self._headers + + def set_headers(self, headers): + """设置headers参数 \n + :param headers: 参数值，传入None可在ini文件标记删除 + :return: 返回当前对象 + """ + if headers is None: + self._headers = None + self._del_set.add('headers') + else: + self._headers = {key.lower(): headers[key] for key in headers} return self def set_a_header(self, attr, value): @@ -371,25 +263,160 @@ class SessionOptions(object): return self - def set_proxies(self, proxies): - """设置proxies参数 \n - {'http': 'http://xx.xx.xx.xx:xxxx', - 'https': 'http://xx.xx.xx.xx:xxxx'} - :param proxies: 参数值 + @property + def cookies(self): + """以list形式返回cookies""" + if self._cookies is None: + self._cookies = [] + return self._cookies + + def set_cookies(self, cookies): + """设置cookies信息 \n + :param cookies: cookies，可为CookieJar, list, tuple, str, dict，传入None可在ini文件标记删除 :return: 返回当前对象 """ - self._proxies = proxies + cookies = cookies if cookies is None else list(cookies_to_tuple(cookies)) + self._sets('cookies', cookies) return self - def set_paths(self, download_path=None): - """设置默认下载路径 \n - :param download_path: 下载路径 + @property + def auth(self): + """返回auth设置信息""" + return self._auth + + def set_auth(self, auth): + """设置认证元组或对象 \n + :param auth: 认证元组或对象 :return: 返回当前对象 """ - if download_path is not None: - self._download_path = str(download_path) + self._sets('auth', auth) return self + @property + def hooks(self): + """返回回调方法""" + if self._hooks is None: + self._hooks = {} + return self._hooks + + def set_hooks(self, hooks): + """设置回调方法 \n + :param hooks: + :return: 返回当前对象 + """ + self._hooks = hooks + return self + + @property + def params(self): + """返回params设置信息""" + if self._params is None: + self._params = {} + return self._params + + def set_params(self, params): + """设置查询参数字典 \n + :param params: 查询参数字典 + :return: 返回当前对象 + """ + self._sets('params', params) + return self + + @property + def verify(self): + """返回是否验证SSL证书设置""" + return self._verify + + def set_verify(self, on_off): + """设置是否验证SSL证书 \n + :param on_off: 是否验证 SSL 证书 + :return: 返回当前对象 + """ + self._sets('verify', on_off) + return self + + @property + def cert(self): + """返回cert设置信息""" + return self._cert + + def set_cert(self, cert): + """SSL客户端证书文件的路径(.pem格式)，或(‘cert’, ‘key’)元组 \n + :param cert: 证书路径或元组 + :return: 返回当前对象 + """ + self._sets('cert', cert) + return self + + @property + def adapters(self): + """返回适配器设置信息""" + if self._adapters is None: + self._adapters = [] + return self._adapters + + def add_adapter(self, url, adapter): + """添加适配器 \n + :param url: 适配器对应url + :param adapter: 适配器对象 + :return: 返回当前对象 + """ + self._adapters.append((url, adapter)) + return self + + @property + def stream(self): + """返回stream设置信息""" + return self._stream + + def set_stream(self, on_off): + """设置是否使用流式响应内容 \n + :param on_off: 是否使用流式响应内容 + :return: 返回当前对象 + """ + self._sets('stream', on_off) + return self + + @property + def trust_env(self): + """返回trust_env设置信息""" + return self._trust_env + + def set_trust_env(self, on_off): + """设置是否信任环境 \n + :param on_off: 是否信任环境 + :return: 返回当前对象 + """ + self._sets('trust_env', on_off) + return self + + @property + def max_redirects(self): + """返回最大重定向次数""" + return self._max_redirects + + def set_max_redirects(self, times): + """设置最大重定向次数 \n + :param times: 最大重定向次数 + :return: 返回当前对象 + """ + self._sets('max_redirects', times) + return self + + def _sets(self, arg, val): + """给属性赋值或标记删除 + :param arg: 属性名称 + :param val: 参数值 + :return: None + """ + if val is None: + self.__setattr__(f'_{arg}', None) + self._del_set.add(arg) + else: + self.__setattr__(f'_{arg}', val) + if arg in self._del_set: + self._del_set.remove(arg) + def save(self, path=None): """保存设置到文件 \n :param path: ini文件的路径，传入 'default' 保存到默认ini文件 @@ -417,10 +444,22 @@ class SessionOptions(object): options = session_options_to_dict(self) for i in options: - om.set_item('session_options', i, options[i]) + if i not in ('download_path', 'timeout', 'proxies'): + om.set_item('session_options', i, options[i]) om.set_item('paths', 'download_path', self.download_path) - om.set_item('session_options', 'timeout', self.timeout) + om.set_item('timeouts', 'implicit', self.timeout) + om.set_item('proxies', 'http', self.proxies.get('http', None)) + om.set_item('proxies', 'https', self.proxies.get('https', None)) + + for i in self._del_set: + if i == 'download_path': + om.set_item('paths', 'download_path', '') + elif i == 'proxies': + om.set_item('proxies', 'http', '') + om.set_item('proxies', 'https', '') + else: + om.remove_item('session_options', i) path = str(path) om.save(path) @@ -800,23 +839,14 @@ def session_options_to_dict(options): return options re_dict = dict() - attrs = ['headers', 'proxies', 'hooks', 'params', 'verify', 'stream', 'trust_env', - 'max_redirects', 'timeout'] # 'adapters', - - cookies = options.__getattribute__('_cookies') - - if cookies is not None: - re_dict['cookies'] = cookies_to_tuple(cookies) + attrs = ['headers', 'cookies', 'proxies', 'params', 'verify', 'stream', 'trust_env', + 'max_redirects', 'timeout', 'download_path'] for attr in attrs: val = options.__getattribute__(f'_{attr}') if val is not None: re_dict[attr] = val - # cert属性默认值为None，为免无法区分是否被设置，故主动赋值 - re_dict['cert'] = options.__getattribute__('_cert') - re_dict['auth'] = options.__getattribute__('_auth') - return re_dict diff --git a/DrissionPage/config.pyi b/DrissionPage/config.pyi index 253705e..341681c 100644 --- a/DrissionPage/config.pyi +++ b/DrissionPage/config.pyi @@ -6,8 +6,10 @@ from configparser import RawConfigParser from http.cookiejar import Cookie from pathlib import Path -from typing import Any, Union, List +from typing import Any, Union, List, Tuple +from requests.adapters import HTTPAdapter +from requests.auth import HTTPBasicAuth from requests.cookies import RequestsCookieJar from selenium.webdriver.chrome.options import Options @@ -27,7 +29,9 @@ class OptionsManager(object): def get_option(self, section: str) -> dict: ... - def set_item(self, section: str, item: str, value: Any) -> OptionsManager: ... + def set_item(self, section: str, item: str, value: Any) -> None: ... + + def remove_item(self, section: str, item: str) -> None: ... def save(self, path: str = None) -> str: ... @@ -46,65 +50,88 @@ class SessionOptions(object): self._params: dict = ... self._verify: bool = ... self._cert: Union[str, tuple] = ... - self._adapters: str = ... + self._adapters: list = ... self._stream: bool = ... self._trust_env: bool = ... self._max_redirects: int = ... self._timeout: float = ... - - @property - def timeout(self) -> Union[int, float]: ... + self._del_set: set = ... @property def download_path(self) -> str: ... - @property - def headers(self) -> dict: ... + def set_paths(self, download_path: Union[str, Path]) -> SessionOptions: ... @property - def cookies(self) -> list: ... - - @property - def auth(self) -> tuple: ... - - @property - def proxies(self) -> dict: ... - - @property - def hooks(self) -> dict: ... - - @property - def params(self) -> dict: ... - - @property - def verify(self) -> bool: ... - - @property - def cert(self) -> Union[str, tuple]: ... - - @property - def adapters(self): ... - - @property - def stream(self) -> bool: ... - - @property - def trust_env(self) -> bool: ... - - @property - def max_redirects(self) -> int: ... + def timeout(self) -> Union[int, float]: ... def set_timeout(self, second: Union[int, float]) -> SessionOptions: ... - def set_headers(self, headers: dict) -> SessionOptions: ... + @property + def headers(self) -> dict: ... + + def set_headers(self, headers: Union[dict, None]) -> SessionOptions: ... def set_a_header(self, attr: str, value: str) -> SessionOptions: ... def remove_a_header(self, attr: str) -> SessionOptions: ... - def set_proxies(self, proxies: dict) -> SessionOptions: ... + @property + def cookies(self) -> list: ... - def set_paths(self, download_path: Union[str, Path]) -> SessionOptions: ... + def set_cookies(self, cookies: Union[RequestsCookieJar, list, tuple, str, dict, None]) -> SessionOptions: ... + + @property + def auth(self) -> Union[Tuple[str, str], HTTPBasicAuth]: ... + + def set_auth(self, auth: Union[Tuple[str, str], HTTPBasicAuth, None]) -> SessionOptions: ... + + @property + def proxies(self) -> dict: ... + + def set_proxies(self, http: Union[str, None], https: Union[str, None] = None) -> SessionOptions: ... + + @property + def hooks(self) -> dict: ... + + def set_hooks(self, hooks: Union[dict, None]) -> SessionOptions: ... + + @property + def params(self) -> dict: ... + + def set_params(self, params: Union[dict, None]) -> SessionOptions: ... + + @property + def verify(self) -> bool: ... + + def set_verify(self, on_off: Union[bool, None]) -> SessionOptions: ... + + @property + def cert(self) -> Union[str, tuple]: ... + + def set_cert(self, cert: Union[str, Tuple[str, str], None]) -> SessionOptions: ... + + @property + def adapters(self): list: ... + + def add_adapter(self, url: str, adapter: HTTPAdapter) -> SessionOptions: ... + + @property + def stream(self) -> bool: ... + + def set_stream(self, on_off: Union[bool, None]) -> SessionOptions: ... + + @property + def trust_env(self) -> bool: ... + + def set_trust_env(self, on_off: Union[bool, None]) -> SessionOptions: ... + + @property + def max_redirects(self) -> int: ... + + def _sets(self, arg: str, val: Any) -> None: ... + + def set_max_redirects(self, times: Union[int, None]) -> SessionOptions: ... def save(self, path: str = None) -> str: ... diff --git a/DrissionPage/configs.ini b/DrissionPage/configs.ini index f29c6bf..1d0af9d 100644 --- a/DrissionPage/configs.ini +++ b/DrissionPage/configs.ini @@ -12,10 +12,13 @@ timeouts = {'implicit': 10.0, 'pageLoad': 30.0, 'script': 30.0} page_load_strategy = normal [session_options] -headers = { - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Connection": "keep-alive", - "Accept-Charset": "GB2312,utf-8;q=0.7,*;q=0.7" - } -timeout = 10 \ No newline at end of file +headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8', 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'connection': 'keep-alive', 'accept-charset': 'GB2312,utf-8;q=0.7,*;q=0.7'} + +[timeouts] +implicit = 10 +pageload = 30 +script = 30 + +[proxies] +http = +https = \ No newline at end of file diff --git a/DrissionPage/session_page.py b/DrissionPage/session_page.py index ab62187..02cfbb6 100644 --- a/DrissionPage/session_page.py +++ b/DrissionPage/session_page.py @@ -37,7 +37,7 @@ class SessionPage(BasePage): """ if Session_or_Options is None or isinstance(Session_or_Options, SessionOptions): options = Session_or_Options or SessionOptions() - self._set_session(options.as_dict()) + self._set_session(options) self.timeout = options.timeout self._download_path = options.download_path elif isinstance(Session_or_Options, Session): @@ -45,23 +45,27 @@ class SessionPage(BasePage): self._download_path = None self._download_kit = None - def _set_session(self, data): + def _set_session(self, opt): """根据传入字典对session进行设置 \n :param data: session配置字典 :return: None """ self._session = Session() - if 'headers' in data: - self._session.headers = CaseInsensitiveDict(data['headers']) - if 'cookies' in data: - self.set_cookies(data['cookies']) + if opt.headers: + self._session.headers = CaseInsensitiveDict(opt.headers) + if opt.cookies: + self.set_cookies(opt.cookies) + if opt.adapters: + for url, adapter in opt.adapters: + self._session.mount(url, adapter) attrs = ['auth', 'proxies', 'hooks', 'params', 'verify', - 'cert', 'stream', 'trust_env', 'max_redirects'] # , 'adapters' + 'cert', 'stream', 'trust_env', 'max_redirects'] for i in attrs: - if i in data: - self._session.__setattr__(i, data[i]) + attr = opt.__getattribute__(i) + if attr: + self._session.__setattr__(i, attr) def set_cookies(self, cookies): cookies = cookies_to_tuple(cookies) diff --git a/DrissionPage/session_page.pyi b/DrissionPage/session_page.pyi index 6c8a6d3..23d12ac 100644 --- a/DrissionPage/session_page.pyi +++ b/DrissionPage/session_page.pyi @@ -32,7 +32,7 @@ class SessionPage(BasePage): def _create_session(self, Session_or_Options: Union[Session, SessionOptions]) -> None: ... - def _set_session(self, data: dict) -> None: ... + def _set_session(self, opt: SessionOptions) -> None: ... def set_cookies(self, cookies: Union[RequestsCookieJar, list, tuple, str, dict]) -> None: ...