s模式返回空时重试;默认不启动下载管理功能;SessionOptions的set_paths()改成set_download_path()

This commit is contained in:
g1879 2023-11-20 22:35:42 +08:00
parent b3d1c54980
commit 18951def81
11 changed files with 62 additions and 35 deletions

View File

@ -4,7 +4,6 @@
@Contact : g1879@qq.com @Contact : g1879@qq.com
""" """
from abc import abstractmethod from abc import abstractmethod
from pathlib import Path
from re import sub from re import sub
from urllib.parse import quote from urllib.parse import quote
@ -378,7 +377,7 @@ class BasePage(BaseParser):
self.retry_times = 3 self.retry_times = 3
self.retry_interval = 2 self.retry_interval = 2
self._DownloadKit = None self._DownloadKit = None
self._download_path = str(Path('.').absolute()) self._download_path = None
@property @property
def title(self): def title(self):

View File

@ -30,7 +30,7 @@ class ChromiumOptions(object):
self.ini_path = om.ini_path self.ini_path = om.ini_path
options = om.chrome_options options = om.chrome_options
self._download_path = om.paths.get('download_path', '') self._download_path = om.paths.get('download_path', None) or None
self._arguments = options.get('arguments', []) self._arguments = options.get('arguments', [])
self._browser_path = options.get('browser_path', '') self._browser_path = options.get('browser_path', '')
self._extensions = options.get('extensions', []) self._extensions = options.get('extensions', [])
@ -68,7 +68,7 @@ class ChromiumOptions(object):
self.ini_path = None self.ini_path = None
self._browser_path = "chrome" self._browser_path = "chrome"
self._arguments = [] self._arguments = []
self._download_path = '' self._download_path = None
self._extensions = [] self._extensions = []
self._prefs = {} self._prefs = {}
self._flags = {} self._flags = {}

View File

@ -21,7 +21,7 @@ class SessionOptions(object):
:param ini_path: ini文件路径 :param ini_path: ini文件路径
""" """
self.ini_path = None self.ini_path = None
self._download_path = '' self._download_path = None
self._timeout = 10 self._timeout = 10
self._del_set = set() # 记录要从ini文件删除的参数 self._del_set = set() # 记录要从ini文件删除的参数
@ -75,7 +75,7 @@ class SessionOptions(object):
self.set_proxies(om.proxies.get('http', None), om.proxies.get('https', None)) self.set_proxies(om.proxies.get('http', None), om.proxies.get('https', None))
self._timeout = om.timeouts.get('implicit', 10) self._timeout = om.timeouts.get('implicit', 10)
self._download_path = om.paths.get('download_path', '') self._download_path = om.paths.get('download_path', None) or None
# ===========须独立处理的项开始============ # ===========须独立处理的项开始============
@property @property
@ -83,13 +83,12 @@ class SessionOptions(object):
"""返回默认下载路径属性信息""" """返回默认下载路径属性信息"""
return self._download_path return self._download_path
def set_paths(self, download_path=None): def set_download_path(self, path=None):
"""设置默认下载路径 """设置默认下载路径
:param download_path: 下载路径 :param path: 下载路径
:return: 返回当前对象 :return: 返回当前对象
""" """
if download_path is not None: self._download_path = str(path)
self._download_path = str(download_path)
return self return self
@property @property
@ -418,6 +417,17 @@ class SessionOptions(object):
self._max_redirects = session.max_redirects self._max_redirects = session.max_redirects
return self return self
# --------------即将废弃---------------
def set_paths(self, download_path=None):
"""设置默认下载路径
:param download_path: 下载路径
:return: 返回当前对象
"""
if download_path is not None:
self._download_path = str(download_path)
return self
def session_options_to_dict(options): def session_options_to_dict(options):
"""把session配置对象转换为字典 """把session配置对象转换为字典

View File

@ -34,7 +34,7 @@ class SessionOptions(object):
@property @property
def download_path(self) -> str: ... def download_path(self) -> str: ...
def set_paths(self, download_path: Union[str, Path]) -> SessionOptions: ... def set_download_path(self, path: Union[str, Path]) -> SessionOptions: ...
@property @property
def timeout(self) -> float: ... def timeout(self) -> float: ...

View File

@ -5,7 +5,6 @@
""" """
from json import loads, JSONDecodeError from json import loads, JSONDecodeError
from os.path import sep from os.path import sep
from pathlib import Path
from re import findall from re import findall
from threading import Thread from threading import Thread
from time import perf_counter, sleep from time import perf_counter, sleep
@ -55,22 +54,20 @@ class ChromiumBase(BasePage):
self._scroll = None self._scroll = None
self._upload_list = None self._upload_list = None
self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc self._doc_got = False # 用于在LoadEventFired和FrameStoppedLoading间标记是否已获取doc
self._download_path = None
self._download_path = str(Path('.').absolute())
if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): if isinstance(address, int) or (isinstance(address, str) and address.isdigit()):
address = f'127.0.0.1:{address}' address = f'127.0.0.1:{address}'
self._d_set_start_options(address, None) self._d_set_start_options(address)
self._d_set_runtime_settings() self._d_set_runtime_settings()
self._connect_browser(tab_id) self._connect_browser(tab_id)
if timeout is not None: if timeout is not None:
self.timeout = timeout self.timeout = timeout
def _d_set_start_options(self, address, none): def _d_set_start_options(self, address):
"""设置浏览器启动属性 """设置浏览器启动属性
:param address: 'ip:port' :param address: 'ip:port'
:param none: 用于后代继承
:return: None :return: None
""" """
self.address = address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://') self.address = address.replace('localhost', '127.0.0.1').lstrip('http://').lstrip('https://')

View File

@ -83,7 +83,7 @@ class ChromiumBase(BasePage):
def _wait_to_stop(self): ... def _wait_to_stop(self): ...
def _d_set_start_options(self, address, none) -> None: ... def _d_set_start_options(self, address) -> None: ...
def _d_set_runtime_settings(self) -> None: ... def _d_set_runtime_settings(self) -> None: ...

View File

@ -87,11 +87,11 @@ class ChromiumPage(ChromiumBase):
if self._driver_options.timeouts['implicit'] is not None: if self._driver_options.timeouts['implicit'] is not None:
self._timeout = self._driver_options.timeouts['implicit'] self._timeout = self._driver_options.timeouts['implicit']
self._load_mode = self._driver_options.load_mode self._load_mode = self._driver_options.load_mode
self._download_path = str(Path(self._driver_options.download_path).absolute()) self._download_path = None if self._driver_options.download_path is None \
else str(Path(self._driver_options.download_path).absolute())
def _page_init(self): def _page_init(self):
"""浏览器相关设置""" """浏览器相关设置"""
self._rect = None
self._browser.connect_to_page() self._browser.connect_to_page()
# ----------挂件---------- # ----------挂件----------

View File

@ -3,6 +3,7 @@
@Author : g1879 @Author : g1879
@Contact : g1879@qq.com @Contact : g1879@qq.com
""" """
from pathlib import Path
from re import search from re import search
from time import sleep from time import sleep
from urllib.parse import urlparse from urllib.parse import urlparse
@ -51,7 +52,8 @@ class SessionPage(BasePage):
def _s_set_runtime_settings(self): def _s_set_runtime_settings(self):
"""设置运行时用到的属性""" """设置运行时用到的属性"""
self._timeout = self._session_options.timeout self._timeout = self._session_options.timeout
self._download_path = self._session_options.download_path self._download_path = None if self._session_options.download_path is None \
else str(Path(self._session_options.download_path).absolute())
def _create_session(self): def _create_session(self):
"""创建内建Session对象""" """创建内建Session对象"""
@ -277,7 +279,7 @@ class SessionPage(BasePage):
elif mode == 'post': elif mode == 'post':
r = self.session.post(url, data=data, **kwargs) r = self.session.post(url, data=data, **kwargs)
if r: if r and r.content:
return set_charset(r), 'Success' return set_charset(r), 'Success'
except Exception as e: except Exception as e:

View File

@ -27,10 +27,17 @@ class DownloadManager(object):
self._tab_missions = {} # {tab_id: DownloadMission} self._tab_missions = {} # {tab_id: DownloadMission}
self._flags = {} # {tab_id: [bool, DownloadMission]} self._flags = {} # {tab_id: [bool, DownloadMission]}
self._browser.driver.set_callback('Browser.downloadProgress', self._onDownloadProgress) if self._page.download_path:
self._browser.driver.set_callback('Browser.downloadWillBegin', self._onDownloadWillBegin) self._browser.driver.set_callback('Browser.downloadProgress', self._onDownloadProgress)
self._browser.run_cdp('Browser.setDownloadBehavior', downloadPath=self._page.download_path, self._browser.driver.set_callback('Browser.downloadWillBegin', self._onDownloadWillBegin)
behavior='allowAndName', eventsEnabled=True) r = self._browser.run_cdp('Browser.setDownloadBehavior', downloadPath=self._page.download_path,
behavior='allowAndName', eventsEnabled=True)
if 'error' in r:
print('浏览器版本太低无法使用下载管理功能。')
self._running = True
else:
self._running = False
@property @property
def missions(self): def missions(self):
@ -40,13 +47,18 @@ class DownloadManager(object):
def set_path(self, tab_id, path): def set_path(self, tab_id, path):
"""设置某个tab的下载路径 """设置某个tab的下载路径
:param tab_id: tab id :param tab_id: tab id
:param path: 下载路径 :param path: 下载路径绝对路径str
:return: None :return: None
""" """
TabDownloadSettings(tab_id).path = str(Path(path).absolute()) TabDownloadSettings(tab_id).path = path
if tab_id == self._page.tab_id: if tab_id == self._page.tab_id or not self._running:
self._browser.run_cdp('Browser.setDownloadBehavior', downloadPath=str(Path(path).absolute()), self._browser.driver.set_callback('Browser.downloadProgress', self._onDownloadProgress)
behavior='allowAndName', eventsEnabled=True) self._browser.driver.set_callback('Browser.downloadWillBegin', self._onDownloadWillBegin)
r = self._browser.run_cdp('Browser.setDownloadBehavior', downloadPath=path,
behavior='allowAndName', eventsEnabled=True)
if 'error' in r:
print('浏览器版本太低无法使用下载管理功能。')
self._running = True
def set_rename(self, tab_id, rename=None, suffix=None): def set_rename(self, tab_id, rename=None, suffix=None):
"""设置某个tab的重命名文件名 """设置某个tab的重命名文件名

View File

@ -1,4 +1,8 @@
from pathlib import Path # -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
"""
from typing import Dict, Optional, Union, Literal from typing import Dict, Optional, Union, Literal
from .._base.browser import Browser from .._base.browser import Browser
@ -11,13 +15,14 @@ class DownloadManager(object):
_missions: Dict[str, DownloadMission] = ... _missions: Dict[str, DownloadMission] = ...
_tab_missions: dict = ... _tab_missions: dict = ...
_flags: dict = ... _flags: dict = ...
_running: bool = ...
def __init__(self, browser: Browser): ... def __init__(self, browser: Browser): ...
@property @property
def missions(self) -> Dict[str, DownloadMission]: ... def missions(self) -> Dict[str, DownloadMission]: ...
def set_path(self, tab_id: str, path: Union[Path, str]) -> None: ... def set_path(self, tab_id: str, path: str) -> None: ...
def set_rename(self, tab_id: str, rename: str = None, suffix: str = None) -> None: ... def set_rename(self, tab_id: str, rename: str = None, suffix: str = None) -> None: ...

View File

@ -139,7 +139,8 @@ class TabSetter(ChromiumBaseSetter):
:param path: 下载路径 :param path: 下载路径
:return: None :return: None
""" """
self._page._download_path = str(Path(path).absolute()) path = str(Path(path).absolute())
self._page._download_path = path
self._page.browser._dl_mgr.set_path(self._page.tab_id, path) self._page.browser._dl_mgr.set_path(self._page.tab_id, path)
if self._page._DownloadKit: if self._page._DownloadKit:
self._page._DownloadKit.set.goal_path(path) self._page._DownloadKit.set.goal_path(path)
@ -209,7 +210,8 @@ class SessionPageSetter(object):
:param path: 下载路径 :param path: 下载路径
:return: None :return: None
""" """
self._page._download_path = str(Path(path).absolute()) path = str(Path(path).absolute())
self._page._download_path = path
if self._page._DownloadKit: if self._page._DownloadKit:
self._page._DownloadKit.set.goal_path(path) self._page._DownloadKit.set.goal_path(path)