调整项目文件目录

This commit is contained in:
g1879 2023-10-20 14:35:44 +08:00
parent 19f99b4d62
commit aafbc7a839
56 changed files with 541 additions and 515 deletions

View File

@ -4,10 +4,10 @@
@Contact : g1879@qq.com @Contact : g1879@qq.com
""" """
# 常用页面类 # 常用页面类
from .chromium_page import ChromiumPage from ._pages.chromium_page import ChromiumPage
from .session_page import SessionPage from ._pages.session_page import SessionPage
from .web_page import WebPage from ._pages.web_page import WebPage
# 启动配置类 # 启动配置类
from .configs.chromium_options import ChromiumOptions from ._configs.chromium_options import ChromiumOptions
from .configs.session_options import SessionOptions from ._configs.session_options import SessionOptions

View File

@ -10,10 +10,10 @@ from urllib.parse import quote
from DownloadKit import DownloadKit from DownloadKit import DownloadKit
from .commons.constants import Settings, NoneElement from DrissionPage._commons.constants import Settings, NoneElement
from .commons.locator import get_loc from DrissionPage._commons.locator import get_loc
from .commons.web import format_html from DrissionPage._commons.web import format_html
from .errors import ElementNotFoundError from DrissionPage.errors import ElementNotFoundError
class BaseParser(object): class BaseParser(object):
@ -367,7 +367,7 @@ class BasePage(BaseParser):
self.retry_times = 3 self.retry_times = 3
self.retry_interval = 2 self.retry_interval = 2
self._DownloadKit = None self._DownloadKit = None
self._download_path = str(Path('.').absolute()) self._download_path = str(Path('../..').absolute())
@property @property
def title(self): def title(self):

View File

@ -8,7 +8,7 @@ from typing import Union, Tuple, List
from DownloadKit import DownloadKit from DownloadKit import DownloadKit
from .commons.constants import NoneElement from DrissionPage._commons.constants import NoneElement
class BaseParser(object): class BaseParser(object):

View File

@ -1,7 +1,11 @@
# -*- coding:utf-8 -*- # -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
"""
from time import sleep from time import sleep
from .browser_download_manager import BrowserDownloadManager from DrissionPage._units.browser_download_manager import BrowserDownloadManager
from .chromium_driver import BrowserDriver from .chromium_driver import BrowserDriver

View File

@ -1,8 +1,12 @@
# -*- coding:utf-8 -*- # -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
"""
from typing import List, Optional, Union from typing import List, Optional, Union
from .browser_download_manager import BrowserDownloadManager from DrissionPage._units.browser_download_manager import BrowserDownloadManager
from .chromium_page import ChromiumPage from DrissionPage._pages.chromium_page import ChromiumPage
from .chromium_driver import BrowserDriver from .chromium_driver import BrowserDriver

View File

@ -3,7 +3,7 @@
@Author : g1879 @Author : g1879
@Contact : g1879@qq.com @Contact : g1879@qq.com
""" """
from DrissionPage.configs.chromium_options import ChromiumOptions from DrissionPage._configs.chromium_options import ChromiumOptions
def connect_browser(option: ChromiumOptions) -> tuple: ... def connect_browser(option: ChromiumOptions) -> tuple: ...

View File

@ -1,6 +1,11 @@
# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
"""
from click import command, option from click import command, option
from DrissionPage import ChromiumPage from DrissionPage._pages.chromium_page import ChromiumPage
from DrissionPage.easy_set import set_paths, configs_to_here as ch from DrissionPage.easy_set import set_paths, configs_to_here as ch

View File

@ -8,7 +8,7 @@ from pathlib import Path
from typing import Union from typing import Union
from types import FunctionType from types import FunctionType
from chromium_page import ChromiumPage from DrissionPage._pages.chromium_page import ChromiumPage
def get_usable_path(path: Union[str, Path], is_file: bool = True, parents: bool = True) -> Path: ... def get_usable_path(path: Union[str, Path], is_file: bool = True, parents: bool = True) -> Path: ...

View File

@ -10,8 +10,8 @@ from requests import Session
from requests.cookies import RequestsCookieJar from requests.cookies import RequestsCookieJar
from DrissionPage.base import DrissionElement, BasePage from DrissionPage.base import DrissionElement, BasePage
from DrissionPage.chromium_element import ChromiumElement from DrissionPage._chromium_element import ChromiumElement
from DrissionPage.chromium_base import ChromiumBase from DrissionPage._chromium_base import ChromiumBase
def get_ele_txt(e: DrissionElement) -> str: ... def get_ele_txt(e: DrissionElement) -> str: ...

View File

@ -6,7 +6,7 @@
from pathlib import Path from pathlib import Path
from tempfile import gettempdir, TemporaryDirectory from tempfile import gettempdir, TemporaryDirectory
from DrissionPage.commons.tools import port_is_using, clean_folder from DrissionPage._commons.tools import port_is_using, clean_folder
from .options_manage import OptionsManager from .options_manage import OptionsManager

View File

@ -8,7 +8,7 @@ from pathlib import Path
from requests import Session from requests import Session
from requests.structures import CaseInsensitiveDict from requests.structures import CaseInsensitiveDict
from DrissionPage.commons.web import cookies_to_tuple, set_session_cookies from DrissionPage._commons.web import cookies_to_tuple, set_session_cookies
from .options_manage import OptionsManager from .options_manage import OptionsManager

View File

@ -7,17 +7,17 @@ from os.path import basename, sep
from pathlib import Path from pathlib import Path
from time import perf_counter, sleep from time import perf_counter, sleep
from .base import DrissionElement, BaseElement from DrissionPage._base.base import DrissionElement, BaseElement
from .commons.constants import FRAME_ELEMENT, NoneElement, Settings from DrissionPage._commons.constants import FRAME_ELEMENT, NoneElement, Settings
from .commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions from DrissionPage._commons.keys import keys_to_typing, keyDescriptionForString, keyDefinitions
from .commons.locator import get_loc from DrissionPage._commons.locator import get_loc
from .commons.tools import get_usable_path from DrissionPage._commons.tools import get_usable_path
from .commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, location_in_viewport, offset_scroll from DrissionPage._commons.web import make_absolute_link, get_ele_txt, format_html, is_js_func, location_in_viewport, offset_scroll
from .errors import ContextLossError, ElementLossError, JavaScriptError, NoRectError, ElementNotFoundError, \ from DrissionPage.errors import ContextLossError, ElementLossError, JavaScriptError, NoRectError, ElementNotFoundError, \
CDPError, NoResourceError, CanNotClickError CDPError, NoResourceError, CanNotClickError
from .session_element import make_session_ele from .session_element import make_session_ele
from .setter import ChromiumElementSetter from DrissionPage._units.setter import ChromiumElementSetter
from .waiter import ChromiumElementWaiter from DrissionPage._units.waiter import ChromiumElementWaiter
class ChromiumElement(DrissionElement): class ChromiumElement(DrissionElement):
@ -1203,7 +1203,7 @@ def make_chromium_ele(page, node_id=None, obj_id=None):
ele = ChromiumElement(page, obj_id=obj_id, node_id=node_id, backend_id=backend_id) ele = ChromiumElement(page, obj_id=obj_id, node_id=node_id, backend_id=backend_id)
if ele.tag in FRAME_ELEMENT: if ele.tag in FRAME_ELEMENT:
from .chromium_frame import ChromiumFrame from ._chromium_frame import ChromiumFrame
ele = ChromiumFrame(page, ele) ele = ChromiumFrame(page, ele)
return ele return ele

View File

@ -6,15 +6,15 @@
from pathlib import Path from pathlib import Path
from typing import Union, Tuple, List, Any from typing import Union, Tuple, List, Any
from .base import DrissionElement, BaseElement from DrissionPage._base.base import DrissionElement, BaseElement
from .chromium_base import ChromiumBase from DrissionPage._commons.constants import NoneElement
from .chromium_frame import ChromiumFrame from DrissionPage._elements.session_element import SessionElement
from .chromium_page import ChromiumPage from DrissionPage._pages.chromium_base import ChromiumBase
from .commons.constants import NoneElement from DrissionPage._pages.chromium_frame import ChromiumFrame
from .session_element import SessionElement from DrissionPage._pages.chromium_page import ChromiumPage
from .setter import ChromiumElementSetter from DrissionPage._pages.web_page import WebPage
from .waiter import ChromiumElementWaiter from DrissionPage._units.setter import ChromiumElementSetter
from .web_page import WebPage from DrissionPage._units.waiter import ChromiumElementWaiter
class ChromiumElement(DrissionElement): class ChromiumElement(DrissionElement):

View File

@ -9,10 +9,10 @@ from re import match, DOTALL
from lxml.etree import tostring from lxml.etree import tostring
from lxml.html import HtmlElement, fromstring from lxml.html import HtmlElement, fromstring
from .base import DrissionElement, BasePage, BaseElement from DrissionPage._base.base import DrissionElement, BasePage, BaseElement
from .commons.constants import NoneElement from DrissionPage._commons.constants import NoneElement
from .commons.locator import get_loc from DrissionPage._commons.locator import get_loc
from .commons.web import get_ele_txt, make_absolute_link from DrissionPage._commons.web import get_ele_txt, make_absolute_link
class SessionElement(DrissionElement): class SessionElement(DrissionElement):

View File

@ -7,12 +7,12 @@ from typing import Union, List, Tuple
from lxml.html import HtmlElement from lxml.html import HtmlElement
from .base import DrissionElement, BaseElement from DrissionPage._base.base import DrissionElement, BaseElement
from .chromium_base import ChromiumBase from DrissionPage._commons.constants import NoneElement
from .chromium_element import ChromiumElement from DrissionPage._elements.chromium_element import ChromiumElement
from .chromium_frame import ChromiumFrame from DrissionPage._pages.chromium_base import ChromiumBase
from .commons.constants import NoneElement from DrissionPage._pages.chromium_frame import ChromiumFrame
from .session_page import SessionPage from DrissionPage._pages.session_page import SessionPage
class SessionElement(DrissionElement): class SessionElement(DrissionElement):

View File

@ -13,20 +13,20 @@ from time import perf_counter, sleep, time
from requests import get from requests import get
from .action_chains import ActionChains from DrissionPage._base.base import BasePage
from .base import BasePage from DrissionPage._base.chromium_driver import ChromiumDriver
from .chromium_driver import ChromiumDriver from DrissionPage._commons.constants import HANDLE_ALERT_METHOD, ERROR, NoneElement
from .chromium_element import ChromiumScroll, ChromiumElement, run_js, make_chromium_ele from DrissionPage._commons.locator import get_loc
from .commons.constants import HANDLE_ALERT_METHOD, ERROR, NoneElement from DrissionPage._commons.tools import get_usable_path, clean_folder
from .commons.locator import get_loc from DrissionPage._commons.web import location_in_viewport
from .commons.tools import get_usable_path, clean_folder from DrissionPage._elements.chromium_element import ChromiumScroll, ChromiumElement, run_js, make_chromium_ele
from .commons.web import location_in_viewport from DrissionPage._elements.session_element import make_session_ele
from .errors import ContextLossError, ElementLossError, AlertExistsError, CDPError, TabClosedError, \ from DrissionPage._units.network_listener import NetworkListener
from DrissionPage._units.setter import ChromiumBaseSetter
from DrissionPage._units.waiter import ChromiumBaseWaiter
from DrissionPage.errors import ContextLossError, ElementLossError, AlertExistsError, CDPError, TabClosedError, \
NoRectError, BrowserConnectError, GetDocumentError NoRectError, BrowserConnectError, GetDocumentError
from .network_listener import NetworkListener from _units.action_chains import ActionChains
from .session_element import make_session_ele
from .setter import ChromiumBaseSetter
from .waiter import ChromiumBaseWaiter
class ChromiumBase(BasePage): class ChromiumBase(BasePage):
@ -48,7 +48,7 @@ class ChromiumBase(BasePage):
self._actions = None self._actions = None
self._listener = None self._listener = None
self._download_path = str(Path('.').absolute()) self._download_path = str(Path('../..').absolute())
if isinstance(address, int) or (isinstance(address, str) and address.isdigit()): if isinstance(address, int) or (isinstance(address, str) and address.isdigit()):
address = f'127.0.0.1:{address}' address = f'127.0.0.1:{address}'

View File

@ -8,18 +8,18 @@ from typing import Union, Tuple, List, Any
from DataRecorder import Recorder from DataRecorder import Recorder
from .browser import Browser from DrissionPage._base.base import BasePage
from .action_chains import ActionChains from DrissionPage._base.browser import Browser
from .base import BasePage from DrissionPage._base.chromium_driver import ChromiumDriver
from .chromium_driver import ChromiumDriver from DrissionPage._commons.constants import NoneElement
from .chromium_element import ChromiumElement, ChromiumScroll from DrissionPage._elements.chromium_element import ChromiumElement, ChromiumScroll
from .chromium_frame import ChromiumFrame from DrissionPage._elements.session_element import SessionElement
from .chromium_page import ChromiumPage from DrissionPage._pages.chromium_frame import ChromiumFrame
from .commons.constants import NoneElement from DrissionPage._pages.chromium_page import ChromiumPage
from .network_listener import NetworkListener from DrissionPage._units.action_chains import ActionChains
from .session_element import SessionElement from DrissionPage._units.network_listener import NetworkListener
from .setter import ChromiumBaseSetter from DrissionPage._units.setter import ChromiumBaseSetter
from .waiter import ChromiumBaseWaiter from DrissionPage._units.waiter import ChromiumBaseWaiter
class ChromiumBase(BasePage): class ChromiumBase(BasePage):

View File

@ -10,11 +10,11 @@ from time import sleep, perf_counter
from requests import get from requests import get
from .chromium_base import ChromiumBase, ChromiumPageScroll from DrissionPage._elements.chromium_element import ChromiumElement
from .chromium_element import ChromiumElement from DrissionPage._pages.chromium_base import ChromiumBase, ChromiumPageScroll
from .errors import ContextLossError from DrissionPage._units.setter import ChromiumFrameSetter
from .setter import ChromiumFrameSetter from DrissionPage._units.waiter import FrameWaiter
from .waiter import FrameWaiter from DrissionPage.errors import ContextLossError
class ChromiumFrame(ChromiumBase): class ChromiumFrame(ChromiumBase):

View File

@ -6,12 +6,13 @@
from pathlib import Path from pathlib import Path
from typing import Union, Tuple, List, Any from typing import Union, Tuple, List, Any
from DrissionPage import ChromiumPage, WebPage from DrissionPage._elements.chromium_element import ChromiumElement, Locations, ChromiumElementStates
from .chromium_tab import ChromiumTab from DrissionPage._pages.chromium_base import ChromiumBase, ChromiumPageScroll
from .chromium_base import ChromiumBase, ChromiumPageScroll from DrissionPage._pages.chromium_page import ChromiumPage
from .chromium_element import ChromiumElement, Locations, ChromiumElementStates from DrissionPage._pages.chromium_tab import ChromiumTab
from .setter import ChromiumFrameSetter from DrissionPage._pages.web_page import WebPage
from .waiter import FrameWaiter from DrissionPage._units.setter import ChromiumFrameSetter
from DrissionPage._units.waiter import FrameWaiter
class ChromiumFrame(ChromiumBase): class ChromiumFrame(ChromiumBase):

View File

@ -8,14 +8,14 @@ from time import perf_counter, sleep
from requests import get from requests import get
from .browser import Browser from DrissionPage._base.browser import Browser
from .chromium_base import ChromiumBase, Timeout from DrissionPage._base.chromium_driver import ChromiumDriver
from .chromium_driver import ChromiumDriver from DrissionPage._commons.browser import connect_browser
from .chromium_tab import ChromiumTab from DrissionPage._configs.chromium_options import ChromiumOptions
from .commons.browser import connect_browser from DrissionPage._pages.chromium_base import ChromiumBase, Timeout
from .configs.chromium_options import ChromiumOptions from DrissionPage._pages.chromium_tab import ChromiumTab
from .setter import ChromiumPageSetter from DrissionPage._units.setter import ChromiumPageSetter
from .waiter import ChromiumPageWaiter from DrissionPage._units.waiter import ChromiumPageWaiter
class ChromiumPage(ChromiumBase): class ChromiumPage(ChromiumBase):

View File

@ -5,13 +5,13 @@
""" """
from typing import Union, Tuple, List, Optional from typing import Union, Tuple, List, Optional
from .browser import Browser from DrissionPage._base.browser import Browser
from .chromium_base import ChromiumBase from DrissionPage._pages.chromium_base import ChromiumBase
from .chromium_driver import ChromiumDriver from DrissionPage._base.chromium_driver import ChromiumDriver
from .chromium_tab import ChromiumTab from DrissionPage._pages.chromium_tab import ChromiumTab
from .configs.chromium_options import ChromiumOptions from DrissionPage._configs.chromium_options import ChromiumOptions
from .setter import ChromiumPageSetter from DrissionPage._units.setter import ChromiumPageSetter
from .waiter import ChromiumPageWaiter from DrissionPage._units.waiter import ChromiumPageWaiter
class ChromiumPage(ChromiumBase): class ChromiumPage(ChromiumBase):

View File

@ -5,13 +5,12 @@
""" """
from copy import copy from copy import copy
from .base import BasePage from DrissionPage._base.base import BasePage
from .chromium_base import ChromiumBase from DrissionPage._commons.web import set_session_cookies, set_browser_cookies
from .commons.web import set_session_cookies, set_browser_cookies from DrissionPage._pages.chromium_base import ChromiumBase
from .session_page import SessionPage from DrissionPage._pages.session_page import SessionPage
from .setter import TabSetter from DrissionPage._units.setter import TabSetter, WebPageTabSetter
from .setter import WebPageTabSetter from DrissionPage._units.waiter import ChromiumTabWaiter
from .waiter import ChromiumTabWaiter
class ChromiumTab(ChromiumBase): class ChromiumTab(ChromiumBase):

View File

@ -7,17 +7,16 @@ from typing import Union, Tuple, Any, List
from requests import Session, Response from requests import Session, Response
from .browser import Browser from DrissionPage._base.browser import Browser
from .chromium_base import ChromiumBase from DrissionPage._elements.chromium_element import ChromiumElement
from .chromium_element import ChromiumElement from DrissionPage._elements.session_element import SessionElement
from .chromium_frame import ChromiumFrame from DrissionPage._pages.chromium_base import ChromiumBase
from .chromium_page import ChromiumPage, ChromiumTabRect from DrissionPage._pages.chromium_frame import ChromiumFrame
from .session_element import SessionElement from DrissionPage._pages.chromium_page import ChromiumPage, ChromiumTabRect
from .session_page import SessionPage from DrissionPage._pages.session_page import SessionPage
from .setter import TabSetter from DrissionPage._pages.web_page import WebPage
from .setter import WebPageTabSetter from DrissionPage._units.setter import TabSetter, WebPageTabSetter
from .waiter import ChromiumTabWaiter from DrissionPage._units.waiter import ChromiumTabWaiter
from .web_page import WebPage
class ChromiumTab(ChromiumBase): class ChromiumTab(ChromiumBase):

View File

@ -1,336 +1,336 @@
# -*- coding:utf-8 -*- # -*- coding:utf-8 -*-
""" """
@Author : g1879 @Author : g1879
@Contact : g1879@qq.com @Contact : g1879@qq.com
""" """
from re import search from re import search
from time import sleep from time import sleep
from urllib.parse import urlparse from urllib.parse import urlparse
from requests import Session from requests import Session
from requests.structures import CaseInsensitiveDict from requests.structures import CaseInsensitiveDict
from tldextract import extract from tldextract import extract
from .base import BasePage from DrissionPage._base.base import BasePage
from .commons.web import cookie_to_dict from DrissionPage._commons.web import cookie_to_dict
from .configs.session_options import SessionOptions from DrissionPage._configs.session_options import SessionOptions
from .session_element import SessionElement, make_session_ele from DrissionPage._elements.session_element import SessionElement, make_session_ele
from .setter import SessionPageSetter from DrissionPage._units.setter import SessionPageSetter
class SessionPage(BasePage): class SessionPage(BasePage):
"""SessionPage封装了页面操作的常用功能使用requests来获取、解析网页""" """SessionPage封装了页面操作的常用功能使用requests来获取、解析网页"""
def __init__(self, session_or_options=None, timeout=None): def __init__(self, session_or_options=None, timeout=None):
""" """
:param session_or_options: Session对象或SessionOptions对象 :param session_or_options: Session对象或SessionOptions对象
:param timeout: 连接超时时间为None时从ini文件读取 :param timeout: 连接超时时间为None时从ini文件读取
""" """
super(SessionPage, SessionPage).__init__(self) super(SessionPage, SessionPage).__init__(self)
self._response = None self._response = None
self._session = None self._session = None
self._set = None self._set = None
self._s_set_start_options(session_or_options, None) self._s_set_start_options(session_or_options, None)
self._s_set_runtime_settings() self._s_set_runtime_settings()
self._create_session() self._create_session()
if timeout is not None: if timeout is not None:
self.timeout = timeout self.timeout = timeout
def _s_set_start_options(self, session_or_options, none): def _s_set_start_options(self, session_or_options, none):
"""启动配置 """启动配置
:param session_or_options: SessionSessionOptions :param session_or_options: SessionSessionOptions
:param none: 用于后代继承 :param none: 用于后代继承
:return: None :return: None
""" """
if not session_or_options or isinstance(session_or_options, SessionOptions): if not session_or_options or isinstance(session_or_options, SessionOptions):
self._session_options = session_or_options or SessionOptions(session_or_options) self._session_options = session_or_options or SessionOptions(session_or_options)
elif isinstance(session_or_options, Session): elif isinstance(session_or_options, Session):
self._session_options = SessionOptions() self._session_options = SessionOptions()
self._session = session_or_options self._session = session_or_options
def _s_set_runtime_settings(self): def _s_set_runtime_settings(self):
"""设置运行时用到的属性""" """设置运行时用到的属性"""
self._timeout = self._session_options.timeout self._timeout = self._session_options.timeout
self._download_path = self._session_options.download_path self._download_path = self._session_options.download_path
def _create_session(self): def _create_session(self):
"""创建内建Session对象""" """创建内建Session对象"""
if not self._session: if not self._session:
self._session = self._session_options.make_session() self._session = self._session_options.make_session()
def __call__(self, loc_or_str, timeout=None): def __call__(self, loc_or_str, timeout=None):
"""在内部查找元素 """在内部查找元素
ele2 = ele1('@id=ele_id') ele2 = ele1('@id=ele_id')
:param loc_or_str: 元素的定位信息可以是loc元组或查询字符串 :param loc_or_str: 元素的定位信息可以是loc元组或查询字符串
:param timeout: 不起实际作用用于和ChromiumElement对应便于无差别调用 :param timeout: 不起实际作用用于和ChromiumElement对应便于无差别调用
:return: SessionElement对象或属性文本 :return: SessionElement对象或属性文本
""" """
return self.ele(loc_or_str) return self.ele(loc_or_str)
# -----------------共有属性和方法------------------- # -----------------共有属性和方法-------------------
@property @property
def title(self): def title(self):
"""返回网页title""" """返回网页title"""
ele = self._ele('xpath://title', raise_err=False) ele = self._ele('xpath://title', raise_err=False)
return ele.text if ele else None return ele.text if ele else None
@property @property
def url(self): def url(self):
"""返回当前访问url""" """返回当前访问url"""
return self._url return self._url
@property @property
def _session_url(self): def _session_url(self):
"""返回当前访问url""" """返回当前访问url"""
return self._url return self._url
@property @property
def html(self): def html(self):
"""返回页面的html文本""" """返回页面的html文本"""
return self.response.text if self.response else '' return self.response.text if self.response else ''
@property @property
def json(self): def json(self):
"""当返回内容是json格式时返回对应的字典非json格式时返回None""" """当返回内容是json格式时返回对应的字典非json格式时返回None"""
try: try:
return self.response.json() return self.response.json()
except Exception: except Exception:
return None return None
@property @property
def user_agent(self): def user_agent(self):
"""返回user agent""" """返回user agent"""
return self.session.headers.get('user-agent', '') return self.session.headers.get('user-agent', '')
@property @property
def session(self): def session(self):
"""返回session对象""" """返回session对象"""
return self._session return self._session
@property @property
def response(self): def response(self):
"""返回访问url得到的response对象""" """返回访问url得到的response对象"""
return self._response return self._response
@property @property
def set(self): def set(self):
"""返回用于等待的对象""" """返回用于等待的对象"""
if self._set is None: if self._set is None:
self._set = SessionPageSetter(self) self._set = SessionPageSetter(self)
return self._set return self._set
def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None, **kwargs): def get(self, url, show_errmsg=False, retry=None, interval=None, timeout=None, **kwargs):
"""用get方式跳转到url """用get方式跳转到url
:param url: 目标url :param url: 目标url
:param show_errmsg: 是否显示和抛出异常 :param show_errmsg: 是否显示和抛出异常
:param retry: 重试次数 :param retry: 重试次数
:param interval: 重试间隔 :param interval: 重试间隔
:param timeout: 连接超时时间 :param timeout: 连接超时时间
:param kwargs: 连接参数 :param kwargs: 连接参数
:return: url是否可用 :return: url是否可用
""" """
return self._s_connect(url, 'get', None, show_errmsg, retry, interval, **kwargs) return self._s_connect(url, 'get', None, show_errmsg, retry, interval, **kwargs)
def ele(self, loc_or_ele, timeout=None): def ele(self, loc_or_ele, timeout=None):
"""返回页面中符合条件的第一个元素、属性或节点文本 """返回页面中符合条件的第一个元素、属性或节点文本
:param loc_or_ele: 元素的定位信息可以是元素对象loc元组或查询字符串 :param loc_or_ele: 元素的定位信息可以是元素对象loc元组或查询字符串
:param timeout: 不起实际作用用于和ChromiumElement对应便于无差别调用 :param timeout: 不起实际作用用于和ChromiumElement对应便于无差别调用
:return: SessionElement对象或属性文本 :return: SessionElement对象或属性文本
""" """
return self._ele(loc_or_ele) return self._ele(loc_or_ele)
def eles(self, loc_or_str, timeout=None): def eles(self, loc_or_str, timeout=None):
"""返回页面中所有符合条件的元素、属性或节点文本 """返回页面中所有符合条件的元素、属性或节点文本
:param loc_or_str: 元素的定位信息可以是loc元组或查询字符串 :param loc_or_str: 元素的定位信息可以是loc元组或查询字符串
:param timeout: 不起实际作用用于和ChromiumElement对应便于无差别调用 :param timeout: 不起实际作用用于和ChromiumElement对应便于无差别调用
:return: SessionElement对象或属性文本组成的列表 :return: SessionElement对象或属性文本组成的列表
""" """
return self._ele(loc_or_str, single=False) return self._ele(loc_or_str, single=False)
def s_ele(self, loc_or_ele=None): def s_ele(self, loc_or_ele=None):
"""返回页面中符合条件的第一个元素、属性或节点文本 """返回页面中符合条件的第一个元素、属性或节点文本
:param loc_or_ele: 元素的定位信息可以是元素对象loc元组或查询字符串 :param loc_or_ele: 元素的定位信息可以是元素对象loc元组或查询字符串
:return: SessionElement对象或属性文本 :return: SessionElement对象或属性文本
""" """
return make_session_ele(self.html) if loc_or_ele is None else self._ele(loc_or_ele) return make_session_ele(self.html) if loc_or_ele is None else self._ele(loc_or_ele)
def s_eles(self, loc_or_str): def s_eles(self, loc_or_str):
"""返回页面中符合条件的所有元素、属性或节点文本 """返回页面中符合条件的所有元素、属性或节点文本
:param loc_or_str: 元素的定位信息可以是元素对象loc元组或查询字符串 :param loc_or_str: 元素的定位信息可以是元素对象loc元组或查询字符串
:return: SessionElement对象或属性文本 :return: SessionElement对象或属性文本
""" """
return self._ele(loc_or_str, single=False) return self._ele(loc_or_str, single=False)
def _find_elements(self, loc_or_ele, timeout=None, single=True, raise_err=None): def _find_elements(self, loc_or_ele, timeout=None, single=True, raise_err=None):
"""返回页面中符合条件的元素、属性或节点文本,默认返回第一个 """返回页面中符合条件的元素、属性或节点文本,默认返回第一个
:param loc_or_ele: 元素的定位信息可以是元素对象loc元组或查询字符串 :param loc_or_ele: 元素的定位信息可以是元素对象loc元组或查询字符串
:param timeout: 不起实际作用用于和父类对应 :param timeout: 不起实际作用用于和父类对应
:param single: True则返回第一个False则返回全部 :param single: True则返回第一个False则返回全部
:param raise_err: 找不到元素是是否抛出异常为None时根据全局设置 :param raise_err: 找不到元素是是否抛出异常为None时根据全局设置
:return: SessionElement对象 :return: SessionElement对象
""" """
return loc_or_ele if isinstance(loc_or_ele, SessionElement) else make_session_ele(self, loc_or_ele, single) return loc_or_ele if isinstance(loc_or_ele, SessionElement) else make_session_ele(self, loc_or_ele, single)
def get_cookies(self, as_dict=False, all_domains=False, all_info=False): def get_cookies(self, as_dict=False, all_domains=False, all_info=False):
"""返回cookies """返回cookies
:param as_dict: 是否以字典方式返回False则以list返回 :param as_dict: 是否以字典方式返回False则以list返回
:param all_domains: 是否返回所有域的cookies :param all_domains: 是否返回所有域的cookies
:param all_info: 是否返回所有信息False则只返回namevaluedomain :param all_info: 是否返回所有信息False则只返回namevaluedomain
:return: cookies信息 :return: cookies信息
""" """
if all_domains: if all_domains:
cookies = self.session.cookies cookies = self.session.cookies
else: else:
if self.url: if self.url:
ex_url = extract(self._session_url) ex_url = extract(self._session_url)
domain = f'{ex_url.domain}.{ex_url.suffix}' if ex_url.suffix else ex_url.domain domain = f'{ex_url.domain}.{ex_url.suffix}' if ex_url.suffix else ex_url.domain
cookies = tuple(x for x in self.session.cookies if domain in x.domain or x.domain == '') cookies = tuple(x for x in self.session.cookies if domain in x.domain or x.domain == '')
else: else:
cookies = tuple(x for x in self.session.cookies) cookies = tuple(x for x in self.session.cookies)
if as_dict: if as_dict:
return {x.name: x.value for x in cookies} return {x.name: x.value for x in cookies}
elif all_info: elif all_info:
return [cookie_to_dict(cookie) for cookie in cookies] return [cookie_to_dict(cookie) for cookie in cookies]
else: else:
r = [] r = []
for c in cookies: for c in cookies:
c = cookie_to_dict(c) c = cookie_to_dict(c)
r.append({'name': c['name'], 'value': c['value'], 'domain': c['domain']}) r.append({'name': c['name'], 'value': c['value'], 'domain': c['domain']})
return r return r
def post(self, url, data=None, show_errmsg=False, retry=None, interval=None, **kwargs): def post(self, url, data=None, show_errmsg=False, retry=None, interval=None, **kwargs):
"""用post方式跳转到url """用post方式跳转到url
:param url: 目标url :param url: 目标url
:param data: 提交的数据 :param data: 提交的数据
:param show_errmsg: 是否显示和抛出异常 :param show_errmsg: 是否显示和抛出异常
:param retry: 重试次数 :param retry: 重试次数
:param interval: 重试间隔 :param interval: 重试间隔
:param kwargs: 连接参数 :param kwargs: 连接参数
:return: url是否可用 :return: url是否可用
""" """
return self._s_connect(url, 'post', data, show_errmsg, retry, interval, **kwargs) return self._s_connect(url, 'post', data, show_errmsg, retry, interval, **kwargs)
def _s_connect(self, url, mode, data=None, show_errmsg=False, retry=None, interval=None, **kwargs): def _s_connect(self, url, mode, data=None, show_errmsg=False, retry=None, interval=None, **kwargs):
"""执行get或post连接 """执行get或post连接
:param url: 目标url :param url: 目标url
:param mode: 'get' 'post' :param mode: 'get' 'post'
:param data: 提交的数据 :param data: 提交的数据
:param show_errmsg: 是否显示和抛出异常 :param show_errmsg: 是否显示和抛出异常
:param retry: 重试次数 :param retry: 重试次数
:param interval: 重试间隔 :param interval: 重试间隔
:param kwargs: 连接参数 :param kwargs: 连接参数
:return: url是否可用 :return: url是否可用
""" """
retry, interval = self._before_connect(url, retry, interval) retry, interval = self._before_connect(url, retry, interval)
self._response, info = self._make_response(self._url, mode, data, retry, interval, show_errmsg, **kwargs) self._response, info = self._make_response(self._url, mode, data, retry, interval, show_errmsg, **kwargs)
if self._response is None: if self._response is None:
self._url_available = False self._url_available = False
else: else:
if self._response.ok: if self._response.ok:
self._url_available = True self._url_available = True
else: else:
if show_errmsg: if show_errmsg:
raise ConnectionError(f'状态码:{self._response.status_code}.') raise ConnectionError(f'状态码:{self._response.status_code}.')
self._url_available = False self._url_available = False
return self._url_available return self._url_available
def _make_response(self, url, mode='get', data=None, retry=None, interval=None, show_errmsg=False, **kwargs): def _make_response(self, url, mode='get', data=None, retry=None, interval=None, show_errmsg=False, **kwargs):
"""生成Response对象 """生成Response对象
:param url: 目标url :param url: 目标url
:param mode: 'get' 'post' :param mode: 'get' 'post'
:param data: post方式要提交的数据 :param data: post方式要提交的数据
:param show_errmsg: 是否显示和抛出异常 :param show_errmsg: 是否显示和抛出异常
:param kwargs: 其它参数 :param kwargs: 其它参数
:return: tuple第一位为Response或None第二位为出错信息或'Success' :return: tuple第一位为Response或None第二位为出错信息或'Success'
""" """
kwargs = CaseInsensitiveDict(kwargs) kwargs = CaseInsensitiveDict(kwargs)
if 'headers' not in kwargs: if 'headers' not in kwargs:
kwargs['headers'] = {} kwargs['headers'] = {}
else: else:
kwargs['headers'] = CaseInsensitiveDict(kwargs['headers']) kwargs['headers'] = CaseInsensitiveDict(kwargs['headers'])
# 设置referer和host值 # 设置referer和host值
parsed_url = urlparse(url) parsed_url = urlparse(url)
hostname = parsed_url.hostname hostname = parsed_url.hostname
scheme = parsed_url.scheme scheme = parsed_url.scheme
if not check_headers(kwargs, self.session.headers, 'Referer'): if not check_headers(kwargs, self.session.headers, 'Referer'):
kwargs['headers']['Referer'] = self.url if self.url else f'{scheme}://{hostname}' kwargs['headers']['Referer'] = self.url if self.url else f'{scheme}://{hostname}'
if 'Host' not in kwargs['headers']: if 'Host' not in kwargs['headers']:
kwargs['headers']['Host'] = hostname kwargs['headers']['Host'] = hostname
if not check_headers(kwargs, self.session.headers, 'timeout'): if not check_headers(kwargs, self.session.headers, 'timeout'):
kwargs['timeout'] = self.timeout kwargs['timeout'] = self.timeout
r = err = None r = err = None
retry = retry if retry is not None else self.retry_times retry = retry if retry is not None else self.retry_times
interval = interval if interval is not None else self.retry_interval interval = interval if interval is not None else self.retry_interval
for i in range(retry + 1): for i in range(retry + 1):
try: try:
if mode == 'get': if mode == 'get':
r = self.session.get(url, **kwargs) r = self.session.get(url, **kwargs)
elif mode == 'post': elif mode == 'post':
r = self.session.post(url, data=data, **kwargs) r = self.session.post(url, data=data, **kwargs)
if r: if r:
return set_charset(r), 'Success' return set_charset(r), 'Success'
except Exception as e: except Exception as e:
err = e err = e
# if r and r.status_code in (403, 404): # if r and r.status_code in (403, 404):
# break # break
if i < retry: if i < retry:
sleep(interval) sleep(interval)
if show_errmsg: if show_errmsg:
print(f'重试 {url}') print(f'重试 {url}')
if r is None: if r is None:
if show_errmsg: if show_errmsg:
if err: if err:
raise err raise err
else: else:
raise ConnectionError('连接失败') raise ConnectionError('连接失败')
return None, '连接失败' if err is None else err return None, '连接失败' if err is None else err
if not r.ok: if not r.ok:
if show_errmsg: if show_errmsg:
raise ConnectionError(f'状态码:{r.status_code}') raise ConnectionError(f'状态码:{r.status_code}')
return r, f'状态码:{r.status_code}' return r, f'状态码:{r.status_code}'
def check_headers(kwargs, headers, arg): def check_headers(kwargs, headers, arg):
"""检查kwargs或headers中是否有arg所示属性""" """检查kwargs或headers中是否有arg所示属性"""
return arg in kwargs['headers'] or arg in headers return arg in kwargs['headers'] or arg in headers
def set_charset(response): def set_charset(response):
"""设置Response对象的编码""" """设置Response对象的编码"""
# 在headers中获取编码 # 在headers中获取编码
content_type = response.headers.get('content-type', '').lower() content_type = response.headers.get('content-type', '').lower()
if not content_type.endswith(';'): if not content_type.endswith(';'):
content_type += ';' content_type += ';'
charset = search(r'charset[=: ]*(.*)?;?', content_type) charset = search(r'charset[=: ]*(.*)?;?', content_type)
if charset: if charset:
response.encoding = charset.group(1) response.encoding = charset.group(1)
# 在headers中获取不到编码且如果是网页 # 在headers中获取不到编码且如果是网页
elif content_type.replace(' ', '').startswith('text/html'): elif content_type.replace(' ', '').startswith('text/html'):
re_result = search(b'<meta.*?charset=[ \\\'"]*([^"\\\' />]+).*?>', response.content) re_result = search(b'<meta.*?charset=[ \\\'"]*([^"\\\' />]+).*?>', response.content)
if re_result: if re_result:
charset = re_result.group(1).decode() charset = re_result.group(1).decode()
else: else:
charset = response.apparent_encoding charset = response.apparent_encoding
response.encoding = charset response.encoding = charset
return response return response

View File

@ -8,11 +8,11 @@ from typing import Any, Union, Tuple, List
from requests import Session, Response from requests import Session, Response
from requests.structures import CaseInsensitiveDict from requests.structures import CaseInsensitiveDict
from .base import BasePage from DrissionPage._base.base import BasePage
from .commons.constants import NoneElement from DrissionPage._commons.constants import NoneElement
from .configs.session_options import SessionOptions from DrissionPage._configs.session_options import SessionOptions
from .session_element import SessionElement from DrissionPage._elements.session_element import SessionElement
from .setter import SessionPageSetter from DrissionPage._units.setter import SessionPageSetter
class SessionPage(BasePage): class SessionPage(BasePage):

View File

@ -3,13 +3,13 @@
@Author : g1879 @Author : g1879
@Contact : g1879@qq.com @Contact : g1879@qq.com
""" """
from .base import BasePage from DrissionPage._base.base import BasePage
from .chromium_page import ChromiumPage from DrissionPage._commons.web import set_session_cookies, set_browser_cookies
from .chromium_tab import WebPageTab from DrissionPage._configs.chromium_options import ChromiumOptions
from .commons.web import set_session_cookies, set_browser_cookies from DrissionPage._pages.chromium_page import ChromiumPage
from .configs.chromium_options import ChromiumOptions from DrissionPage._pages.chromium_tab import WebPageTab
from DrissionPage._units.setter import WebPageSetter
from .session_page import SessionPage from .session_page import SessionPage
from .setter import WebPageSetter
class WebPage(SessionPage, ChromiumPage, BasePage): class WebPage(SessionPage, ChromiumPage, BasePage):

View File

@ -5,20 +5,19 @@
""" """
from typing import Union, Tuple, List, Any from typing import Union, Tuple, List, Any
from DownloadKit import DownloadKit
from requests import Session, Response from requests import Session, Response
from .base import BasePage from ._base import BasePage
from .chromium_driver import ChromiumDriver from ._chromium_driver import ChromiumDriver
from .chromium_element import ChromiumElement from ._chromium_element import ChromiumElement
from .chromium_frame import ChromiumFrame from .chromium_frame import ChromiumFrame
from .chromium_page import ChromiumPage from .chromium_page import ChromiumPage
from .chromium_tab import WebPageTab from .chromium_tab import WebPageTab
from .configs.chromium_options import ChromiumOptions from ._configs.chromium_options import ChromiumOptions
from .configs.session_options import SessionOptions from ._configs.session_options import SessionOptions
from .session_element import SessionElement from ._session_element import SessionElement
from .session_page import SessionPage from .session_page import SessionPage
from .setter import WebPageSetter from ._units.setter import WebPageSetter
class WebPage(SessionPage, ChromiumPage, BasePage): class WebPage(SessionPage, ChromiumPage, BasePage):

View File

@ -5,8 +5,8 @@
""" """
from time import sleep from time import sleep
from .commons.keys import modifierBit, keyDescriptionForString from DrissionPage._commons.keys import modifierBit, keyDescriptionForString
from .commons.web import location_in_viewport from DrissionPage._commons.web import location_in_viewport
class ActionChains: class ActionChains:

View File

@ -5,10 +5,9 @@
""" """
from typing import Union, Tuple from typing import Union, Tuple
from .chromium_base import ChromiumBase from DrissionPage._base.chromium_driver import ChromiumDriver
from .chromium_driver import ChromiumDriver from DrissionPage._elements.chromium_element import ChromiumElement
from .chromium_element import ChromiumElement from DrissionPage._pages.chromium_base import ChromiumBase
from .chromium_page import ChromiumPage
class ActionChains: class ActionChains:

View File

@ -4,7 +4,7 @@ from pathlib import Path
from shutil import move from shutil import move
from time import sleep, perf_counter from time import sleep, perf_counter
from .commons.tools import get_usable_path from DrissionPage._commons.tools import get_usable_path
class BrowserDownloadManager(object): class BrowserDownloadManager(object):

View File

@ -1,8 +1,8 @@
from pathlib import Path from pathlib import Path
from typing import Dict, Optional, Union from typing import Dict, Optional, Union
from .browser import Browser from DrissionPage._base.browser import Browser
from .chromium_page import ChromiumPage from DrissionPage._pages.chromium_page import ChromiumPage
class BrowserDownloadManager(object): class BrowserDownloadManager(object):

View File

@ -1,4 +1,8 @@
# -*- coding:utf-8 -*- # -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
"""
from base64 import b64decode from base64 import b64decode
from json import JSONDecodeError, loads from json import JSONDecodeError, loads
from queue import Queue from queue import Queue
@ -8,7 +12,7 @@ from time import perf_counter, sleep
from requests.structures import CaseInsensitiveDict from requests.structures import CaseInsensitiveDict
from .errors import CDPError from DrissionPage.errors import CDPError
class NetworkListener(object): class NetworkListener(object):

View File

@ -1,10 +1,15 @@
# -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
"""
from queue import Queue from queue import Queue
from typing import Union, Dict, List, Iterable, Tuple from typing import Union, Dict, List, Iterable, Tuple
from requests.structures import CaseInsensitiveDict from requests.structures import CaseInsensitiveDict
from chromium_base import ChromiumBase from DrissionPage._base.chromium_driver import ChromiumDriver
from chromium_driver import ChromiumDriver from DrissionPage._pages.chromium_base import ChromiumBase
class NetworkListener(object): class NetworkListener(object):

View File

@ -7,8 +7,8 @@ from pathlib import Path
from requests.structures import CaseInsensitiveDict from requests.structures import CaseInsensitiveDict
from .commons.tools import show_or_hide_browser from DrissionPage._commons.tools import show_or_hide_browser
from .commons.web import set_browser_cookies, set_session_cookies from DrissionPage._commons.web import set_browser_cookies, set_session_cookies
class ChromiumBaseSetter(object): class ChromiumBaseSetter(object):

View File

@ -11,13 +11,13 @@ from requests.adapters import HTTPAdapter
from requests.auth import HTTPBasicAuth from requests.auth import HTTPBasicAuth
from requests.cookies import RequestsCookieJar from requests.cookies import RequestsCookieJar
from .chromium_base import ChromiumBase, ChromiumPageScroll from DrissionPage._elements.chromium_element import ChromiumElement
from .chromium_element import ChromiumElement from DrissionPage._pages.chromium_base import ChromiumBase, ChromiumPageScroll
from .chromium_frame import ChromiumFrame from DrissionPage._pages.chromium_frame import ChromiumFrame
from .chromium_page import ChromiumPage from DrissionPage._pages.chromium_page import ChromiumPage
from .chromium_tab import ChromiumTab from DrissionPage._pages.chromium_tab import ChromiumTab
from .session_page import SessionPage from DrissionPage._pages.session_page import SessionPage
from .web_page import WebPage from DrissionPage._pages.web_page import WebPage
FILE_EXISTS = Literal['skip', 'rename', 'overwrite', 's', 'r', 'o'] FILE_EXISTS = Literal['skip', 'rename', 'overwrite', 's', 'r', 'o']

View File

@ -1,8 +1,8 @@
# -*- coding:utf-8 -*- # -*- coding:utf-8 -*-
from time import sleep, perf_counter from time import sleep, perf_counter
from .commons.constants import Settings from DrissionPage._commons.constants import Settings
from .errors import WaitTimeoutError from DrissionPage.errors import WaitTimeoutError
class ChromiumBaseWaiter(object): class ChromiumBaseWaiter(object):

View File

@ -5,11 +5,11 @@
""" """
from typing import Union from typing import Union
from DrissionPage._elements.chromium_element import ChromiumElement
from DrissionPage._pages.chromium_base import ChromiumBase
from DrissionPage._pages.chromium_frame import ChromiumFrame
from DrissionPage._pages.chromium_page import ChromiumPage
from .browser_download_manager import DownloadMission from .browser_download_manager import DownloadMission
from .chromium_base import ChromiumBase
from .chromium_element import ChromiumElement
from .chromium_frame import ChromiumFrame
from .chromium_page import ChromiumPage
class ChromiumBaseWaiter(object): class ChromiumBaseWaiter(object):

View File

@ -6,9 +6,9 @@
""" """
from FlowViewer import Listener, RequestMan from FlowViewer import Listener, RequestMan
from .session_element import make_session_ele from ._elements.session_element import make_session_ele
from .action_chains import ActionChains from ._units.action_chains import ActionChains
from .commons.keys import Keys from ._commons.keys import Keys
from .commons.by import By from ._commons.by import By
from .commons.constants import Settings from ._commons.constants import Settings

View File

@ -1,7 +1,10 @@
# -*- coding:utf-8 -*- # -*- coding:utf-8 -*-
from .session_element import make_session_ele as make_session_ele """
@Author : g1879
from .action_chains import ActionChains as ActionChains @Contact : g1879@qq.com
from .commons.keys import Keys as Keys """
from .commons.by import By as By from ._commons.by import By as By
from .commons.constants import Settings as Settings from ._commons.constants import Settings as Settings
from ._commons.keys import Keys as Keys
from ._elements.session_element import make_session_ele as make_session_ele
from ._units.action_chains import ActionChains as ActionChains

View File

@ -7,8 +7,8 @@ from os import popen
from pathlib import Path from pathlib import Path
from re import search from re import search
from .configs.chromium_options import ChromiumOptions from ._configs.chromium_options import ChromiumOptions
from .configs.options_manage import OptionsManager from ._configs.options_manage import OptionsManager
def configs_to_here(save_name=None): def configs_to_here(save_name=None):

View File

@ -1,4 +1,8 @@
# -*- coding:utf-8 -*- # -*- coding:utf-8 -*-
"""
@Author : g1879
@Contact : g1879@qq.com
"""
class BaseError(Exception): class BaseError(Exception):