blackteay/DrissionPage
1
0
Fork 0
mirror of https://gitee.com/g1879/DrissionPage.git synced 2024-12-10 04:00:23 +08:00
Code Issues Projects Releases Wiki Activity

增加raw_html和inner_raw_html两个属性

Browse Source
This commit is contained in:
g1879 2021-12-01 14:11:52 +08:00
parent e6d15054b1
commit af69ca22c9
8 changed files with 71 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
DrissionPage/base.py
View File

@ -26,10 +26,15 @@ class BaseParser(object):
def eles(self, loc_or_str: Union[Tuple[str, str], str]):
return self._ele(loc_or_str, False)
@property
def html(self) -> str:
"""返回已转码的html文本"""
return format_html(self.raw_html)
# ----------------以下属性或方法待后代实现----------------
@property
def html(self):
return
def raw_html(self) -> str:
return ''
@abstractmethod
def s_ele(self, loc_or_ele):
@ -65,9 +70,9 @@ class BaseElement(BaseParser):
def tag(self):
return
@property
def html(self):
return
# @property
# def html(self):
# return
@property
def parent(self):
@ -119,6 +124,11 @@ class DrissionElement(BaseElement):
"""返回元素注释文本组成的列表"""
return self.eles('xpath:.//comment()')
@property
def inner_html(self) -> str:
"""返回已转码的html文本"""
return format_html(self.inner_raw_html)
def texts(self, text_node_only: bool = False) -> list:
"""返回元素内所有直接子节点的文本，包括元素和文本节点 \n
:param text_node_only: 是否只返回文本节点
@ -186,8 +196,8 @@ class DrissionElement(BaseElement):
# ----------------以下属性或方法由后代实现----------------
@property
def inner_html(self):
return
def inner_raw_html(self) -> str:
return ''
@property
def attrs(self):
@ -255,9 +265,9 @@ class BasePage(BaseParser):
def url(self):
return
@property
def html(self):
return
# @property
# def html(self):
# return
@property
def json(self):

18
DrissionPage/common.py
View File

@ -33,16 +33,16 @@ def get_ele_txt(e) -> str:
return e.raw_text
def get_node_txt(ele, pre: bool = False):
str_list = []
tag = ele.tag.lower()
if tag in noText_list: # 标签内的文本不返回
return str_list
tag = ele.tag
if tag == 'br':
return '\n'
if tag == 'pre':
if not pre and tag == 'pre':
pre = True
str_list = []
if tag in noText_list and not pre: # 标签内的文本不返回
return str_list
nodes = ele.eles('xpath:./text() | *')
prev_ele = ''
for el in nodes:
@ -59,13 +59,13 @@ def get_ele_txt(e) -> str:
str_list.append(txt)
else: # 元素节点
if el.tag.lower() not in nowrap_list and str_list and str_list[-1] != '\n': # 元素间换行的情况
if el.tag not in nowrap_list and str_list and str_list[-1] != '\n': # 元素间换行的情况
str_list.append('\n')
if el.tag.lower() in tab_list and prev_ele in tab_list: # 表格的行
if el.tag in tab_list and prev_ele in tab_list: # 表格的行
str_list.append('\t')
str_list.extend(get_node_txt(el, pre))
prev_ele = el.tag.lower()
prev_ele = el.tag
if tag in wrap_after_list and str_list and str_list[-1] != '\n': # 有些元素后面要添加回车
str_list.append('\n')

15
DrissionPage/driver_element.py
View File

@ -49,14 +49,14 @@ class DriverElement(DrissionElement):
return self._inner_ele.tag_name.lower()
@property
def html(self) -> str:
"""返回元素outerHTML文本"""
return self.attr('outerHTML')
def raw_html(self) -> str:
"""返回未转码的的outerHTML文本"""
return self.inner_ele.get_attribute('outerHTML')
@property
def inner_html(self) -> str:
"""返回元素innerHTML文本"""
return self.attr('innerHTML')
def inner_raw_html(self) -> str:
"""返回元素未转码的innerHTML文本"""
return self.inner_ele.get_attribute('innerHTML')
@property
def attrs(self) -> dict:
@ -80,7 +80,8 @@ class DriverElement(DrissionElement):
@property
def text(self) -> str:
"""返回元素内所有文本"""
return get_ele_txt(make_session_ele(self.html))
return get_ele_txt(make_session_ele(self.raw_html))
# return get_ele_txt(self)
@property
def raw_text(self) -> str:

8
DrissionPage/driver_page.py
View File

@ -17,7 +17,7 @@ from selenium.webdriver.remote.webelement import WebElement
from selenium.webdriver.support.wait import WebDriverWait
from .base import BasePage
from .common import get_usable_path, format_html
from .common import get_usable_path
from .driver_element import DriverElement, make_driver_ele, _wait_ele
from .session_element import make_session_ele
@ -51,9 +51,9 @@ class DriverPage(BasePage):
return self.driver.current_url
@property
def html(self) -> str:
"""返回页面html文本"""
return format_html(self.driver.find_element('xpath', "//*").get_attribute("outerHTML"))
def raw_html(self) -> str:
"""返回页面没有转码的html文本"""
return self.driver.find_element('xpath', "//*").get_attribute("outerHTML")
@property
def json(self) -> dict:

8
DrissionPage/mix_page.py
View File

@ -88,6 +88,14 @@ class MixPage(SessionPage, DriverPage, BasePage):
elif self._mode == 'd':
return super(SessionPage, self).html
@property
def raw_html(self) -> str:
"""返回页面html文本"""
if self._mode == 's':
return super().raw_html
elif self._mode == 'd':
return super(SessionPage, self).raw_html
@property
def json(self) -> dict:
"""当返回内容是json格式时，返回对应的字典"""

22
DrissionPage/session_element.py
View File

@ -12,7 +12,7 @@ from lxml.etree import tostring
from lxml.html import HtmlElement, fromstring
from .base import DrissionElement, BasePage, BaseElement
from .common import str_to_loc, translate_loc, format_html, get_ele_txt
from .common import str_to_loc, translate_loc, get_ele_txt
class SessionElement(DrissionElement):
@ -39,15 +39,15 @@ class SessionElement(DrissionElement):
return self._inner_ele.tag
@property
def html(self) -> str:
"""返回元素outerHTML文本"""
html = format_html(tostring(self._inner_ele, method="html").decode())
def raw_html(self) -> str:
"""返回未转码的outerHTML文本"""
html = tostring(self._inner_ele, method="html").decode()
return html[:html.rfind('>') + 1] # tostring()会把跟紧元素的文本节点也带上，因此要去掉
@property
def inner_html(self) -> str:
"""返回元素innerHTML文本"""
r = match(r'<.*?>(.*)</.*?>', self.html, flags=DOTALL)
def inner_raw_html(self) -> str:
"""返回元素未转码的innerHTML文本"""
r = match(r'<.*?>(.*)</.*?>', self.raw_html, flags=DOTALL)
return '' if not r else r.group(1)
@property
@ -225,7 +225,7 @@ def make_session_ele(html_or_ele: Union[str, BaseElement, BasePage],
elif loc[0] == 'css selector' and loc[1].lstrip().startswith('>'):
loc_str = f'{html_or_ele.css_path}{loc[1]}'
if html_or_ele.page:
html_or_ele = fromstring(html_or_ele.page.html)
html_or_ele = fromstring(html_or_ele.page.raw_html)
else: # 接收html文本，无page的情况
html_or_ele = fromstring(html_or_ele('xpath:/ancestor::*').html)
@ -245,12 +245,12 @@ def make_session_ele(html_or_ele: Union[str, BaseElement, BasePage],
# 获取整个页面html再定位到当前元素，以实现查找上级元素
page = html_or_ele.page
xpath = html_or_ele.xpath
html_or_ele = fromstring(html_or_ele.page.html)
html_or_ele = fromstring(html_or_ele.page.raw_html)
html_or_ele = html_or_ele.xpath(xpath)[0]
elif isinstance(html_or_ele, BasePage): # MixPage, DriverPage 或 SessionPage
page = html_or_ele
html_or_ele = fromstring(html_or_ele.html)
html_or_ele = fromstring(html_or_ele.raw_html)
elif isinstance(html_or_ele, str): # 直接传入html文本
page = None
@ -258,7 +258,7 @@ def make_session_ele(html_or_ele: Union[str, BaseElement, BasePage],
elif isinstance(html_or_ele, BaseElement): # ShadowRootElement
page = html_or_ele.page
html_or_ele = fromstring(html_or_ele.html)
html_or_ele = fromstring(html_or_ele.raw_html)
else:
raise TypeError('html_or_ele参数只能是元素、页面对象或html文本。')

6
DrissionPage/session_page.py
View File

@ -46,9 +46,9 @@ class SessionPage(BasePage):
return self._url
@property
def html(self) -> str:
"""返回页面html文本"""
return format_html(self.response.text) if self.response else ''
def raw_html(self) -> str:
"""返回页面已转码的html文本"""
return self.response.text if self.response else ''
@property
def json(self) -> dict:

11
DrissionPage/shadow_root_element.py
View File

@ -41,10 +41,15 @@ class ShadowRootElement(BaseElement):
"""元素标签名"""
return 'shadow-root'
# @property
# def html(self) -> str:
# """内部已转码的html文本"""
# return format_html(self.inner_ele.get_attribute('innerHTML'))
@property
def html(self) -> str:
"""内部html文本"""
return format_html(self.inner_ele.get_attribute('innerHTML'))
def raw_html(self) -> str:
"""内部没有转码的html文本"""
return self.inner_ele.get_attribute('innerHTML')
@property
def parent(self) -> DriverElement: