diff --git a/DrissionPage/base.py b/DrissionPage/base.py index 0c9b5d5..404fb24 100644 --- a/DrissionPage/base.py +++ b/DrissionPage/base.py @@ -26,14 +26,9 @@ class BaseParser(object): def eles(self, loc_or_str: Union[Tuple[str, str], str], timeout=None): return self._ele(loc_or_str, timeout, False) - @property - def html(self) -> str: - """返回已转码的html文本""" - return format_html(self.raw_html) - # ----------------以下属性或方法待后代实现---------------- @property - def raw_html(self) -> str: + def html(self) -> str: return '' @abstractmethod @@ -120,11 +115,6 @@ class DrissionElement(BaseElement): """返回元素注释文本组成的列表""" return self.eles('xpath:.//comment()') - @property - def inner_html(self) -> str: - """返回已转码的html文本""" - return format_html(self.inner_raw_html) - def texts(self, text_node_only: bool = False) -> list: """返回元素内所有直接子节点的文本,包括元素和文本节点 \n :param text_node_only: 是否只返回文本节点 @@ -191,10 +181,6 @@ class DrissionElement(BaseElement): return ele_or_node # ----------------以下属性或方法由后代实现---------------- - @property - def inner_raw_html(self) -> str: - return '' - @property def attrs(self): return diff --git a/DrissionPage/common.py b/DrissionPage/common.py index bf8188b..beddbdf 100644 --- a/DrissionPage/common.py +++ b/DrissionPage/common.py @@ -18,7 +18,7 @@ def get_ele_txt(e) -> str: :return: 元素内所有文本 """ # 前面无须换行的元素 - nowrap_list = ('sub', 'em', 'strong', 'a', 'font', 'b', 'span', 's', 'i', 'del', 'ins', 'img', 'td', 'th', + nowrap_list = ('br', 'sub', 'em', 'strong', 'a', 'font', 'b', 'span', 's', 'i', 'del', 'ins', 'img', 'td', 'th', 'abbr', 'bdi', 'bdo', 'cite', 'code', 'data', 'dfn', 'kbd', 'mark', 'q', 'rp', 'rt', 'ruby', 'samp', 'small', 'sub', 'time', 'u', 'var', 'wbr', 'button', 'slot', 'content') # 后面添加换行的元素 @@ -35,7 +35,7 @@ def get_ele_txt(e) -> str: def get_node_txt(ele, pre: bool = False): tag = ele.tag if tag == 'br': - return '\n' + return [True] if not pre and tag == 'pre': pre = True @@ -67,13 +67,16 @@ def get_ele_txt(e) -> str: str_list.extend(get_node_txt(el, pre)) prev_ele = el.tag - if tag in wrap_after_list and str_list and str_list[-1] != '\n': # 有些元素后面要添加回车 + if tag in wrap_after_list and str_list and str_list[-1] not in ('\n', True): # 有些元素后面要添加回车 str_list.append('\n') return str_list - re_str = ''.join(get_node_txt(e)) - return format_html(re_str, False).strip(' \n') + re_str = get_node_txt(e) + if re_str and re_str[-1] == '\n': + re_str.pop() + re_str = ''.join([i if i is not True else '\n' for i in re_str]) + return format_html(re_str) def str_to_loc(loc: str) -> tuple: @@ -240,19 +243,12 @@ def _make_search_str(search_str: str) -> str: return search_str -def format_html(text: str, trans: bool = True) -> str: +def format_html(text: str) -> str: """处理html编码字符 \n :param text: html文本 - :param trans: 是否转码 :return: 格式化后的html文本 """ - if not text: - return text - - if trans: - text = unescape(text) - - return text.replace('\xa0', ' ') + return unescape(text).replace('\xa0', ' ') if text else text def clean_folder(folder_path: str, ignore: list = None) -> None: diff --git a/DrissionPage/driver_element.py b/DrissionPage/driver_element.py index bd56fc6..ece3543 100644 --- a/DrissionPage/driver_element.py +++ b/DrissionPage/driver_element.py @@ -49,13 +49,13 @@ class DriverElement(DrissionElement): return self._inner_ele.tag_name.lower() @property - def raw_html(self) -> str: - """返回未转码的的outerHTML文本""" + def html(self) -> str: + """返回元素outerHTML文本""" return self.inner_ele.get_attribute('outerHTML') @property - def inner_raw_html(self) -> str: - """返回元素未转码的innerHTML文本""" + def inner_html(self) -> str: + """返回元素innerHTML文本""" return self.inner_ele.get_attribute('innerHTML') @property @@ -80,7 +80,7 @@ class DriverElement(DrissionElement): @property def text(self) -> str: """返回元素内所有文本""" - return get_ele_txt(make_session_ele(self.raw_html)) + return get_ele_txt(make_session_ele(self.html)) @property def raw_text(self) -> str: diff --git a/DrissionPage/driver_page.py b/DrissionPage/driver_page.py index 7a03745..e441c7d 100644 --- a/DrissionPage/driver_page.py +++ b/DrissionPage/driver_page.py @@ -51,8 +51,8 @@ class DriverPage(BasePage): return self.driver.current_url @property - def raw_html(self) -> str: - """返回页面没有转码的html文本""" + def html(self) -> str: + """返回页面的html文本""" return self.driver.find_element('xpath', "//*").get_attribute("outerHTML") @property diff --git a/DrissionPage/mix_page.py b/DrissionPage/mix_page.py index 6674400..0b4a398 100644 --- a/DrissionPage/mix_page.py +++ b/DrissionPage/mix_page.py @@ -88,14 +88,6 @@ class MixPage(SessionPage, DriverPage, BasePage): elif self._mode == 'd': return super(SessionPage, self).html - @property - def raw_html(self) -> str: - """返回页面html文本""" - if self._mode == 's': - return super().raw_html - elif self._mode == 'd': - return super(SessionPage, self).raw_html - @property def json(self) -> dict: """当返回内容是json格式时,返回对应的字典""" diff --git a/DrissionPage/session_element.py b/DrissionPage/session_element.py index e80c981..2f01204 100644 --- a/DrissionPage/session_element.py +++ b/DrissionPage/session_element.py @@ -39,15 +39,15 @@ class SessionElement(DrissionElement): return self._inner_ele.tag @property - def raw_html(self) -> str: - """返回未转码的outerHTML文本""" + def html(self) -> str: + """返回outerHTML文本""" html = tostring(self._inner_ele, method="html").decode() return html[:html.rfind('>') + 1] # tostring()会把跟紧元素的文本节点也带上,因此要去掉 @property - def inner_raw_html(self) -> str: - """返回元素未转码的innerHTML文本""" - r = match(r'<.*?>(.*)', self.raw_html, flags=DOTALL) + def inner_html(self) -> str: + """返回元素innerHTML文本""" + r = match(r'<.*?>(.*)', self.html, flags=DOTALL) return '' if not r else r.group(1) @property @@ -227,7 +227,7 @@ def make_session_ele(html_or_ele: Union[str, BaseElement, BasePage], elif loc[0] == 'css selector' and loc[1].lstrip().startswith('>'): loc_str = f'{html_or_ele.css_path}{loc[1]}' if html_or_ele.page: - html_or_ele = fromstring(html_or_ele.page.raw_html) + html_or_ele = fromstring(html_or_ele.page.html) else: # 接收html文本,无page的情况 html_or_ele = fromstring(html_or_ele('xpath:/ancestor::*').html) @@ -247,12 +247,12 @@ def make_session_ele(html_or_ele: Union[str, BaseElement, BasePage], # 获取整个页面html再定位到当前元素,以实现查找上级元素 page = html_or_ele.page xpath = html_or_ele.xpath - html_or_ele = fromstring(html_or_ele.page.raw_html) + html_or_ele = fromstring(html_or_ele.page.html) html_or_ele = html_or_ele.xpath(xpath)[0] elif isinstance(html_or_ele, BasePage): # MixPage, DriverPage 或 SessionPage page = html_or_ele - html_or_ele = fromstring(html_or_ele.raw_html) + html_or_ele = fromstring(html_or_ele.html) elif isinstance(html_or_ele, str): # 直接传入html文本 page = None @@ -260,7 +260,7 @@ def make_session_ele(html_or_ele: Union[str, BaseElement, BasePage], elif isinstance(html_or_ele, BaseElement): # ShadowRootElement page = html_or_ele.page - html_or_ele = fromstring(html_or_ele.raw_html) + html_or_ele = fromstring(html_or_ele.html) else: raise TypeError('html_or_ele参数只能是元素、页面对象或html文本。') diff --git a/DrissionPage/session_page.py b/DrissionPage/session_page.py index adefd7c..75a64f6 100644 --- a/DrissionPage/session_page.py +++ b/DrissionPage/session_page.py @@ -46,8 +46,8 @@ class SessionPage(BasePage): return self._url @property - def raw_html(self) -> str: - """返回页面已转码的html文本""" + def html(self) -> str: + """返回页面的html文本""" return self.response.text if self.response else '' @property diff --git a/DrissionPage/shadow_root_element.py b/DrissionPage/shadow_root_element.py index 431f12d..7f151c5 100644 --- a/DrissionPage/shadow_root_element.py +++ b/DrissionPage/shadow_root_element.py @@ -41,8 +41,8 @@ class ShadowRootElement(BaseElement): return 'shadow-root' @property - def raw_html(self) -> str: - """内部没有转码的html文本""" + def html(self) -> str: + """返回内部的html文本""" return self.inner_ele.get_attribute('innerHTML') @property diff --git a/README.md b/README.md index bea23e7..6a53ca1 100644 --- a/README.md +++ b/README.md @@ -32,17 +32,14 @@ requests 爬虫面对要登录的网站时,要分析数据包、JS 源码, ## 特性 - 代码高度集成,以简洁的代码为第一追求。 - - 页面对象可在 selenium 和 requests 模式间任意切换,保留登录状态。 - +- 极简单但强大的元素查找功能,支持链式操作,代码极其简洁。 - 两种模式提供一致的 API,使用体验一致。 - - 人性化设计,集成众多实用功能,大大降低开发工作量。 ## 亮点 - 每次运行程序可以反复使用已经打开的浏览器。如手动设置网页到某个状态,再用程序接管,或手动处理登录,再用程序爬内容。无须每次运行从头启动浏览器,超级方便。 -- 极简单但强大的元素查找功能,支持链式操作,代码极其简洁。 - 使用 ini 文件保存常用配置,自动调用,也提供便捷的设置api,远离繁杂的配置项。 - 强大的下载工具,操作浏览器时也能享受快捷可靠的下载功能。 - 下载工具支持多种方式处理文件名冲突、自动创建目标路径、断链重试等。