Update TextPreprocessor.py

This commit is contained in:
CyberWon 2024-06-10 17:24:03 +08:00 committed by GitHub
parent 277b258360
commit 0d54936a1c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -56,7 +56,7 @@ class TextPreprocessor:
def preprocess(self, text:str, lang:str, text_split_method:str)->List[Dict]: def preprocess(self, text:str, lang:str, text_split_method:str)->List[Dict]:
print(i18n("############ 切分文本 ############")) print(i18n("############ 切分文本 ############"))
texts = self.replace_consecutive_punctuation(texts) text = self.replace_consecutive_punctuation(text)
texts = self.pre_seg_text(text, lang, text_split_method) texts = self.pre_seg_text(text, lang, text_split_method)
result = [] result = []
print(i18n("############ 提取文本Bert特征 ############")) print(i18n("############ 提取文本Bert特征 ############"))