Merge pull request #2122 from KamioRinn/Fix-Short-CJK-LangSegmenter

Fix Short CJK LangSegmenter

GPT_SoVITS/inference_webui.py

@@ -366,6 +366,7 @@ def get_spepc(hps, filename):
     return spec
 
 def clean_text_inf(text, language, version):
+    language = language.replace("all_","")
     phones, word2ph, norm_text = clean_text(text, language, version)
     phones = cleaned_text_to_sequence(phones, version)
     return phones, word2ph, norm_text
@@ -395,11 +396,10 @@ def get_first(text):
 from text import chinese
 def get_phones_and_bert(text,language,version,final=False):
     if language in {"en", "all_zh", "all_ja", "all_ko", "all_yue"}:
-        language = language.replace("all_","")
         formattext = text
         while "  " in formattext:
             formattext = formattext.replace("  ", " ")
-        if language == "zh":
+        if language == "all_zh":
             if re.search(r'[A-Za-z]', formattext):
                 formattext = re.sub(r'[a-z]', lambda x: x.group(0).upper(), formattext)
                 formattext = chinese.mix_text_normalize(formattext)
@@ -407,7 +407,7 @@ def get_phones_and_bert(text,language,version,final=False):
             else:
                 phones, word2ph, norm_text = clean_text_inf(formattext, language, version)
                 bert = get_bert_feature(norm_text, word2ph).to(device)
-        elif language == "yue" and re.search(r'[A-Za-z]', formattext):
+        elif language == "all_yue" and re.search(r'[A-Za-z]', formattext):
                 formattext = re.sub(r'[a-z]', lambda x: x.group(0).upper(), formattext)
                 formattext = chinese.mix_text_normalize(formattext)
                 return get_phones_and_bert(formattext,"yue",version)

GPT_SoVITS/text/LangSegmenter/langsegmenter.py

@@ -78,6 +78,32 @@ def full_en(text):
     return bool(re.match(pattern, text))
 
 
+def full_cjk(text):
+    # 来自wiki
+    cjk_ranges = [
+        (0x4E00, 0x9FFF),        # CJK Unified Ideographs
+        (0x3400, 0x4DB5),        # CJK Extension A
+        (0x20000, 0x2A6DD),      # CJK Extension B
+        (0x2A700, 0x2B73F),      # CJK Extension C
+        (0x2B740, 0x2B81F),      # CJK Extension D
+        (0x2B820, 0x2CEAF),      # CJK Extension E
+        (0x2CEB0, 0x2EBEF),      # CJK Extension F
+        (0x30000, 0x3134A),      # CJK Extension G
+        (0x31350, 0x323AF),      # CJK Extension H
+        (0x2EBF0, 0x2EE5D),      # CJK Extension H
+    ]
+
+    pattern = r'[0-9、-〜。！？.!?… ]+$'
+
+    cjk_text = ""
+    for char in text:
+        code_point = ord(char)
+        in_cjk = any(start <= code_point <= end for start, end in cjk_ranges)
+        if in_cjk or re.match(pattern, char):
+            cjk_text += char
+    return cjk_text
+
+
 def split_jako(tag_lang,item):
     if tag_lang == "ja":
         pattern = r"([\u3041-\u3096\u3099\u309A\u30A1-\u30FA\u30FC]+(?:[0-9、-〜。！？.!?… ]+[\u3041-\u3096\u3099\u309A\u30A1-\u30FA\u30FC]*)*)"
@@ -158,8 +184,12 @@ class LangSegmenter():
 
             # 未存在非日韩文夹日韩文
             if len(temp_list) == 1:
-                # 跳过未知语言
+                # 未知语言检查是否为CJK
                 if dict_item['lang'] == 'x':
+                    cjk_text = full_cjk(dict_item['text'])
+                    if cjk_text:
+                        dict_item = {'lang':'zh','text':cjk_text}
+                        lang_list = merge_lang(lang_list,dict_item)
                     continue
                 else:
                     lang_list = merge_lang(lang_list,dict_item)
@@ -167,12 +197,14 @@ class LangSegmenter():
 
             # 存在非日韩文夹日韩文
             for _, temp_item in enumerate(temp_list):
-                # 待观察是否会出现带英文或语言为x的中日英韩文
+                # 未知语言检查是否为CJK
                 if temp_item['lang'] == 'x':
-                    continue
+                    cjk_text = full_cjk(dict_item['text'])
-
+                    if cjk_text:
+                        dict_item = {'lang':'zh','text':cjk_text}
+                        lang_list = merge_lang(lang_list,dict_item)
+                else:
                     lang_list = merge_lang(lang_list,temp_item)
-
         return lang_list