GPT-SoVITS/GPT_SoVITS/text/LangSegmenter/langsegmenter.py

import logging
import jieba
import re
jieba.setLogLevel(logging.CRITICAL)

# 更改fast_langdetect大模型位置
from pathlib import Path
import fast_langdetect
fast_langdetect.ft_detect.infer.CACHE_DIRECTORY = Path(__file__).parent.parent.parent / "pretrained_models" / "fast_langdetect"
import sys
sys.modules["fast_langdetect"] = fast_langdetect

from split_lang import LangSplitter


def full_en(text):
    pattern = r'^[A-Za-z0-9\s\u0020-\u007E\u2000-\u206F\u3000-\u303F\uFF00-\uFFEF]+$'
    return bool(re.match(pattern, text))


def split_jako(tag_lang,item):
    if tag_lang == "ja":
        pattern = r"([\u3041-\u3096\u3099\u309A\u30A1-\u30FA\u30FC]+(?:[0-9、-〜。！？.!?… ]+[\u3041-\u3096\u3099\u309A\u30A1-\u30FA\u30FC]*)*)"
    else:
        pattern = r"([\u1100-\u11FF\u3130-\u318F\uAC00-\uD7AF]+(?:[0-9、-〜。！？.!?… ]+[\u1100-\u11FF\u3130-\u318F\uAC00-\uD7AF]*)*)"

    lang_list: list[dict] = []
    tag = 0
    for match in re.finditer(pattern, item['text']):
        if match.start() > tag:
            lang_list.append({'lang':item['lang'],'text':item['text'][tag:match.start()]})

        tag = match.end()
        lang_list.append({'lang':tag_lang,'text':item['text'][match.start():match.end()]})

    if tag < len(item['text']):
        lang_list.append({'lang':item['lang'],'text':item['text'][tag:len(item['text'])]})

    return lang_list


def merge_lang(lang_list, item):
    if lang_list and item['lang'] == lang_list[-1]['lang']:
        lang_list[-1]['text'] += item['text']
    else:
        lang_list.append(item)
    return lang_list

from typing import List
from split_lang import SubString
def _special_merge_for_zh_ja(
    self,
    substrings: List[SubString],
) -> List[SubString]:
    new_substrings: List[SubString] = []

    if len(substrings) == 1:
        return substrings
    # NOTE: 统计每个语言的字符串长度
    substring_text_len_by_lang = {
        "zh": 0,
        "ja": 0,
        "x": 0,
        "digit": 0,
        "punctuation": 0,
        "newline": 0,
    }
    index = 0
    while index < len(substrings):
        current_block = substrings[index]
        substring_text_len_by_lang[current_block.lang] += current_block.length
        if index == 0:
            if (
                substrings[index + 1].lang in ["zh", "ja"]
                and substrings[index].lang in ["zh", "ja", "x"]
                and substrings[index].length * 10 < substrings[index + 1].length
            ):
                right_block = substrings[index + 1]
                new_substrings.append(
                    SubString(
                        is_digit=False,
                        is_punctuation=False,
                        lang=right_block.lang,
                        text=current_block.text + right_block.text,
                        length=current_block.length + right_block.length,
                        index=current_block.index,
                    )
                )
                index += 1
            else:
                new_substrings.append(substrings[index])
        elif index == len(substrings) - 1:
            left_block = new_substrings[-1]
            if (
                left_block.lang in ["zh", "ja"]
                and current_block.lang in ["zh", "ja", "x"]
                and current_block.length * 10 < left_block.length
            ):
                new_substrings[-1].text += current_block.text
                new_substrings[-1].length += current_block.length

                index += 1
            else:
                new_substrings.append(substrings[index])
        else:
            if (
                new_substrings[-1].lang == substrings[index + 1].lang
                and new_substrings[-1].lang in ["zh", "ja"]
                # and substrings[index].lang in ["zh", "ja", "x"]
                and substrings[index].lang != "en"
                and substrings[index].length * 10
                < new_substrings[-1].length + substrings[index + 1].length
            ):
                left_block = new_substrings[-1]
                right_block = substrings[index + 1]
                current_block = substrings[index]
                new_substrings[-1].text += current_block.text + right_block.text
                new_substrings[-1].length += (
                    current_block.length + right_block.length
                )
                index += 1
            else:
                new_substrings.append(substrings[index])
        index += 1
    # NOTE: 如果 substring_count 中 存在 x，则将 x 设置为最多的 lang
    if substring_text_len_by_lang["x"] > 0:
        max_lang = max(
            substring_text_len_by_lang, key=substring_text_len_by_lang.get
        )
        for index, substr in enumerate(new_substrings):
            if substr.lang == "x":
                new_substrings[index].lang = max_lang
    # NOTE: 如果 ja 数量是 zh 数量的 10 倍以上，则该 zh 设置为 ja
    if substring_text_len_by_lang["ja"] >= substring_text_len_by_lang["zh"] * 10:
        for index, substr in enumerate(new_substrings):
            if substr.lang == "zh":
                new_substrings[index].lang = "ja"
    new_substrings = self._merge_substrings(substrings=new_substrings)
    return new_substrings



class LangSegmenter():
    # 默认过滤器, 基于gsv目前四种语言
    DEFAULT_LANG_MAP = {
        "zh": "zh",
        "yue": "zh",  # 粤语
        "wuu": "zh",  # 吴语
        "zh-cn": "zh",
        "zh-tw": "x", # 繁体设置为x
        "ko": "ko",
        "ja": "ja",
        "en": "en",
    }

    
    def getTexts(text):
        LangSplitter._special_merge_for_zh_ja = _special_merge_for_zh_ja
        lang_splitter = LangSplitter(lang_map=LangSegmenter.DEFAULT_LANG_MAP)
        substr = lang_splitter.split_by_lang(text=text)

        lang_list: list[dict] = []

        for _, item in enumerate(substr):
            dict_item = {'lang':item.lang,'text':item.text}

            # 处理短英文被识别为其他语言的问题
            if full_en(dict_item['text']):  
                dict_item['lang'] = 'en'
                lang_list = merge_lang(lang_list,dict_item)
                continue

            # 处理非日语夹日文的问题(不包含CJK)
            ja_list: list[dict] = []
            if dict_item['lang'] != 'ja':
                ja_list = split_jako('ja',dict_item)

            if not ja_list:
                ja_list.append(dict_item)

            # 处理非韩语夹韩语的问题(不包含CJK)
            ko_list: list[dict] = []
            temp_list: list[dict] = []
            for _, ko_item in enumerate(ja_list):
                if ko_item["lang"] != 'ko':
                    ko_list = split_jako('ko',ko_item)

                if ko_list:
                    temp_list.extend(ko_list)
                else:
                    temp_list.append(ko_item)

            # 未存在非日韩文夹日韩文
            if len(temp_list) == 1:
                # 跳过未知语言
                if dict_item['lang'] == 'x':
                    continue
                else:
                    lang_list = merge_lang(lang_list,dict_item)
                    continue

            # 存在非日韩文夹日韩文
            for _, temp_item in enumerate(temp_list):
                # 待观察是否会出现带英文或语言为x的中日英韩文
                if temp_item['lang'] == 'x':
                    continue

                lang_list = merge_lang(lang_list,temp_item)

        return lang_list
    

if __name__ == "__main__":
    text = "MyGO?,你也喜欢まいご吗？"
    print(LangSegmenter.getTexts(text))

    text = "ねえ、知ってる？最近、僕は天文学を勉強してるんだ。君の瞳が星空みたいにキラキラしてるからさ。"
    print(LangSegmenter.getTexts(text))