mirror of
https://github.com/RVC-Boss/GPT-SoVITS.git
synced 2026-11-17 18:43:37 +08:00
Compare commits
4 Commits
4aabecfcc7
...
1e85c3e58b
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1e85c3e58b | ||
|
|
6d51ffceaf | ||
|
|
0ba97c0ec5 | ||
|
|
09add9e2b4 |
@ -3,6 +3,7 @@ numpy<2.0
|
||||
scipy
|
||||
tensorboard
|
||||
librosa==0.10.2
|
||||
resampy>=0.4.2,<0.5
|
||||
numba
|
||||
pytorch-lightning>=2.4
|
||||
gradio<5
|
||||
|
||||
@ -10,9 +10,20 @@ def get_models():
|
||||
|
||||
|
||||
asr_dict = {
|
||||
"Fun-ASR-Nano (31语种+方言, 推荐)": {"lang": ["zh", "en", "ja", "ko", "yue", "auto"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
|
||||
"SenseVoice (极速, 5语种)": {"lang": ["zh", "en", "ja", "ko", "yue", "auto"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
|
||||
"达摩 ASR (中文经典)": {"lang": ["zh", "yue"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
|
||||
"达摩 ASR (中文经典)": {
|
||||
"lang": ["zh"],
|
||||
"size": ["large"],
|
||||
"path": "funasr_asr.py",
|
||||
"backend": "damo",
|
||||
"precision": ["float32"],
|
||||
},
|
||||
"Fun-ASR-Nano(速度更慢,更准)": {
|
||||
"lang": ["zh", "en", "ja", "ko", "yue", "auto"],
|
||||
"size": ["large"],
|
||||
"path": "funasr_asr.py",
|
||||
"backend": "fun-asr-nano",
|
||||
"precision": ["float32"],
|
||||
},
|
||||
"Faster Whisper (多语种)": {
|
||||
"lang": ["auto", "en", "ja", "ko"],
|
||||
"size": get_models(),
|
||||
|
||||
@ -40,6 +40,26 @@ language_code_list = [
|
||||
|
||||
|
||||
def download_model(model_size: str):
|
||||
model_path = ""
|
||||
if "distil" in model_size:
|
||||
if "3.5" in model_size:
|
||||
repo_id = "distil-whisper/distil-large-v3.5-ct2"
|
||||
model_path = "tools/asr/models/faster-distil-whisper-large-v3.5"
|
||||
else:
|
||||
repo_id = "Systran/faster-{}-whisper-{}".format(*model_size.split("-", maxsplit=1))
|
||||
elif model_size == "large-v3-turbo":
|
||||
repo_id = "mobiuslabsgmbh/faster-whisper-large-v3-turbo"
|
||||
model_path = "tools/asr/models/faster-whisper-large-v3-turbo"
|
||||
else:
|
||||
repo_id = f"Systran/faster-whisper-{model_size}"
|
||||
model_path = model_path or f"tools/asr/models/{repo_id.replace('Systran/', '').replace('distil-whisper/', '', 1)}"
|
||||
|
||||
if os.path.isfile(os.path.join(model_path, "config.json")) and os.path.isfile(
|
||||
os.path.join(model_path, "model.bin")
|
||||
):
|
||||
print(f"Using local Faster Whisper model: {model_path}")
|
||||
return model_path
|
||||
|
||||
url = "https://huggingface.co/api/models/gpt2"
|
||||
try:
|
||||
requests.get(url, timeout=3)
|
||||
@ -47,26 +67,6 @@ def download_model(model_size: str):
|
||||
except Exception:
|
||||
source = "ModelScope"
|
||||
|
||||
model_path = ""
|
||||
if source == "HF":
|
||||
if "distil" in model_size:
|
||||
if "3.5" in model_size:
|
||||
repo_id = "distil-whisper/distil-large-v3.5-ct2"
|
||||
model_path = "tools/asr/models/faster-distil-whisper-large-v3.5"
|
||||
else:
|
||||
repo_id = "Systran/faster-{}-whisper-{}".format(*model_size.split("-", maxsplit=1))
|
||||
elif model_size == "large-v3-turbo":
|
||||
repo_id = "mobiuslabsgmbh/faster-whisper-large-v3-turbo"
|
||||
model_path = "tools/asr/models/faster-whisper-large-v3-turbo"
|
||||
else:
|
||||
repo_id = f"Systran/faster-whisper-{model_size}"
|
||||
model_path = (
|
||||
model_path or f"tools/asr/models/{repo_id.replace('Systran/', '').replace('distil-whisper/', '', 1)}"
|
||||
)
|
||||
else:
|
||||
repo_id = "XXXXRT/faster-whisper"
|
||||
model_path = "tools/asr/models"
|
||||
|
||||
files: list[str] = [
|
||||
"config.json",
|
||||
"model.bin",
|
||||
@ -76,12 +76,8 @@ def download_model(model_size: str):
|
||||
if "large-v3" in model_size or "distil" in model_size:
|
||||
files.append("preprocessor_config.json")
|
||||
files.append("vocabulary.json")
|
||||
|
||||
files.remove("vocabulary.txt")
|
||||
|
||||
if source == "ModelScope":
|
||||
files = [f"faster-whisper-{model_size}/{file}".replace("whisper-distil", "distil-whisper") for file in files]
|
||||
|
||||
if source == "HF":
|
||||
print(f"Downloading model from HuggingFace: {repo_id} to {model_path}")
|
||||
snapshot_download_hf(
|
||||
@ -91,13 +87,14 @@ def download_model(model_size: str):
|
||||
allow_patterns=files,
|
||||
)
|
||||
else:
|
||||
print(f"Downloading model from ModelScope: {repo_id} to {model_path}")
|
||||
repo_id = "XXXXRT/faster-whisper"
|
||||
files = [f"faster-whisper-{model_size}/{file}".replace("whisper-distil", "distil-whisper") for file in files]
|
||||
print(f"Downloading model from ModelScope: {repo_id} to tools/asr/models")
|
||||
snapshot_download_ms(
|
||||
repo_id,
|
||||
local_dir=model_path,
|
||||
local_dir="tools/asr/models",
|
||||
allow_patterns=files,
|
||||
)
|
||||
return model_path + f"/faster-whisper-{model_size}".replace("whisper-distil", "distil-whisper")
|
||||
return model_path
|
||||
|
||||
|
||||
|
||||
@ -2,18 +2,45 @@
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import setuptools._distutils as _distutils
|
||||
import setuptools._distutils.util as _distutils_util
|
||||
import setuptools._distutils.version as _distutils_version
|
||||
|
||||
sys.modules.setdefault("distutils", _distutils)
|
||||
sys.modules.setdefault("distutils.util", _distutils_util)
|
||||
sys.modules.setdefault("distutils.version", _distutils_version)
|
||||
|
||||
from funasr import AutoModel
|
||||
from modelscope import snapshot_download
|
||||
from tqdm import tqdm
|
||||
|
||||
funasr_models = {} # 存储模型避免重复加载
|
||||
ASR_MODEL_ROOT = "tools/asr/models"
|
||||
FUN_ASR_NANO_MODEL_ID = "FunAudioLLM/Fun-ASR-Nano-2512"
|
||||
FUN_ASR_NANO_MODEL_SOURCES = (
|
||||
{"model": FUN_ASR_NANO_MODEL_ID, "hub": "hf", "trust_remote_code": True},
|
||||
{"model": FUN_ASR_NANO_MODEL_ID, "hub": "ms", "trust_remote_code": False},
|
||||
)
|
||||
VAD_MODEL_ID = "iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"
|
||||
PUNC_MODEL_ID = "iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
|
||||
PARAFORMER_MODEL_ID = "iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
|
||||
UNIASR_MODEL_ID = "iic/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
|
||||
FUN_ASR_NANO_MODEL_PATH = f"{ASR_MODEL_ROOT}/Fun-ASR-Nano-2512"
|
||||
VAD_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_fsmn_vad_zh-cn-16k-common-pytorch"
|
||||
PUNC_MODEL_PATH = f"{ASR_MODEL_ROOT}/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
|
||||
PARAFORMER_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
|
||||
UNIASR_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
|
||||
|
||||
|
||||
def ensure_model_dir(model_id, model_path, revision=None):
|
||||
if os.path.isfile(os.path.join(model_path, "model.pt")):
|
||||
return model_path
|
||||
|
||||
download_kwargs = {"local_dir": model_path}
|
||||
if revision is not None:
|
||||
download_kwargs["revision"] = revision
|
||||
print(f"Downloading FunASR model: {model_id} -> {model_path}")
|
||||
snapshot_download(model_id, **download_kwargs)
|
||||
return model_path
|
||||
|
||||
|
||||
def only_asr(input_file, language, backend="fun-asr-nano"):
|
||||
@ -26,100 +53,86 @@ def only_asr(input_file, language, backend="fun-asr-nano"):
|
||||
return text
|
||||
|
||||
|
||||
def is_unregistered_model_error(exc):
|
||||
message = str(exc).lower()
|
||||
return "model" in message and "not registered" in message
|
||||
def disable_flash_attention_probe():
|
||||
import transformers.utils
|
||||
import transformers.utils.import_utils
|
||||
|
||||
transformers.utils.import_utils.is_flash_attn_2_available = lambda: False
|
||||
transformers.utils.is_flash_attn_2_available = lambda: False
|
||||
|
||||
|
||||
def create_fun_asr_nano_model(device):
|
||||
common_kwargs = dict(
|
||||
vad_model="fsmn-vad",
|
||||
disable_flash_attention_probe()
|
||||
ensure_model_dir(FUN_ASR_NANO_MODEL_ID, FUN_ASR_NANO_MODEL_PATH)
|
||||
ensure_model_dir(VAD_MODEL_ID, VAD_MODEL_PATH)
|
||||
return AutoModel(
|
||||
model=FUN_ASR_NANO_MODEL_PATH,
|
||||
vad_model=VAD_MODEL_PATH,
|
||||
device=device,
|
||||
disable_update=True,
|
||||
check_latest=False,
|
||||
vad_kwargs={"check_latest": False},
|
||||
hub="ms",
|
||||
)
|
||||
last_error = None
|
||||
for model_source in FUN_ASR_NANO_MODEL_SOURCES:
|
||||
try:
|
||||
return AutoModel(**model_source, **common_kwargs)
|
||||
except (KeyError, RuntimeError, ValueError) as exc:
|
||||
if not is_unregistered_model_error(exc):
|
||||
raise
|
||||
last_error = exc
|
||||
|
||||
raise last_error
|
||||
|
||||
def create_damo_model(device):
|
||||
ensure_model_dir(VAD_MODEL_ID, VAD_MODEL_PATH, revision="v2.0.4")
|
||||
ensure_model_dir(PUNC_MODEL_ID, PUNC_MODEL_PATH, revision="v2.0.4")
|
||||
ensure_model_dir(PARAFORMER_MODEL_ID, PARAFORMER_MODEL_PATH, revision="v2.0.4")
|
||||
return AutoModel(
|
||||
model=PARAFORMER_MODEL_PATH,
|
||||
model_revision="v2.0.4",
|
||||
vad_model=VAD_MODEL_PATH,
|
||||
vad_model_revision="v2.0.4",
|
||||
vad_kwargs={"check_latest": False},
|
||||
punc_model=PUNC_MODEL_PATH,
|
||||
punc_model_revision="v2.0.4",
|
||||
punc_kwargs={"check_latest": False},
|
||||
device=device,
|
||||
disable_update=True,
|
||||
check_latest=False,
|
||||
hub="ms",
|
||||
)
|
||||
|
||||
|
||||
def create_uniasr_model(device):
|
||||
ensure_model_dir(UNIASR_MODEL_ID, UNIASR_MODEL_PATH, revision="master")
|
||||
return AutoModel(
|
||||
model=UNIASR_MODEL_PATH,
|
||||
model_revision="master",
|
||||
device=device,
|
||||
disable_update=True,
|
||||
check_latest=False,
|
||||
hub="ms",
|
||||
)
|
||||
|
||||
|
||||
def create_model(language="zh", **kwargs):
|
||||
backend = kwargs.get("backend", "fun-asr-nano")
|
||||
import torch
|
||||
|
||||
# For non-classic backends, route to multilingual models regardless of language
|
||||
if backend in ("fun-asr-nano", "sensevoice") and language != "yue":
|
||||
import torch
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
cache_key = f"{language}_{backend}"
|
||||
if cache_key in funasr_models:
|
||||
return funasr_models[cache_key]
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
cache_key = f"{backend}_{language}"
|
||||
if cache_key in funasr_models:
|
||||
return funasr_models[cache_key]
|
||||
|
||||
if backend == "fun-asr-nano":
|
||||
model = create_fun_asr_nano_model(device)
|
||||
print(f"FunASR Fun-ASR-Nano 模型加载完成: {language.upper()}")
|
||||
else:
|
||||
model = AutoModel(
|
||||
model="iic/SenseVoiceSmall",
|
||||
vad_model="fsmn-vad",
|
||||
device=device,
|
||||
disable_update=True,
|
||||
)
|
||||
print(f"FunASR SenseVoice 模型加载完成: {language.upper()}")
|
||||
|
||||
funasr_models[cache_key] = model
|
||||
return model
|
||||
|
||||
if language == "zh":
|
||||
path_vad = "tools/asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch"
|
||||
path_punc = "tools/asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
|
||||
path_asr = "tools/asr/models/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
|
||||
snapshot_download(
|
||||
"iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
|
||||
local_dir="tools/asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch",
|
||||
)
|
||||
snapshot_download(
|
||||
"iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
|
||||
local_dir="tools/asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
|
||||
)
|
||||
snapshot_download(
|
||||
"iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
|
||||
local_dir="tools/asr/models/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
|
||||
)
|
||||
model_revision = "v2.0.4"
|
||||
vad_model_revision = punc_model_revision = "v2.0.4"
|
||||
elif language == "yue":
|
||||
path_asr = "tools/asr/models/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
|
||||
snapshot_download(
|
||||
"iic/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
|
||||
local_dir="tools/asr/models/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
|
||||
)
|
||||
path_vad = path_punc = None
|
||||
vad_model_revision = punc_model_revision = ""
|
||||
model_revision = "master"
|
||||
if backend == "damo":
|
||||
if language != "zh":
|
||||
raise ValueError("达摩 ASR 仅支持 zh")
|
||||
model = create_damo_model(device)
|
||||
print(f"FunASR 达摩 Paraformer 模型加载完成: {language.upper()}")
|
||||
elif backend == "fun-asr-nano" and language == "yue":
|
||||
model = create_uniasr_model(device)
|
||||
print(f"FunASR UniASR 粤语模型加载完成: {language.upper()}")
|
||||
elif backend == "fun-asr-nano":
|
||||
model = create_fun_asr_nano_model(device)
|
||||
print(f"FunASR Fun-ASR-Nano 模型加载完成: {language.upper()}")
|
||||
else:
|
||||
raise ValueError(f"{language} is not supported. Supported: zh, yue, ja, en, ko, auto")
|
||||
raise ValueError(f"{backend} is not supported")
|
||||
|
||||
if language in funasr_models:
|
||||
return funasr_models[language]
|
||||
else:
|
||||
model = AutoModel(
|
||||
model=path_asr,
|
||||
model_revision=model_revision,
|
||||
vad_model=path_vad,
|
||||
vad_model_revision=vad_model_revision,
|
||||
punc_model=path_punc,
|
||||
punc_model_revision=punc_model_revision,
|
||||
)
|
||||
print(f"FunASR 模型加载完成: {language.upper()}")
|
||||
|
||||
funasr_models[language] = model
|
||||
return model
|
||||
funasr_models[cache_key] = model
|
||||
return model
|
||||
|
||||
|
||||
def execute_asr(input_folder, output_folder, model_size, language, backend="fun-asr-nano"):
|
||||
@ -163,10 +176,14 @@ if __name__ == "__main__":
|
||||
parser.add_argument(
|
||||
"-p", "--precision", type=str, default="float16", choices=["float16", "float32"], help="fp16 or fp32"
|
||||
) # 还没接入
|
||||
parser.add_argument(
|
||||
"--backend", type=str, default="fun-asr-nano", choices=["fun-asr-nano", "damo"], help="ASR backend"
|
||||
)
|
||||
cmd = parser.parse_args()
|
||||
execute_asr(
|
||||
input_folder=cmd.input_folder,
|
||||
output_folder=cmd.output_folder,
|
||||
model_size=cmd.model_size,
|
||||
language=cmd.language,
|
||||
backend=cmd.backend,
|
||||
)
|
||||
|
||||
4
webui.py
4
webui.py
@ -381,6 +381,8 @@ def open_asr(asr_inp_dir, asr_opt_dir, asr_model, asr_model_size, asr_lang, asr_
|
||||
cmd += f" -s {asr_model_size}"
|
||||
cmd += f" -l {asr_lang}"
|
||||
cmd += f" -p {asr_precision}"
|
||||
if "backend" in asr_dict[asr_model]:
|
||||
cmd += f" --backend {asr_dict[asr_model]['backend']}"
|
||||
output_file_name = os.path.basename(asr_inp_dir)
|
||||
output_folder = asr_opt_dir or "output/asr_opt"
|
||||
output_file_path = os.path.abspath(f"{output_folder}/{output_file_name}.list")
|
||||
@ -1412,7 +1414,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
|
||||
label=i18n("ASR 模型"),
|
||||
choices=list(asr_dict.keys()),
|
||||
interactive=True,
|
||||
value="达摩 ASR (中文)",
|
||||
value="达摩 ASR (中文经典)",
|
||||
)
|
||||
asr_size = gr.Dropdown(
|
||||
label=i18n("ASR 模型尺寸"), choices=["large"], interactive=True, value="large"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user