Compare commits

...

3 Commits

Author SHA1 Message Date
jackwu982
8c10a6c9a4
Merge 61afcf2600646fca8e27b412cd9564fefb6a6678 into 6d51ffceafbf240145619af4c92408a6e54b7924 2026-11-12 10:12:52 +08:00
RVC-Boss
6d51ffceaf Update ASR configuration, Faster Whisper, FunASR and WebUI from update_asr package 2026-11-12 00:26:17 +08:00
wuyuefeng
61afcf2600 docs(api): 修正 API 文档中的参数格式错误
- 修正了 API_V2 接口中 text_lang 和 split_bucket 参数的格式错误
2024-11-21 13:53:44 +08:00
5 changed files with 143 additions and 116 deletions

View File

@ -22,7 +22,7 @@ POST:
```json
{
"text": "", # str.(required) text to be synthesized
"text_lang: "", # str.(required) language of the text to be synthesized
"text_lang": "", # str.(required) language of the text to be synthesized
"ref_audio_path": "", # str.(required) reference audio path
"aux_ref_audio_paths": [], # list.(optional) auxiliary reference audio paths for multi-speaker tone fusion
"prompt_text": "", # str.(optional) prompt text for the reference audio
@ -434,7 +434,7 @@ async def tts_handle(req: dict, raw_streaming_mode=None):
req (dict):
{
"text": "", # str.(required) text to be synthesized
"text_lang: "", # str.(required) language of the text to be synthesized
"text_lang": "", # str.(required) language of the text to be synthesized
"ref_audio_path": "", # str.(required) reference audio path
"aux_ref_audio_paths": [], # list.(optional) auxiliary reference audio paths for multi-speaker tone fusion
"prompt_text": "", # str.(optional) prompt text for the reference audio

View File

@ -10,9 +10,20 @@ def get_models():
asr_dict = {
"Fun-ASR-Nano (31语种+方言, 推荐)": {"lang": ["zh", "en", "ja", "ko", "yue", "auto"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
"SenseVoice (极速, 5语种)": {"lang": ["zh", "en", "ja", "ko", "yue", "auto"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
"达摩 ASR (中文经典)": {"lang": ["zh", "yue"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
"达摩 ASR (中文经典)": {
"lang": ["zh"],
"size": ["large"],
"path": "funasr_asr.py",
"backend": "damo",
"precision": ["float32"],
},
"Fun-ASR-Nano(速度更慢,更准)": {
"lang": ["zh", "en", "ja", "ko", "yue", "auto"],
"size": ["large"],
"path": "funasr_asr.py",
"backend": "fun-asr-nano",
"precision": ["float32"],
},
"Faster Whisper (多语种)": {
"lang": ["auto", "en", "ja", "ko"],
"size": get_models(),

View File

@ -40,6 +40,26 @@ language_code_list = [
def download_model(model_size: str):
model_path = ""
if "distil" in model_size:
if "3.5" in model_size:
repo_id = "distil-whisper/distil-large-v3.5-ct2"
model_path = "tools/asr/models/faster-distil-whisper-large-v3.5"
else:
repo_id = "Systran/faster-{}-whisper-{}".format(*model_size.split("-", maxsplit=1))
elif model_size == "large-v3-turbo":
repo_id = "mobiuslabsgmbh/faster-whisper-large-v3-turbo"
model_path = "tools/asr/models/faster-whisper-large-v3-turbo"
else:
repo_id = f"Systran/faster-whisper-{model_size}"
model_path = model_path or f"tools/asr/models/{repo_id.replace('Systran/', '').replace('distil-whisper/', '', 1)}"
if os.path.isfile(os.path.join(model_path, "config.json")) and os.path.isfile(
os.path.join(model_path, "model.bin")
):
print(f"Using local Faster Whisper model: {model_path}")
return model_path
url = "https://huggingface.co/api/models/gpt2"
try:
requests.get(url, timeout=3)
@ -47,26 +67,6 @@ def download_model(model_size: str):
except Exception:
source = "ModelScope"
model_path = ""
if source == "HF":
if "distil" in model_size:
if "3.5" in model_size:
repo_id = "distil-whisper/distil-large-v3.5-ct2"
model_path = "tools/asr/models/faster-distil-whisper-large-v3.5"
else:
repo_id = "Systran/faster-{}-whisper-{}".format(*model_size.split("-", maxsplit=1))
elif model_size == "large-v3-turbo":
repo_id = "mobiuslabsgmbh/faster-whisper-large-v3-turbo"
model_path = "tools/asr/models/faster-whisper-large-v3-turbo"
else:
repo_id = f"Systran/faster-whisper-{model_size}"
model_path = (
model_path or f"tools/asr/models/{repo_id.replace('Systran/', '').replace('distil-whisper/', '', 1)}"
)
else:
repo_id = "XXXXRT/faster-whisper"
model_path = "tools/asr/models"
files: list[str] = [
"config.json",
"model.bin",
@ -76,12 +76,8 @@ def download_model(model_size: str):
if "large-v3" in model_size or "distil" in model_size:
files.append("preprocessor_config.json")
files.append("vocabulary.json")
files.remove("vocabulary.txt")
if source == "ModelScope":
files = [f"faster-whisper-{model_size}/{file}".replace("whisper-distil", "distil-whisper") for file in files]
if source == "HF":
print(f"Downloading model from HuggingFace: {repo_id} to {model_path}")
snapshot_download_hf(
@ -91,13 +87,14 @@ def download_model(model_size: str):
allow_patterns=files,
)
else:
print(f"Downloading model from ModelScope: {repo_id} to {model_path}")
repo_id = "XXXXRT/faster-whisper"
files = [f"faster-whisper-{model_size}/{file}".replace("whisper-distil", "distil-whisper") for file in files]
print(f"Downloading model from ModelScope: {repo_id} to tools/asr/models")
snapshot_download_ms(
repo_id,
local_dir=model_path,
local_dir="tools/asr/models",
allow_patterns=files,
)
return model_path + f"/faster-whisper-{model_size}".replace("whisper-distil", "distil-whisper")
return model_path

View File

@ -2,18 +2,45 @@
import argparse
import os
import sys
import traceback
import setuptools._distutils as _distutils
import setuptools._distutils.util as _distutils_util
import setuptools._distutils.version as _distutils_version
sys.modules.setdefault("distutils", _distutils)
sys.modules.setdefault("distutils.util", _distutils_util)
sys.modules.setdefault("distutils.version", _distutils_version)
from funasr import AutoModel
from modelscope import snapshot_download
from tqdm import tqdm
funasr_models = {} # 存储模型避免重复加载
ASR_MODEL_ROOT = "tools/asr/models"
FUN_ASR_NANO_MODEL_ID = "FunAudioLLM/Fun-ASR-Nano-2512"
FUN_ASR_NANO_MODEL_SOURCES = (
{"model": FUN_ASR_NANO_MODEL_ID, "hub": "hf", "trust_remote_code": True},
{"model": FUN_ASR_NANO_MODEL_ID, "hub": "ms", "trust_remote_code": False},
)
VAD_MODEL_ID = "iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"
PUNC_MODEL_ID = "iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
PARAFORMER_MODEL_ID = "iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
UNIASR_MODEL_ID = "iic/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
FUN_ASR_NANO_MODEL_PATH = f"{ASR_MODEL_ROOT}/Fun-ASR-Nano-2512"
VAD_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_fsmn_vad_zh-cn-16k-common-pytorch"
PUNC_MODEL_PATH = f"{ASR_MODEL_ROOT}/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
PARAFORMER_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
UNIASR_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
def ensure_model_dir(model_id, model_path, revision=None):
if os.path.isfile(os.path.join(model_path, "model.pt")):
return model_path
download_kwargs = {"local_dir": model_path}
if revision is not None:
download_kwargs["revision"] = revision
print(f"Downloading FunASR model: {model_id} -> {model_path}")
snapshot_download(model_id, **download_kwargs)
return model_path
def only_asr(input_file, language, backend="fun-asr-nano"):
@ -26,100 +53,86 @@ def only_asr(input_file, language, backend="fun-asr-nano"):
return text
def is_unregistered_model_error(exc):
message = str(exc).lower()
return "model" in message and "not registered" in message
def disable_flash_attention_probe():
import transformers.utils
import transformers.utils.import_utils
transformers.utils.import_utils.is_flash_attn_2_available = lambda: False
transformers.utils.is_flash_attn_2_available = lambda: False
def create_fun_asr_nano_model(device):
common_kwargs = dict(
vad_model="fsmn-vad",
disable_flash_attention_probe()
ensure_model_dir(FUN_ASR_NANO_MODEL_ID, FUN_ASR_NANO_MODEL_PATH)
ensure_model_dir(VAD_MODEL_ID, VAD_MODEL_PATH)
return AutoModel(
model=FUN_ASR_NANO_MODEL_PATH,
vad_model=VAD_MODEL_PATH,
device=device,
disable_update=True,
check_latest=False,
vad_kwargs={"check_latest": False},
hub="ms",
)
last_error = None
for model_source in FUN_ASR_NANO_MODEL_SOURCES:
try:
return AutoModel(**model_source, **common_kwargs)
except (KeyError, RuntimeError, ValueError) as exc:
if not is_unregistered_model_error(exc):
raise
last_error = exc
raise last_error
def create_damo_model(device):
ensure_model_dir(VAD_MODEL_ID, VAD_MODEL_PATH, revision="v2.0.4")
ensure_model_dir(PUNC_MODEL_ID, PUNC_MODEL_PATH, revision="v2.0.4")
ensure_model_dir(PARAFORMER_MODEL_ID, PARAFORMER_MODEL_PATH, revision="v2.0.4")
return AutoModel(
model=PARAFORMER_MODEL_PATH,
model_revision="v2.0.4",
vad_model=VAD_MODEL_PATH,
vad_model_revision="v2.0.4",
vad_kwargs={"check_latest": False},
punc_model=PUNC_MODEL_PATH,
punc_model_revision="v2.0.4",
punc_kwargs={"check_latest": False},
device=device,
disable_update=True,
check_latest=False,
hub="ms",
)
def create_uniasr_model(device):
ensure_model_dir(UNIASR_MODEL_ID, UNIASR_MODEL_PATH, revision="master")
return AutoModel(
model=UNIASR_MODEL_PATH,
model_revision="master",
device=device,
disable_update=True,
check_latest=False,
hub="ms",
)
def create_model(language="zh", **kwargs):
backend = kwargs.get("backend", "fun-asr-nano")
import torch
# For non-classic backends, route to multilingual models regardless of language
if backend in ("fun-asr-nano", "sensevoice") and language != "yue":
import torch
device = "cuda" if torch.cuda.is_available() else "cpu"
cache_key = f"{language}_{backend}"
if cache_key in funasr_models:
return funasr_models[cache_key]
device = "cuda" if torch.cuda.is_available() else "cpu"
cache_key = f"{backend}_{language}"
if cache_key in funasr_models:
return funasr_models[cache_key]
if backend == "fun-asr-nano":
model = create_fun_asr_nano_model(device)
print(f"FunASR Fun-ASR-Nano 模型加载完成: {language.upper()}")
else:
model = AutoModel(
model="iic/SenseVoiceSmall",
vad_model="fsmn-vad",
device=device,
disable_update=True,
)
print(f"FunASR SenseVoice 模型加载完成: {language.upper()}")
funasr_models[cache_key] = model
return model
if language == "zh":
path_vad = "tools/asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch"
path_punc = "tools/asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
path_asr = "tools/asr/models/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
snapshot_download(
"iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
local_dir="tools/asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch",
)
snapshot_download(
"iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
local_dir="tools/asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
)
snapshot_download(
"iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
local_dir="tools/asr/models/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
)
model_revision = "v2.0.4"
vad_model_revision = punc_model_revision = "v2.0.4"
elif language == "yue":
path_asr = "tools/asr/models/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
snapshot_download(
"iic/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
local_dir="tools/asr/models/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
)
path_vad = path_punc = None
vad_model_revision = punc_model_revision = ""
model_revision = "master"
if backend == "damo":
if language != "zh":
raise ValueError("达摩 ASR 仅支持 zh")
model = create_damo_model(device)
print(f"FunASR 达摩 Paraformer 模型加载完成: {language.upper()}")
elif backend == "fun-asr-nano" and language == "yue":
model = create_uniasr_model(device)
print(f"FunASR UniASR 粤语模型加载完成: {language.upper()}")
elif backend == "fun-asr-nano":
model = create_fun_asr_nano_model(device)
print(f"FunASR Fun-ASR-Nano 模型加载完成: {language.upper()}")
else:
raise ValueError(f"{language} is not supported. Supported: zh, yue, ja, en, ko, auto")
raise ValueError(f"{backend} is not supported")
if language in funasr_models:
return funasr_models[language]
else:
model = AutoModel(
model=path_asr,
model_revision=model_revision,
vad_model=path_vad,
vad_model_revision=vad_model_revision,
punc_model=path_punc,
punc_model_revision=punc_model_revision,
)
print(f"FunASR 模型加载完成: {language.upper()}")
funasr_models[language] = model
return model
funasr_models[cache_key] = model
return model
def execute_asr(input_folder, output_folder, model_size, language, backend="fun-asr-nano"):
@ -163,10 +176,14 @@ if __name__ == "__main__":
parser.add_argument(
"-p", "--precision", type=str, default="float16", choices=["float16", "float32"], help="fp16 or fp32"
) # 还没接入
parser.add_argument(
"--backend", type=str, default="fun-asr-nano", choices=["fun-asr-nano", "damo"], help="ASR backend"
)
cmd = parser.parse_args()
execute_asr(
input_folder=cmd.input_folder,
output_folder=cmd.output_folder,
model_size=cmd.model_size,
language=cmd.language,
backend=cmd.backend,
)

View File

@ -381,6 +381,8 @@ def open_asr(asr_inp_dir, asr_opt_dir, asr_model, asr_model_size, asr_lang, asr_
cmd += f" -s {asr_model_size}"
cmd += f" -l {asr_lang}"
cmd += f" -p {asr_precision}"
if "backend" in asr_dict[asr_model]:
cmd += f" --backend {asr_dict[asr_model]['backend']}"
output_file_name = os.path.basename(asr_inp_dir)
output_folder = asr_opt_dir or "output/asr_opt"
output_file_path = os.path.abspath(f"{output_folder}/{output_file_name}.list")
@ -1412,7 +1414,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
label=i18n("ASR 模型"),
choices=list(asr_dict.keys()),
interactive=True,
value="达摩 ASR (中文)",
value="达摩 ASR (中文经典)",
)
asr_size = gr.Dropdown(
label=i18n("ASR 模型尺寸"), choices=["large"], interactive=True, value="large"