Integrate v5 training and inference updates with single-GPU training support

This commit is contained in:
RVC-Boss 2026-10-06 16:39:59 +08:00
parent a9beed7c44
commit 3b95794c49
14 changed files with 515 additions and 417 deletions

View File

@ -31,10 +31,7 @@ from BigVGAN.bigvgan import BigVGAN
from feature_extractor.cnhubert import CNHubert
from module.mel_processing import mel_spectrogram_torch, spectrogram_torch
from module.models import SynthesizerTrn, SynthesizerTrnV3, Generator
from module.v5_inference import (
synthesize_v5_mel, V5_DEFAULT_CFG,
V5_REQUEST_DEFAULT_CFG, V5_VERSIONS, sampling_defaults, resolve_sampling,
)
from module.models_v5 import synthesize_v5_mel, V5_VERSIONS
from peft import LoraConfig, get_peft_model
from process_ckpt import get_sovits_version_from_path_fast, load_sovits_new
from transformers import AutoModelForMaskedLM, AutoTokenizer
@ -353,10 +350,9 @@ class TTS_Config:
},
}
for v5_variant in V5_VERSIONS:
model_name = "v5dev" if v5_variant == "v5" else v5_variant
default_configs[v5_variant] = dict(
default_configs["v4"], version=v5_variant,
vits_weights_path=f"GPT_SoVITS/pretrained_models/gsv-v5-pretrained/s2G{model_name}.pth",
vits_weights_path=f"GPT_SoVITS/pretrained_models/gsv-v5-pretrained/s2G{v5_variant}.pth",
)
configs: dict = None
v1_languages: list = ["auto", "en", "zh", "ja", "all_zh", "all_ja"]
@ -369,7 +365,6 @@ class TTS_Config:
"v2ProPlus": 486,
"v3" : 486,
"v4" : 486,
"v5" : 486,
"v5dev": 486,
"v5turbo": 486,
}
@ -420,7 +415,7 @@ class TTS_Config:
version = self.configs.get("version", None)
self.version = version
assert self.version in ["v1", "v2", "v3", "v4", "v5", "v5dev", "v5turbo", "v2Pro", "v2ProPlus"], "Invalid version!"
assert self.version in ["v1", "v2", "v3", "v4", "v5dev", "v5turbo", "v2Pro", "v2ProPlus"], "Invalid version!"
self.t2s_weights_path = self.configs.get("t2s_weights_path", None)
self.vits_weights_path = self.configs.get("vits_weights_path", None)
self.bert_base_path = self.configs.get("bert_base_path", None)
@ -1161,9 +1156,10 @@ class TTS:
use_cuda_graph = inputs.get("use_cuda_graph", True)
use_flash_attention = inputs.get("use_flash_attention", True)
repetition_penalty = inputs.get("repetition_penalty", 1.35)
sample_steps, cfg_rate = resolve_sampling(
self.configs.version, inputs.get("sample_steps"), inputs.get("cfg_rate"),
)
sample_steps = inputs.get("sample_steps")
cfg_rate = inputs.get("cfg_rate")
sample_steps = (4 if self.configs.version == "v5turbo" else 32) if sample_steps is None else int(sample_steps)
cfg_rate = (1.30 if self.configs.version == "v5dev" else 0.0) if cfg_rate is None else float(cfg_rate)
super_sampling = inputs.get("super_sampling", False)
streaming_mode = inputs.get("streaming_mode", False)
overlap_length = inputs.get("overlap_length", 2)
@ -1744,7 +1740,8 @@ class TTS:
self, semantic_tokens: torch.Tensor, phones: torch.Tensor, speed: float = 1.0,
sample_steps: int = None, cfg_rate: float = None
):
sample_steps, cfg_rate = resolve_sampling(self.configs.version, sample_steps, cfg_rate)
sample_steps = (4 if self.configs.version == "v5turbo" else 32) if sample_steps is None else int(sample_steps)
cfg_rate = (1.30 if self.configs.version == "v5dev" else 0.0) if cfg_rate is None else float(cfg_rate)
prompt_semantic_tokens = self.prompt_cache["prompt_semantic"].unsqueeze(0).unsqueeze(0).to(self.configs.device)
prompt_phones = torch.LongTensor(self.prompt_cache["phones"]).unsqueeze(0).to(self.configs.device)
raw_entry = self.prompt_cache["refer_spec"][0]
@ -1823,7 +1820,8 @@ class TTS:
sample_steps: int = None,
cfg_rate: float = None,
) -> List[torch.Tensor]:
sample_steps, cfg_rate = resolve_sampling(self.configs.version, sample_steps, cfg_rate)
sample_steps = (4 if self.configs.version == "v5turbo" else 32) if sample_steps is None else int(sample_steps)
cfg_rate = (1.30 if self.configs.version == "v5dev" else 0.0) if cfg_rate is None else float(cfg_rate)
if self.configs.version in V5_VERSIONS:
return [
self.using_vocoder_synthesis(

View File

@ -160,8 +160,8 @@ class DiT(nn.Module):
cond0: float["b n d"], # masked cond audio # noqa: F722
x_lens,
time: float["b"] | float[""], # time step # noqa: F821 F722
dt_base_bootstrap,
text0, # : int["b nt"] # noqa: F722#####condition feature
dt_base_bootstrap=None,
text0=None, # : int["b nt"] # noqa: F722#####condition feature
use_grad_ckpt=False, # bool
###no-use
drop_audio_cond=False, # cfg for cond audio

View File

@ -53,11 +53,6 @@ from config import change_choices, get_weights_names, name2gpt_path, name2sovits
SoVITS_names, GPT_names = get_weights_names()
from config import pretrained_sovits_name
path_sovits_v3 = pretrained_sovits_name["v3"]
path_sovits_v4 = pretrained_sovits_name["v4"]
is_exist_s2gv3 = os.path.exists(path_sovits_v3)
is_exist_s2gv4 = os.path.exists(path_sovits_v4)
if os.path.exists("./weight.json"):
pass
else:
@ -91,7 +86,7 @@ is_share = os.environ.get("is_share", "False")
is_share = eval(is_share)
if "_CUDA_VISIBLE_DEVICES" in os.environ:
os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["_CUDA_VISIBLE_DEVICES"]
is_half = eval(os.environ.get("is_half", "True")) and torch.cuda.is_available()
is_half = eval(os.environ.get("is_half", "True")) and torch.cuda.is_available()
# is_half=False
punctuation = set(["!", "?", "…", ",", ".", "-", " "])
import gradio as gr
@ -105,11 +100,8 @@ cnhubert.cnhubert_base_path = cnhubert_base_path
import random
from tools.acceleration import create_acceleration, cuda_graph_available, flash_attention_available
from GPT_SoVITS.module.models import Generator, SynthesizerTrn, SynthesizerTrnV3
from GPT_SoVITS.module.v5_inference import (
synthesize_v5_mel, V5_DEFAULT_CFG,
V5_REQUEST_DEFAULT_CFG, V5_VERSIONS, sampling_defaults, resolve_sampling,
)
from module.models import Generator, SynthesizerTrn, SynthesizerTrnV3
from module.models_v5 import synthesize_v5_mel, V5_VERSIONS
def set_seed(seed):
@ -237,7 +229,7 @@ else:
# symbol_version-model_version-if_lora_v3
from process_ckpt import get_sovits_version_from_path_fast, load_sovits_new
v3v4set = {"v3", "v4"} | V5_VERSIONS
v3v4set = {"v3", "v4"} | V5_VERSIONS
t2s_model_cudagraph = None
gpt_path_global = None
@ -251,16 +243,16 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None)
t2s_model_cudagraph = None
version, model_version, if_lora_v3 = get_sovits_version_from_path_fast(sovits_path)
print(sovits_path, version, model_version, if_lora_v3)
is_exist = is_exist_s2gv3 if model_version == "v3" else is_exist_s2gv4
path_sovits = path_sovits_v3 if model_version == "v3" else path_sovits_v4
path_sovits = pretrained_sovits_name[model_version]
is_exist = os.path.exists(path_sovits)
if if_lora_v3 == True and is_exist == False:
info = path_sovits + "SoVITS %s" % model_version + i18n("底模缺失,无法加载相应 LoRA 权重")
gr.Warning(info)
raise FileExistsError(info)
dict_language = dict_language_v1 if version == "v1" else dict_language_v2
prompt_text_update = prompt_language_update = text_update = text_language_update = {"__type__": "update"}
visible_sample_steps = model_version in v3v4set
visible_inp_refs = not visible_sample_steps
dict_language = dict_language_v1 if version == "v1" else dict_language_v2
prompt_text_update = prompt_language_update = text_update = text_language_update = {"__type__": "update"}
visible_sample_steps = model_version in v3v4set
visible_inp_refs = not visible_sample_steps
if prompt_language is not None and text_language is not None:
if prompt_language in list(dict_language.keys()):
prompt_text_update, prompt_language_update = (
@ -291,14 +283,14 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None)
{
"__type__": "update",
"visible": visible_sample_steps,
"value": sampling_defaults(model_version, 32 if model_version == "v3" else 8)[0],
"choices": [4, 8, 16, 32],
"value": (4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8),
"choices": [4, 8, 16, 32],
},
{"__type__": "update", "visible": visible_inp_refs},
{"__type__": "update", "value": False, "interactive": True if model_version not in v3v4set else False},
{"__type__": "update", "visible": True if model_version == "v3" else False},
{"__type__": "update", "value": i18n("模型加载中,请等待"), "interactive": False},
{"__type__": "update", "visible": model_version in V5_VERSIONS, "value": sampling_defaults(model_version)[1]},
{"__type__": "update", "value": i18n("模型加载中,请等待"), "interactive": False},
{"__type__": "update", "visible": model_version in V5_VERSIONS, "value": (1.30 if model_version == "v5dev" else 0.0)},
)
dict_s2 = load_sovits_new(sovits_path)
@ -335,17 +327,16 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None)
if "pretrained" not in sovits_path:
try:
del vq_model.enc_q
except:
pass
if is_half == True:
vq_model = vq_model.half().to(device)
else:
vq_model = vq_model.to(device)
except:
pass
if is_half == True:
vq_model = vq_model.half().to(device)
else:
vq_model = vq_model.to(device)
vq_model.eval()
if if_lora_v3 == False:
print("loading sovits_%s" % model_version, vq_model.load_state_dict(dict_s2["weight"], strict=False))
else:
path_sovits = path_sovits_v3 if model_version == "v3" else path_sovits_v4
print(
"loading sovits_%spretrained_G" % model_version,
vq_model.load_state_dict(load_sovits_new(path_sovits)["weight"], strict=False),
@ -374,14 +365,14 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None)
{
"__type__": "update",
"visible": visible_sample_steps,
"value": sampling_defaults(model_version, 32 if model_version == "v3" else 8)[0],
"choices": [4, 8, 16, 32],
"value": (4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8),
"choices": [4, 8, 16, 32],
},
{"__type__": "update", "visible": visible_inp_refs},
{"__type__": "update", "value": False, "interactive": True if model_version not in v3v4set else False},
{"__type__": "update", "visible": True if model_version == "v3" else False},
{"__type__": "update", "value": i18n("合成语音"), "interactive": True},
{"__type__": "update", "visible": model_version in V5_VERSIONS, "value": sampling_defaults(model_version)[1]},
{"__type__": "update", "value": i18n("合成语音"), "interactive": True},
{"__type__": "update", "visible": model_version in V5_VERSIONS, "value": (1.30 if model_version == "v5dev" else 0.0)},
)
with open("./weight.json") as f:
data = f.read()
@ -484,9 +475,9 @@ def init_bigvgan():
bigvgan_model = bigvgan_model.to(device)
def init_hifigan():
global hifigan_model, bigvgan_model, sv_cn_model, hifigan_version
clean_hifigan_model()
def init_hifigan():
global hifigan_model, bigvgan_model, sv_cn_model, hifigan_version
clean_hifigan_model()
hifigan_model = Generator(
initial_channel=100,
resblock="1",
@ -501,17 +492,17 @@ def init_hifigan():
hifigan_model.eval()
hifigan_model.remove_weight_norm()
state_dict_g = torch.load(
"%s/GPT_SoVITS/pretrained_models/gsv-%s-pretrained/vocoder.pth" %
(now_dir, "v5" if model_version in V5_VERSIONS else model_version),
"%s/GPT_SoVITS/pretrained_models/gsv-%s-pretrained/vocoder.pth" %
(now_dir, "v5" if model_version in V5_VERSIONS else model_version),
map_location="cpu",
weights_only=False,
)
print("loading vocoder", hifigan_model.load_state_dict(state_dict_g))
hifigan_version = model_version
print("loading vocoder", hifigan_model.load_state_dict(state_dict_g))
hifigan_version = model_version
clean_bigvgan_model()
clean_sv_cn_model()
if is_half == True:
hifigan_model = hifigan_model.half().to(device)
if is_half == True:
hifigan_model = hifigan_model.half().to(device)
else:
hifigan_model = hifigan_model.to(device)
@ -526,95 +517,95 @@ def init_sv_cn():
clean_hifigan_model()
bigvgan_model = hifigan_model = sv_cn_model = None
hifigan_version = None
bigvgan_model = hifigan_model = sv_cn_model = None
hifigan_version = None
if model_version == "v3":
init_bigvgan()
if model_version in ({"v4"} | V5_VERSIONS):
if model_version in ({"v4"} | V5_VERSIONS):
init_hifigan()
if model_version in {"v2Pro", "v2ProPlus"}:
init_sv_cn()
resample_transform_dict = {}
resampy_filter_cache = {}
resample_transform_dict = {}
resampy_filter_cache = {}
def _resampy_gpu_filter(sr0, sr1, device):
key = "%s-%s-%s" % (sr0, sr1, str(device))
if key not in resampy_filter_cache:
import resampy
interp_win, precision, rolloff = resampy.filters.get_filter("kaiser_best")
interp_win = torch.from_numpy(interp_win.astype(np.float32)).to(device)
interp_delta = torch.diff(interp_win, append=interp_win[-1:])
resampy_filter_cache[key] = (interp_win, interp_delta, int(precision))
return resampy_filter_cache[key]
def resample(audio_tensor, sr0, sr1, device, match_librosa=False):
if match_librosa:
if int(sr0) == int(sr1):
return audio_tensor.clone()
ratio = float(sr1) / float(sr0)
scale = min(1.0, ratio)
n_orig = audio_tensor.shape[-1]
n_out = int(n_orig * ratio)
interp_win, interp_delta, num_table = _resampy_gpu_filter(sr0, sr1, audio_tensor.device)
if ratio < 1.0:
interp_win = interp_win * ratio
interp_delta = interp_delta * ratio
index_step = int(scale * num_table)
nwin = interp_win.numel()
t_all = torch.arange(n_out, device=audio_tensor.device, dtype=torch.float32) / ratio
out = torch.zeros((audio_tensor.shape[0], n_out), device=audio_tensor.device, dtype=audio_tensor.dtype)
chunk = 4096
max_taps = (nwin + index_step - 1) // index_step
for begin in range(0, n_out, chunk):
t = t_all[begin:begin + chunk]
n = torch.floor(t).long()
frac = scale * (t - n.to(t.dtype))
index_frac = frac * num_table
offset = torch.floor(index_frac).long()
eta = index_frac - offset.to(index_frac.dtype)
ii = torch.arange(max_taps, device=audio_tensor.device).view(1, -1)
li = offset.view(-1, 1) + ii * index_step
lmax = torch.minimum(n + 1, (nwin - offset) // index_step).view(-1, 1)
lm = ii < lmax
li_safe = li.clamp(0, nwin - 1)
lw = interp_win[li_safe] + eta.view(-1, 1) * interp_delta[li_safe]
left_idx = (n.view(-1, 1) - ii).clamp(0, n_orig - 1)
left = (lw * lm).to(audio_tensor.dtype)
lv = audio_tensor[:, left_idx]
frac_r = scale - frac
index_frac = frac_r * num_table
offset = torch.floor(index_frac).long()
eta = index_frac - offset.to(index_frac.dtype)
ri = offset.view(-1, 1) + ii * index_step
rmax = torch.minimum(n_orig - n - 1, (nwin - offset) // index_step).view(-1, 1)
rm = ii < rmax
ri_safe = ri.clamp(0, nwin - 1)
rw = interp_win[ri_safe] + eta.view(-1, 1) * interp_delta[ri_safe]
right_idx = (n.view(-1, 1) + ii + 1).clamp(0, n_orig - 1)
right = (rw * rm).to(audio_tensor.dtype)
out[:, begin:begin + t.numel()] = (
(lv * left.unsqueeze(0)).sum(-1)
+ (audio_tensor[:, right_idx] * right.unsqueeze(0)).sum(-1)
)
return out
global resample_transform_dict
def _resampy_gpu_filter(sr0, sr1, device):
key = "%s-%s-%s" % (sr0, sr1, str(device))
if key not in resampy_filter_cache:
import resampy
interp_win, precision, rolloff = resampy.filters.get_filter("kaiser_best")
interp_win = torch.from_numpy(interp_win.astype(np.float32)).to(device)
interp_delta = torch.diff(interp_win, append=interp_win[-1:])
resampy_filter_cache[key] = (interp_win, interp_delta, int(precision))
return resampy_filter_cache[key]
def resample(audio_tensor, sr0, sr1, device, match_librosa=False):
if match_librosa:
if int(sr0) == int(sr1):
return audio_tensor.clone()
ratio = float(sr1) / float(sr0)
scale = min(1.0, ratio)
n_orig = audio_tensor.shape[-1]
n_out = int(n_orig * ratio)
interp_win, interp_delta, num_table = _resampy_gpu_filter(sr0, sr1, audio_tensor.device)
if ratio < 1.0:
interp_win = interp_win * ratio
interp_delta = interp_delta * ratio
index_step = int(scale * num_table)
nwin = interp_win.numel()
t_all = torch.arange(n_out, device=audio_tensor.device, dtype=torch.float32) / ratio
out = torch.zeros((audio_tensor.shape[0], n_out), device=audio_tensor.device, dtype=audio_tensor.dtype)
chunk = 4096
max_taps = (nwin + index_step - 1) // index_step
for begin in range(0, n_out, chunk):
t = t_all[begin:begin + chunk]
n = torch.floor(t).long()
frac = scale * (t - n.to(t.dtype))
index_frac = frac * num_table
offset = torch.floor(index_frac).long()
eta = index_frac - offset.to(index_frac.dtype)
ii = torch.arange(max_taps, device=audio_tensor.device).view(1, -1)
li = offset.view(-1, 1) + ii * index_step
lmax = torch.minimum(n + 1, (nwin - offset) // index_step).view(-1, 1)
lm = ii < lmax
li_safe = li.clamp(0, nwin - 1)
lw = interp_win[li_safe] + eta.view(-1, 1) * interp_delta[li_safe]
left_idx = (n.view(-1, 1) - ii).clamp(0, n_orig - 1)
left = (lw * lm).to(audio_tensor.dtype)
lv = audio_tensor[:, left_idx]
frac_r = scale - frac
index_frac = frac_r * num_table
offset = torch.floor(index_frac).long()
eta = index_frac - offset.to(index_frac.dtype)
ri = offset.view(-1, 1) + ii * index_step
rmax = torch.minimum(n_orig - n - 1, (nwin - offset) // index_step).view(-1, 1)
rm = ii < rmax
ri_safe = ri.clamp(0, nwin - 1)
rw = interp_win[ri_safe] + eta.view(-1, 1) * interp_delta[ri_safe]
right_idx = (n.view(-1, 1) + ii + 1).clamp(0, n_orig - 1)
right = (rw * rm).to(audio_tensor.dtype)
out[:, begin:begin + t.numel()] = (
(lv * left.unsqueeze(0)).sum(-1)
+ (audio_tensor[:, right_idx] * right.unsqueeze(0)).sum(-1)
)
return out
global resample_transform_dict
key = "%s-%s-%s" % (sr0, sr1, str(device))
if key not in resample_transform_dict:
resample_transform_dict[key] = torchaudio.transforms.Resample(sr0, sr1).to(device)
return resample_transform_dict[key](audio_tensor)
def load_audio_gpu(path, target_sr, device, mono=True, match_librosa=False):
audio, source_sr = torchaudio.load(path)
audio = audio.to(device=device, dtype=torch.float32)
if mono and audio.dim() == 2 and audio.shape[0] > 1:
audio = audio.mean(dim=0, keepdim=True)
if source_sr != target_sr:
audio = resample(audio, source_sr, target_sr, device, match_librosa=match_librosa)
return audio.squeeze(0)
return resample_transform_dict[key](audio_tensor)
def load_audio_gpu(path, target_sr, device, mono=True, match_librosa=False):
audio, source_sr = torchaudio.load(path)
audio = audio.to(device=device, dtype=torch.float32)
if mono and audio.dim() == 2 and audio.shape[0] > 1:
audio = audio.mean(dim=0, keepdim=True)
if source_sr != target_sr:
audio = resample(audio, source_sr, target_sr, device, match_librosa=match_librosa)
return audio.squeeze(0)
def get_spepc(hps, filename, dtype, device, is_v2pro=False):
@ -865,17 +856,19 @@ def get_tts_wav(
speed=1,
if_freeze=False,
inp_refs=None,
sample_steps=None,
sample_steps=None,
if_sr=False,
pause_second=0.3,
use_cuda_graph=True,
use_flash_attention=True,
cfg_rate=None,
use_flash_attention=True,
cfg_rate=None,
):
global cache, t2s_model_cudagraph
sample_steps, cfg_rate = resolve_sampling(
model_version, sample_steps, cfg_rate, 32 if model_version == "v3" else 8,
)
global cache, t2s_model_cudagraph
if sample_steps is None:
sample_steps = 4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8
if cfg_rate is None:
cfg_rate = 1.30 if model_version == "v5dev" else 0.0
sample_steps, cfg_rate = int(sample_steps), float(cfg_rate)
if ref_wav_path:
pass
else:
@ -891,7 +884,7 @@ def get_tts_wav(
ref_free = False # s2v3暂不支持ref_free
else:
if_sr = False
if model_version not in ({"v3", "v4", "v2Pro", "v2ProPlus"} | V5_VERSIONS):
if model_version not in ({"v3", "v4", "v2Pro", "v2ProPlus"} | V5_VERSIONS):
clean_bigvgan_model()
clean_hifigan_model()
clean_sv_cn_model()
@ -919,12 +912,12 @@ def get_tts_wav(
zero_wav_torch = zero_wav_torch.to(device)
if not ref_free:
with torch.no_grad():
wav16k = load_audio_gpu(ref_wav_path, 16000, device, match_librosa=True)
if wav16k.shape[-1] > 160000 or wav16k.shape[-1] < 48000:
gr.Warning(i18n("参考音频在3~10秒范围外,请更换!"))
raise OSError(i18n("参考音频在3~10秒范围外,请更换!"))
if is_half == True:
wav16k = wav16k.half().to(device)
wav16k = load_audio_gpu(ref_wav_path, 16000, device, match_librosa=True)
if wav16k.shape[-1] > 160000 or wav16k.shape[-1] < 48000:
gr.Warning(i18n("参考音频在3~10秒范围外,请更换!"))
raise OSError(i18n("参考音频在3~10秒范围外,请更换!"))
if is_half == True:
wav16k = wav16k.half().to(device)
else:
wav16k = wav16k.to(device)
wav16k = torch.cat([wav16k, zero_wav_torch])
@ -1072,7 +1065,7 @@ def get_tts_wav(
if sr != tgt_sr:
ref_audio = resample(ref_audio, sr, tgt_sr, device)
# print("ref_audio",ref_audio.abs().mean())
mel2 = mel_fn(ref_audio) if model_version == "v3" else mel_fn_v4(ref_audio)
mel2 = mel_fn(ref_audio) if model_version == "v3" else mel_fn_v4(ref_audio)
mel2 = norm_spec(mel2)
T_min = min(mel2.shape[2], fea_ref.shape[2])
mel2 = mel2[:, :, :T_min]
@ -1088,30 +1081,30 @@ def get_tts_wav(
fea_todo, ge = vq_model.decode_encp(pred_semantic, phoneme_ids1, refer, ge, speed)
cfm_resss = []
idx = 0
if model_version in V5_VERSIONS:
cfm_res = synthesize_v5_mel(vq_model, fea_ref, fea_todo, mel2, sample_steps, cfg_rate)
cfm_res = cfm_res.to(fea_todo.dtype)
else:
while 1:
fea_todo_chunk = fea_todo[:, :, idx : idx + chunk_len]
if fea_todo_chunk.shape[-1] == 0:
break
idx += chunk_len
fea = torch.cat([fea_ref, fea_todo_chunk], 2).transpose(2, 1)
cfm_res = vq_model.cfm.inference(
fea, torch.LongTensor([fea.size(1)]).to(fea.device), mel2, sample_steps, inference_cfg_rate=0
)
cfm_res = cfm_res[:, :, mel2.shape[2] :]
mel2 = cfm_res[:, :, -T_min:]
fea_ref = fea_todo_chunk[:, :, -T_min:]
cfm_resss.append(cfm_res)
cfm_res = torch.cat(cfm_resss, 2)
if model_version in V5_VERSIONS:
cfm_res = synthesize_v5_mel(vq_model, fea_ref, fea_todo, mel2, sample_steps, cfg_rate)
cfm_res = cfm_res.to(fea_todo.dtype)
else:
while 1:
fea_todo_chunk = fea_todo[:, :, idx : idx + chunk_len]
if fea_todo_chunk.shape[-1] == 0:
break
idx += chunk_len
fea = torch.cat([fea_ref, fea_todo_chunk], 2).transpose(2, 1)
cfm_res = vq_model.cfm.inference(
fea, torch.LongTensor([fea.size(1)]).to(fea.device), mel2, sample_steps, inference_cfg_rate=0
)
cfm_res = cfm_res[:, :, mel2.shape[2] :]
mel2 = cfm_res[:, :, -T_min:]
fea_ref = fea_todo_chunk[:, :, -T_min:]
cfm_resss.append(cfm_res)
cfm_res = torch.cat(cfm_resss, 2)
cfm_res = denorm_spec(cfm_res)
if model_version == "v3":
if bigvgan_model == None:
init_bigvgan()
else: # v4
if hifigan_model is None or hifigan_version != model_version:
if hifigan_model is None or hifigan_version != model_version:
init_hifigan()
vocoder_model = bigvgan_model if model_version == "v3" else hifigan_model
with torch.inference_mode():
@ -1287,14 +1280,14 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
GPT_dropdown = gr.Dropdown(
label=i18n("GPT模型列表"),
choices=sorted(GPT_names, key=custom_sort_key),
value=next((name for name, path in name2gpt_path.items() if path == gpt_path), gpt_path),
value=next((name for name, path in name2gpt_path.items() if path == gpt_path), gpt_path),
interactive=True,
scale=14,
)
SoVITS_dropdown = gr.Dropdown(
label=i18n("SoVITS模型列表"),
choices=sorted(SoVITS_names, key=custom_sort_key),
value=next((name for name, path in name2sovits_path.items() if path == sovits_path), sovits_path),
value=next((name for name, path in name2sovits_path.items() if path == sovits_path), sovits_path),
interactive=True,
scale=14,
)
@ -1320,18 +1313,18 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
)
)
prompt_text = gr.Textbox(label=i18n("参考音频的文本"), value="", lines=5, max_lines=5, scale=1)
with gr.Column(scale=14):
with gr.Row():
prompt_language = gr.Dropdown(
label=i18n("参考音频的语种"),
choices=list(dict_language.keys()),
value=i18n("中文"), scale=1,
)
cfg_rate = gr.Slider(
minimum=0, maximum=2, step=0.05,
value=sampling_defaults(model_version)[1], label="CFG",
visible=model_version in V5_VERSIONS, interactive=True, scale=1,
)
with gr.Column(scale=14):
with gr.Row():
prompt_language = gr.Dropdown(
label=i18n("参考音频的语种"),
choices=list(dict_language.keys()),
value=i18n("中文"), scale=1,
)
cfg_rate = gr.Slider(
minimum=0, maximum=2, step=0.05,
value=(1.30 if model_version == "v5dev" else 0.0), label="CFG",
visible=model_version in V5_VERSIONS, interactive=True, scale=1,
)
inp_refs = (
gr.File(
label=i18n(
@ -1351,19 +1344,19 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
sample_steps = (
gr.Radio(
label=i18n("采样步数,如果觉得电,提高试试,如果觉得慢,降低试试"),
value=sampling_defaults(model_version, 32 if model_version == "v3" else 8)[0],
choices=[4, 8, 16, 32],
value=(4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8),
choices=[4, 8, 16, 32],
visible=True,
)
if model_version in v3v4set
else gr.Radio(
label=i18n("采样步数,如果觉得电,提高试试,如果觉得慢,降低试试"),
choices=[4, 8, 16, 32],
choices=[4, 8, 16, 32],
visible=False,
value=sampling_defaults(model_version, 32 if model_version == "v3" else 8)[0],
value=(4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8),
)
)
if_sr_Checkbox = gr.Checkbox(
if_sr_Checkbox = gr.Checkbox(
label=i18n("v3输出如果觉得闷可以试试开超分"),
value=False,
interactive=True,
@ -1470,8 +1463,8 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
if_sr_Checkbox,
pause_second_slider,
use_cuda_graph_checkbox,
flash_attn_checkbox,
cfg_rate,
flash_attn_checkbox,
cfg_rate,
],
[output],
)
@ -1489,8 +1482,8 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
inp_refs,
ref_text_free,
if_sr_Checkbox,
inference_button,
cfg_rate,
inference_button,
cfg_rate,
],
)
GPT_dropdown.change(change_gpt_weights, [GPT_dropdown], [])

View File

@ -63,7 +63,7 @@ version = model_version = os.environ.get("version", "v2")
import gradio as gr
from TTS_infer_pack.text_segmentation_method import get_method
from TTS_infer_pack.TTS import NO_PROMPT_ERROR, TTS, TTS_Config
from module.v5_inference import V5_VERSIONS, sampling_defaults, resolve_sampling
from module.models_v5 import V5_VERSIONS
from tools.acceleration import cuda_graph_available, flash_attention_available
from tools.assets import css, js, top_html
@ -127,11 +127,6 @@ from config import change_choices, get_weights_names, name2gpt_path, name2sovits
SoVITS_names, GPT_names = get_weights_names()
from config import pretrained_sovits_name
path_sovits_v3 = pretrained_sovits_name["v3"]
path_sovits_v4 = pretrained_sovits_name["v4"]
is_exist_s2gv3 = os.path.exists(path_sovits_v3)
is_exist_s2gv4 = os.path.exists(path_sovits_v4)
tts_config = TTS_Config("GPT_SoVITS/configs/tts_infer.yaml")
tts_config.device = device
tts_config.is_half = is_half
@ -182,7 +177,11 @@ def inference(
super_sampling=False,
cfg_rate=None,
):
sample_steps, cfg_rate = resolve_sampling(tts_pipeline.configs.version, sample_steps, cfg_rate)
if sample_steps is None:
sample_steps = 4 if tts_pipeline.configs.version == "v5turbo" else 32
if cfg_rate is None:
cfg_rate = 1.30 if tts_pipeline.configs.version == "v5dev" else 0.0
sample_steps, cfg_rate = int(sample_steps), float(cfg_rate)
seed = -1 if keep_random else seed
actual_seed = seed if seed not in [-1, "", None] else random.randint(0, 2**32 - 1)
inputs = {
@ -252,8 +251,8 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None)
global version, model_version, dict_language, if_lora_v3
version, model_version, if_lora_v3 = get_sovits_version_from_path_fast(sovits_path)
# print(sovits_path,version, model_version, if_lora_v3)
is_exist = is_exist_s2gv3 if model_version == "v3" else is_exist_s2gv4
path_sovits = path_sovits_v3 if model_version == "v3" else path_sovits_v4
path_sovits = pretrained_sovits_name[model_version]
is_exist = os.path.exists(path_sovits)
if if_lora_v3 == True and is_exist == False:
info = path_sovits + "SoVITS %s" % model_version + i18n("底模缺失,无法加载相应 LoRA 权重")
gr.Warning(info)
@ -289,11 +288,11 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None)
prompt_language_update,
text_update,
text_language_update,
{"__type__": "update", "interactive": visible_sample_steps, "value": sampling_defaults(model_version)[0]},
{"__type__": "update", "interactive": visible_sample_steps, "value": (4 if model_version == "v5turbo" else 32)},
{"__type__": "update", "visible": visible_inp_refs},
{"__type__": "update", "interactive": True if model_version not in v3v4set else False},
{"__type__": "update", "value": i18n("模型加载中,请等待"), "interactive": False},
{"__type__": "update", "visible": model_version in V5_VERSIONS, "value": sampling_defaults(model_version)[1]},
{"__type__": "update", "visible": model_version in V5_VERSIONS, "value": (1.30 if model_version == "v5dev" else 0.0)},
)
tts_pipeline.init_vits_weights(sovits_path)
@ -304,11 +303,11 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None)
prompt_language_update,
text_update,
text_language_update,
{"__type__": "update", "interactive": visible_sample_steps, "value": sampling_defaults(model_version)[0]},
{"__type__": "update", "interactive": visible_sample_steps, "value": (4 if model_version == "v5turbo" else 32)},
{"__type__": "update", "visible": visible_inp_refs},
{"__type__": "update", "interactive": True if model_version not in v3v4set else False},
{"__type__": "update", "value": i18n("合成语音"), "interactive": True},
{"__type__": "update", "visible": model_version in V5_VERSIONS, "value": sampling_defaults(model_version)[1]},
{"__type__": "update", "visible": model_version in V5_VERSIONS, "value": (1.30 if model_version == "v5dev" else 0.0)},
)
with open("./weight.json") as f:
data = f.read()
@ -369,7 +368,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
)
cfg_rate = gr.Slider(
minimum=0, maximum=2, step=0.05,
value=sampling_defaults(model_version)[1], label="CFG",
value=(1.30 if model_version == "v5dev" else 0.0), label="CFG",
visible=model_version in V5_VERSIONS, interactive=True, scale=1,
)
with gr.Column():
@ -402,7 +401,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
value=1 if model_version in V5_VERSIONS else 20, interactive=True
)
sample_steps = gr.Radio(
label="Euler steps (V3/V4/V5)", value=sampling_defaults(model_version)[0], choices=[4, 8, 16, 32], visible=True
label="Euler steps (V3/V4/V5)", value=(4 if model_version == "v5turbo" else 32), choices=[4, 8, 16, 32], visible=True
)
with gr.Row():
fragment_interval = gr.Slider(

View File

@ -1109,11 +1109,18 @@ class CFM(torch.nn.Module):
self.criterion = torch.nn.MSELoss()
self.use_conditioner_cache = True
self.use_static_cache = False
self.use_step_embedding = True
self.cfg_drop_text = True
self.noise_temperature = None
@torch.inference_mode()
def inference(self, mu, x_lens, prompt, n_timesteps, temperature=1.0, inference_cfg_rate=0):
"""Forward diffusion"""
n_timesteps, inference_cfg_rate = int(n_timesteps), float(inference_cfg_rate)
B, T = mu.size(0), mu.size(1)
if self.noise_temperature is not None:
temperature = self.noise_temperature
x = torch.randn([B, self.in_channels, T], device=mu.device, dtype=mu.dtype) * temperature
prompt_len = prompt.size(-1)
prompt_x = torch.zeros_like(x, dtype=mu.dtype)
@ -1125,42 +1132,47 @@ class CFM(torch.nn.Module):
text_cache = None
text_cfg_cache = None
dt_cache = None
d_tensor = torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * d
static_cache = (self.estimator.prepare_static_cache(prompt_x, x_lens, mu)
if self.use_conditioner_cache and self.use_static_cache else None)
d_tensor = (torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * d
if self.use_step_embedding else None)
for j in range(n_timesteps):
t_tensor = torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * t
# v_pred = model(x, t_tensor, d_tensor, **extra_args)
t_tensor = torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * (t if self.use_step_embedding else j * d)
step_condition = {"dt_base_bootstrap": d_tensor, "dt_cache": dt_cache} if self.use_step_embedding else {}
v_pred, text_emb, dt = self.estimator(
x,
prompt_x,
x_lens,
t_tensor,
d_tensor,
mu,
text0=mu,
use_grad_ckpt=False,
drop_audio_cond=False,
drop_text=False,
infer=True,
text_cache=text_cache,
dt_cache=dt_cache,
static_cache=static_cache,
**step_condition,
)
v_pred = v_pred.transpose(2, 1)
if self.use_conditioner_cache:
text_cache = text_emb
dt_cache = dt
if self.use_step_embedding:
step_condition["dt_cache"] = dt_cache
if inference_cfg_rate > 1e-5:
neg, text_cfg_emb, _ = self.estimator(
x,
prompt_x,
x_lens,
t_tensor,
d_tensor,
mu,
text0=mu,
use_grad_ckpt=False,
drop_audio_cond=True,
drop_text=True,
drop_text=self.cfg_drop_text,
infer=True,
text_cache=text_cfg_cache,
dt_cache=dt_cache,
text_cache=text_cfg_cache if self.cfg_drop_text else text_cache,
static_cache=static_cache,
**step_condition,
)
neg = neg.transpose(2, 1)
if self.use_conditioner_cache:
@ -1289,14 +1301,13 @@ class SynthesizerTrnV3(nn.Module):
self.bridge = nn.Sequential(nn.Conv1d(inter_channels, inter_channels2, 1, stride=1), nn.LeakyReLU())
self.wns1 = Encoder(inter_channels2, inter_channels2, inter_channels2, 5, 1, 8, gin_channels=gin_channels)
self.linear_mel = nn.Conv1d(inter_channels2, 100, 1, stride=1)
self.cfm = CFM(
100,
DiT(**dict(dim=1024, depth=22, heads=16, ff_mult=2, text_dim=inter_channels2,
conv_layers=4, use_step_embedding=version not in {"v5", "v5dev", "v5turbo"})),
) # text_dim is condition feature dim
if version in {"v5", "v5dev", "v5turbo"}:
from module.v5_inference import CFMV5
self.cfm = CFMV5(100, self.cfm.estimator)
dit = DiT(**dict(dim=1024, depth=22, heads=16, ff_mult=2, text_dim=inter_channels2,
conv_layers=4, use_step_embedding=version not in {"v5dev", "v5turbo"}))
if version in {"v5dev", "v5turbo"}:
from module.models_v5 import CFMV5
self.cfm = CFMV5(100, dit)
else:
self.cfm = CFM(100, dit)
if self.freeze_quantizer == True:
set_no_grad(self.ssl_proj)
set_no_grad(self.quantizer)

View File

@ -0,0 +1,84 @@
import os
import torch
from module.models import CFM
V5_TEMPERATURE = 0.875
V5_REFERENCE_ANCHOR = 1.0
V5_ROLLING_TAIL_FRAMES = 32
V5_MAX_TARGET_CHUNK_FRAMES = 640
V5_REFERENCE_FRAMES = 500
V5_TOTAL_CHUNK_FRAMES = 1000
V5_VERSIONS = {"v5dev", "v5turbo"}
class CFMV5(CFM):
def __init__(self, in_channels, dit):
super().__init__(in_channels, dit)
self.use_step_embedding = False
self.cfg_drop_text = False
self.noise_temperature = V5_TEMPERATURE
self.use_static_cache = os.environ.get(
"GSVV5_CFM_STATIC_CACHE", os.environ.get("GSVV4_CFM_STATIC_CACHE", "1")
) != "0"
@torch.inference_mode()
def inference(self, mu, x_lens, prompt, n_timesteps, temperature=V5_TEMPERATURE,
inference_cfg_rate=1.30):
return super().inference(mu, x_lens, prompt, n_timesteps, temperature, inference_cfg_rate)
def forward(self, x1, x_lens, prompt_lens, mu, use_grad_ckpt):
b = x1.shape[0]
t = torch.rand([b], device=mu.device, dtype=x1.dtype)
x0 = torch.randn_like(x1, device=mu.device)
vt = x1 - x0
xt = x0 + t[:, None, None] * vt
prompt = torch.zeros_like(x1)
for i in range(b):
prompt[i, :, : prompt_lens[i]] = x1[i, :, : prompt_lens[i]]
xt[i, :, : prompt_lens[i]] = 0
vt_pred = self.estimator(
xt, prompt, x_lens, t, text0=mu, use_grad_ckpt=use_grad_ckpt,
).transpose(2, 1)
loss = 0
for i in range(b):
loss += self.criterion(vt_pred[i, :, prompt_lens[i] : x_lens[i]], vt[i, :, prompt_lens[i] : x_lens[i]])
return loss / b
@torch.inference_mode()
def synthesize_v5_mel(model, reference_features, target_features, reference_mel,
sample_steps=None, cfg_rate=None):
steps = (4 if model.version == "v5turbo" else 32) if sample_steps is None else int(sample_steps)
cfg = (1.30 if model.version == "v5dev" else 0.0) if cfg_rate is None else float(cfg_rate)
reference_frames = min(reference_mel.shape[-1], reference_features.shape[-1])
reference_mel = reference_mel[..., :reference_frames][..., -V5_REFERENCE_FRAMES:]
reference_features = reference_features[..., :reference_frames][..., -V5_REFERENCE_FRAMES:]
reference_frames = reference_mel.shape[-1]
chunk_frames = min(V5_TOTAL_CHUNK_FRAMES - reference_frames, V5_MAX_TARGET_CHUNK_FRAMES)
original_mel = reference_mel
original_features = reference_features
rolling_mel = reference_mel
rolling_features = reference_features
results = []
for index, start in enumerate(range(0, target_features.shape[-1], chunk_frames)):
target = target_features[..., start:start + chunk_frames]
if index == 0:
prompt = original_mel
features = original_features
else:
tail = min(V5_ROLLING_TAIL_FRAMES, reference_frames,
rolling_mel.shape[-1], rolling_features.shape[-1])
prefix = reference_frames - tail
prompt = torch.cat((original_mel[..., :prefix], rolling_mel[..., -tail:]), dim=-1)
features = torch.cat((original_features[..., :prefix], rolling_features[..., -tail:]), dim=-1)
mu = torch.cat((features, target), dim=-1).transpose(2, 1)
lengths = torch.full((mu.shape[0],), mu.shape[1], dtype=torch.long, device=mu.device)
generated = model.cfm.inference(
mu, lengths, prompt, steps, inference_cfg_rate=cfg,
)[..., prompt.shape[-1]:]
results.append(generated)
rolling_mel = generated[..., -reference_frames:].to(reference_mel.dtype)
rolling_features = target[..., -reference_frames:]
return torch.cat(results, dim=-1)

View File

@ -19,14 +19,13 @@ def my_save(fea, path): #####fix issue: torch.save doesn't support chinese path
from io import BytesIO
model_version2byte = {
model_version2byte = {
"v3": b"03",
"v4": b"04",
"v2Pro": b"05",
"v2ProPlus": b"06",
"v5": b"07",
"v5dev": b"07",
"v5turbo": b"08",
"v2ProPlus": b"06",
"v5dev": b"07",
"v5turbo": b"08",
}
@ -64,15 +63,16 @@ def savee(ckpt, name, epoch, steps, hps, model_version=None, lora_rank=None):
"""
finetune version
00:v1
01:v2
02:v3
03:v3lora
04:v4lora
05:v2Pro
06:v2ProPlus
07:v5dev
08:v5turbo
06:v2ProPlus
07:v5dev-lora
08:v5turbo-lora
"""
head2version = {
b"00": ["v1", "v1", False],
@ -81,19 +81,19 @@ head2version = {
b"03": ["v2", "v3", True],
b"04": ["v2", "v4", True],
b"05": ["v2", "v2Pro", False],
b"06": ["v2", "v2ProPlus", False],
b"07": ["v2", "v5dev", False],
b"08": ["v2", "v5turbo", False],
}
b"06": ["v2", "v2ProPlus", False],
b"07": ["v2", "v5dev", True],
b"08": ["v2", "v5turbo", True],
}
hash_pretrained_dict = {
"dc3c97e17592963677a4a1681f30c653": ["v2", "v2", False], # s2G488k.pth#sovits_v1_pretrained
"43797be674a37c1c83ee81081941ed0f": ["v2", "v3", False], # s2Gv3.pth#sovits_v3_pretrained
"6642b37f3dbb1f76882b69937c95a5f3": ["v2", "v2", False], # s2G2333K.pth#sovits_v2_pretrained
"4f26b9476d0c5033e04162c486074374": ["v2", "v4", False], # s2Gv4.pth#sovits_v4_pretrained
"c7e9fce2223f3db685cdfa1e6368728a": ["v2", "v2Pro", False], # s2Gv2Pro.pth#sovits_v2Pro_pretrained
"66b313e39455b57ab1b0bc0b239c9d0a": ["v2", "v2ProPlus", False], # s2Gv2ProPlus.pth#sovits_v2ProPlus_pretrained
"0e6e00af4104438346bda5344e6260e3": ["v2", "v5dev", False], # s2Gv5dev.pth, 07 header
"695048a8e361076a54e6f32a21b949a3": ["v2", "v5turbo", False], # s2Gv5turbo.pth, 08 header
"66b313e39455b57ab1b0bc0b239c9d0a": ["v2", "v2ProPlus", False], # s2Gv2ProPlus.pth#sovits_v2ProPlus_pretrained
"c4ce1f839a0d271e41963f9fb16ade76": ["v2", "v5dev", False], # s2Gv5dev.pth
"6bc5cb2a195cc8ec22b2e723cd4d6f47": ["v2", "v5turbo", False], # s2Gv5turbo.pth
}
import hashlib
@ -106,26 +106,16 @@ def get_hash_from_file(sovits_path):
return hash_md5.hexdigest()
def get_sovits_version_from_path_fast(sovits_path):
###1-if it is pretrained sovits models, by hash
def get_sovits_version_from_path_fast(sovits_path):
###1-if it is pretrained sovits models, by hash
hash = get_hash_from_file(sovits_path)
if hash in hash_pretrained_dict:
return hash_pretrained_dict[hash]
###2-new weights, by head
with open(sovits_path, "rb") as f:
version = f.read(2)
if version != b"PK":
return head2version[version]
filename = os.path.basename(sovits_path).lower()
if filename in {"s2gv5dev.pth", "s2gv5turbo.pth"}:
return ["v2", "v5turbo" if filename == "s2gv5turbo.pth" else "v5dev", False]
# Only unknown legacy PK checkpoints need full structural inspection.
if os.path.getsize(sovits_path) >= 700 * 1024 * 1024:
checkpoint = load_sovits_new(sovits_path)
weights = checkpoint.get("weight", {})
if ("cfm.estimator.time_embed.time_mlp.0.weight" in weights
and "cfm.estimator.d_embed.time_mlp.0.weight" not in weights):
return ["v2", "v5dev", False]
if version != b"PK":
return head2version[version]
###3-old weights, by file size
if_lora_v3 = False
size = os.path.getsize(sovits_path)

View File

@ -55,6 +55,10 @@ def main():
n_gpus = torch.cuda.device_count()
else:
n_gpus = 1
if n_gpus <= 1:
run(0, n_gpus, hps)
return
os.environ["MASTER_ADDR"] = "localhost"
os.environ["MASTER_PORT"] = str(randint(20000, 55555))
@ -69,6 +73,22 @@ def main():
def run(rank, n_gpus, hps):
use_ddp = torch.cuda.is_available() and n_gpus > 1
try:
if use_ddp:
dist.init_process_group(
backend="gloo" if os.name == "nt" else "nccl",
init_method="env://?use_libuv=False",
world_size=n_gpus,
rank=rank,
)
_run(rank, n_gpus, hps, use_ddp)
finally:
if use_ddp and dist.is_initialized():
dist.destroy_process_group()
def _run(rank, n_gpus, hps, use_ddp):
global global_step
if rank == 0:
logger = utils.get_logger(hps.data.exp_dir)
@ -77,12 +97,6 @@ def run(rank, n_gpus, hps):
writer = SummaryWriter(log_dir=hps.s2_ckpt_dir)
writer_eval = SummaryWriter(log_dir=os.path.join(hps.s2_ckpt_dir, "eval"))
dist.init_process_group(
backend="gloo" if os.name == "nt" or not torch.cuda.is_available() else "nccl",
init_method="env://?use_libuv=False",
world_size=n_gpus,
rank=rank,
)
torch.manual_seed(hps.train.seed)
if torch.cuda.is_available():
torch.cuda.set_device(rank)
@ -118,13 +132,13 @@ def run(rank, n_gpus, hps):
collate_fn = TextAudioSpeakerCollate(version=hps.model.version)
train_loader = DataLoader(
train_dataset,
num_workers=5,
num_workers=4,
shuffle=False,
pin_memory=True,
collate_fn=collate_fn,
batch_sampler=train_sampler,
persistent_workers=True,
prefetch_factor=3,
prefetch_factor=2,
)
# if rank == 0:
# eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data, val=True)
@ -196,10 +210,10 @@ def run(rank, n_gpus, hps):
betas=hps.train.betas,
eps=hps.train.eps,
)
if torch.cuda.is_available():
if use_ddp:
net_g = DDP(net_g, device_ids=[rank], find_unused_parameters=True)
net_d = DDP(net_d, device_ids=[rank], find_unused_parameters=True)
else:
elif not torch.cuda.is_available():
net_g = net_g.to(device)
net_d = net_d.to(device)
@ -232,14 +246,10 @@ def run(rank, n_gpus, hps):
):
if rank == 0:
logger.info("loaded pretrained %s" % hps.train.pretrained_s2G)
model_g = net_g.module if hasattr(net_g, "module") else net_g
print(
"loaded pretrained %s" % hps.train.pretrained_s2G,
net_g.module.load_state_dict(
torch.load(hps.train.pretrained_s2G, map_location="cpu", weights_only=False)["weight"],
strict=False,
)
if torch.cuda.is_available()
else net_g.load_state_dict(
model_g.load_state_dict(
torch.load(hps.train.pretrained_s2G, map_location="cpu", weights_only=False)["weight"],
strict=False,
),
@ -251,14 +261,12 @@ def run(rank, n_gpus, hps):
):
if rank == 0:
logger.info("loaded pretrained %s" % hps.train.pretrained_s2D)
model_d = net_d.module if hasattr(net_d, "module") else net_d
print(
"loaded pretrained %s" % hps.train.pretrained_s2D,
net_d.module.load_state_dict(
torch.load(hps.train.pretrained_s2D, map_location="cpu", weights_only=False)["weight"], strict=False
)
if torch.cuda.is_available()
else net_d.load_state_dict(
model_d.load_state_dict(
torch.load(hps.train.pretrained_s2D, map_location="cpu", weights_only=False)["weight"],
strict=False,
),
)
@ -581,6 +589,7 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade
def evaluate(hps, generator, eval_loader, writer_eval):
generator.eval()
generator_to_infer = generator.module if hasattr(generator, "module") else generator
image_dict = {}
audio_dict = {}
print("Evaluating ...")
@ -607,24 +616,13 @@ def evaluate(hps, generator, eval_loader, writer_eval):
ssl = ssl.to(device)
text, text_lengths = text.to(device), text_lengths.to(device)
for test in [0, 1]:
y_hat, mask, *_ = (
generator.module.infer(
ssl,
spec,
spec_lengths,
text,
text_lengths,
test=test,
)
if torch.cuda.is_available()
else generator.infer(
ssl,
spec,
spec_lengths,
text,
text_lengths,
test=test,
)
y_hat, mask, *_ = generator_to_infer.infer(
ssl,
spec,
spec_lengths,
text,
text_lengths,
test=test,
)
y_hat_lengths = mask.sum([1, 2]).long() * hps.data.hop_length

View File

@ -55,6 +55,10 @@ def main():
n_gpus = torch.cuda.device_count()
else:
n_gpus = 1
if n_gpus <= 1:
run(0, n_gpus, hps)
return
os.environ["MASTER_ADDR"] = "localhost"
os.environ["MASTER_PORT"] = str(randint(20000, 55555))
@ -69,6 +73,22 @@ def main():
def run(rank, n_gpus, hps):
use_ddp = torch.cuda.is_available() and n_gpus > 1
try:
if use_ddp:
dist.init_process_group(
backend="gloo" if os.name == "nt" else "nccl",
init_method="env://?use_libuv=False",
world_size=n_gpus,
rank=rank,
)
_run(rank, n_gpus, hps, use_ddp)
finally:
if use_ddp and dist.is_initialized():
dist.destroy_process_group()
def _run(rank, n_gpus, hps, use_ddp):
global global_step
if rank == 0:
logger = utils.get_logger(hps.data.exp_dir)
@ -77,12 +97,6 @@ def run(rank, n_gpus, hps):
writer = SummaryWriter(log_dir=hps.s2_ckpt_dir)
writer_eval = SummaryWriter(log_dir=os.path.join(hps.s2_ckpt_dir, "eval"))
dist.init_process_group(
backend="gloo" if os.name == "nt" or not torch.cuda.is_available() else "nccl",
init_method="env://?use_libuv=False",
world_size=n_gpus,
rank=rank,
)
torch.manual_seed(hps.train.seed)
if torch.cuda.is_available():
torch.cuda.set_device(rank)
@ -118,13 +132,13 @@ def run(rank, n_gpus, hps):
collate_fn = TextAudioSpeakerCollate()
train_loader = DataLoader(
train_dataset,
num_workers=5,
num_workers=4,
shuffle=False,
pin_memory=True,
collate_fn=collate_fn,
batch_sampler=train_sampler,
persistent_workers=True,
prefetch_factor=3,
prefetch_factor=2,
)
# if rank == 0:
# eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data, val=True)
@ -165,10 +179,10 @@ def run(rank, n_gpus, hps):
# betas=hps.train.betas,
# eps=hps.train.eps,
# )
if torch.cuda.is_available():
if use_ddp:
net_g = DDP(net_g, device_ids=[rank], find_unused_parameters=True)
# net_d = DDP(net_d, device_ids=[rank], find_unused_parameters=True)
else:
elif not torch.cuda.is_available():
net_g = net_g.to(device)
# net_d = net_d.to(device)
@ -201,14 +215,10 @@ def run(rank, n_gpus, hps):
):
if rank == 0:
logger.info("loaded pretrained %s" % hps.train.pretrained_s2G)
model_g = net_g.module if hasattr(net_g, "module") else net_g
print(
"loaded pretrained %s" % hps.train.pretrained_s2G,
net_g.module.load_state_dict(
torch.load(hps.train.pretrained_s2G, map_location="cpu", weights_only=False)["weight"],
strict=False,
)
if torch.cuda.is_available()
else net_g.load_state_dict(
model_g.load_state_dict(
torch.load(hps.train.pretrained_s2G, map_location="cpu", weights_only=False)["weight"],
strict=False,
),

View File

@ -6,6 +6,7 @@ import os
import utils
hps = utils.get_hparams(stage=2)
os.environ["version"] = hps.model.version
os.environ["CUDA_VISIBLE_DEVICES"] = hps.train.gpu_numbers.replace("-", ",")
import logging
@ -73,6 +74,22 @@ def main():
def run(rank, n_gpus, hps):
use_ddp = torch.cuda.is_available() and n_gpus > 1
try:
if use_ddp:
dist.init_process_group(
backend="gloo" if os.name == "nt" else "nccl",
init_method="env://?use_libuv=False",
world_size=n_gpus,
rank=rank,
)
_run(rank, n_gpus, hps, use_ddp)
finally:
if use_ddp and dist.is_initialized():
dist.destroy_process_group()
def _run(rank, n_gpus, hps, use_ddp):
global global_step, no_grad_names, save_root, lora_rank
if rank == 0:
logger = utils.get_logger(hps.data.exp_dir)
@ -81,14 +98,6 @@ def run(rank, n_gpus, hps):
writer = SummaryWriter(log_dir=hps.s2_ckpt_dir)
writer_eval = SummaryWriter(log_dir=os.path.join(hps.s2_ckpt_dir, "eval"))
use_ddp = n_gpus > 1
if use_ddp:
dist.init_process_group(
backend="gloo" if os.name == "nt" or not torch.cuda.is_available() else "nccl",
init_method="env://?use_libuv=False",
world_size=n_gpus,
rank=rank,
)
torch.manual_seed(hps.train.seed)
if torch.cuda.is_available():
torch.cuda.set_device(rank)
@ -124,7 +133,7 @@ def run(rank, n_gpus, hps):
shuffle=True,
)
collate_fn = TextAudioSpeakerCollate()
worker_count = 0 if os.name == "nt" and n_gpus <= 1 else min(2 if os.name == "nt" else 5, os.cpu_count() or 1)
worker_count = 4
loader_kwargs = dict(
num_workers=worker_count,
shuffle=False,
@ -134,7 +143,7 @@ def run(rank, n_gpus, hps):
)
if worker_count > 0:
loader_kwargs["persistent_workers"] = True
loader_kwargs["prefetch_factor"] = 2 if os.name == "nt" else 3
loader_kwargs["prefetch_factor"] = 2
train_loader = DataLoader(
train_dataset,
**loader_kwargs,
@ -255,8 +264,6 @@ def run(rank, n_gpus, hps):
None,
)
scheduler_g.step()
if use_ddp and dist.is_initialized():
dist.destroy_process_group()
print("training done")

View File

@ -35,7 +35,7 @@ A Powerful Few-shot Voice Conversion and Text-to-Speech WebUI.<br><br>
3. **Cross-lingual Support:** Inference in languages different from the training dataset, currently supporting English, Japanese, Korean, Cantonese and Chinese.
4. **WebUI Tools:** Integrated tools include voice accompaniment separation, automatic training set segmentation, multilingual ASR with [Fun-ASR-Nano](https://github.com/FunAudioLLM/Fun-ASR), [SenseVoice](https://github.com/FunAudioLLM/SenseVoice), and classic [FunASR](https://github.com/modelscope/FunASR), plus text labeling, assisting beginners in creating training datasets and GPT/SoVITS models.
4. **WebUI Tools:** Integrated tools include voice accompaniment separation, automatic training set segmentation, Chinese ASR, and text labeling, assisting beginners in creating training datasets and GPT/SoVITS models.
**Check out our [demo video](https://www.bilibili.com/video/BV12g4y1m7Uw) here!**
@ -48,8 +48,6 @@ https://github.com/RVC-Boss/GPT-SoVITS/assets/129054828/05bee1fa-bdd8-4d85-9350-
请不要尬黑GPT-SoVITS推理速度慢,谢谢!
CPU-Optimized Inference Version:https://github.com/baicai-1145/GPT-SoVITS-CPUFast
**User guide: [简体中文](https://www.yuque.com/baicaigongchang1145haoyuangong/ib3g1e) | [English](https://rentry.co/GPT-SoVITS-guide#/)**
## Installation
@ -208,7 +206,7 @@ docker exec -it <GPT-SoVITS-CU126-Lite|GPT-SoVITS-CU128-Lite|GPT-SoVITS-CU126|GP
- The suggestion is to **directly specify the model type** in the model name and configuration file name, such as `mel_mand_roformer`, `bs_roformer`. If not specified, the features will be compared from the configuration file to determine which type of model it is. For example, the model `bs_roformer_ep_368_sdr_12.9628.ckpt` and its corresponding configuration file `bs_roformer_ep_368_sdr_12.9628.yaml` are a pair, `kim_mel_band_roformer.ckpt` and `kim_mel_band_roformer.yaml` are also a pair.
4. FunASR models are downloaded automatically on first use. The WebUI offers [Fun-ASR-Nano](https://github.com/FunAudioLLM/Fun-ASR) for multilingual and dialect ASR, [SenseVoice](https://github.com/FunAudioLLM/SenseVoice) for fast transcription, and classic Paraformer/UniASR through [FunASR](https://github.com/modelscope/FunASR) for Chinese and Cantonese. To preinstall the classic Chinese models for offline use, download the [ASR model](https://modelscope.cn/models/iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/files), [VAD model](https://modelscope.cn/models/iic/speech_fsmn_vad_zh-cn-16k-common-pytorch/files), and [punctuation model](https://modelscope.cn/models/iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/files) into `tools/asr/models`.
4. For Chinese ASR (additionally), download models from [Damo ASR Model](https://modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/files), [Damo VAD Model](https://modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch/files), and [Damo Punc Model](https://modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/files) and place them in `tools/asr/models`.
5. For English or Japanese ASR (additionally), download models from [Faster Whisper Large V3](https://huggingface.co/Systran/faster-whisper-large-v3) and place them in `tools/asr/models`. Also, [other models](https://huggingface.co/Systran) may have the similar effect with smaller disk footprint.
@ -368,22 +366,6 @@ Use v2Pro from v1/v2/v3/v4 environment:
3. Download v2Pro pretrained models (v2Pro/s2Dv2Pro.pth, v2Pro/s2Gv2Pro.pth, v2Pro/s2Dv2ProPlus.pth, v2Pro/s2Gv2ProPlus.pth, and sv/pretrained_eres2netv2w24s4ep4.ckpt) from [huggingface](https://huggingface.co/lj1995/GPT-SoVITS/tree/main) and put them into `GPT_SoVITS/pretrained_models`.
## V5 Release Notes
New Features:
1. Significantly improved voice similarity without SoVITS fine-tuning.
2. An updated vocoder significantly reduces high-frequency spectral mirroring and aliasing artifacts.
3. Support for `cuda_graph` and `flash_attention` inference acceleration. Thanks to [@XXXXRT666](https://github.com/XXXXRT666) for the contribution.
Upgrading from V2Pro to V5:
1. Clone the latest code from [GitHub](https://github.com/RVC-Boss/GPT-SoVITS/tree/cuda_graph_accel_v5).
2. Download the V5 pretrained models from [Hugging Face](https://huggingface.co/lj1995/GPT-SoVITS/tree/main/gsv-v5-pretrained) and place them in `GPT_SoVITS/pretrained_models`, preserving the `gsv-v5-pretrained` subdirectory so the model directory is `GPT_SoVITS/pretrained_models/gsv-v5-pretrained`.
## Todo List
- [x] **High Priority:**
@ -429,13 +411,13 @@ python audio_slicer.py \
--hop_size <step_size_for_computing_volume_curve>
```
Run dataset ASR with FunASR from the command line. Fun-ASR-Nano is the default for Chinese, English, Japanese, Korean, and automatic language detection; Cantonese keeps the classic FunASR backend.
This is how dataset ASR processing is done using the command line(Only Chinese)
```bash
python tools/asr/funasr_asr.py -i <input> -o <output> -l zh
python tools/asr/funasr_asr.py -i <input> -o <output>
```
Faster Whisper is also available as an ASR backend.
ASR processing is performed through Faster_Whisper(ASR marking except Chinese)
(No progress bars, GPU performance may cause time delays)
@ -484,9 +466,7 @@ Special thanks to the following projects and contributors:
- [FFmpeg](https://github.com/FFmpeg/FFmpeg)
- [gradio](https://github.com/gradio-app/gradio)
- [faster-whisper](https://github.com/SYSTRAN/faster-whisper)
- [FunASR](https://github.com/modelscope/FunASR)
- [Fun-ASR](https://github.com/FunAudioLLM/Fun-ASR)
- [SenseVoice](https://github.com/FunAudioLLM/SenseVoice)
- [FunASR](https://github.com/alibaba-damo-academy/FunASR)
- [AP-BWE](https://github.com/yxlu-0102/AP-BWE)
Thankful to @Naozumi520 for providing the Cantonese training set and for the guidance on Cantonese-related knowledge.

124
api.py
View File

@ -164,6 +164,7 @@ import numpy as np
from feature_extractor import cnhubert
from io import BytesIO
from module.models import Generator, SynthesizerTrn, SynthesizerTrnV3
from module.models_v5 import V5_VERSIONS, synthesize_v5_mel
from peft import LoraConfig, get_peft_model
from AR.models.t2s_lightning_module import Text2SemanticLightningModule
from text import cleaned_text_to_sequence
@ -199,10 +200,11 @@ def is_full(*items): # 任意一项为空返回False
bigvgan_model = hifigan_model = sv_cn_model = None
hifigan_version = None
def clean_hifigan_model():
global hifigan_model
global hifigan_model, hifigan_version
if hifigan_model:
hifigan_model = hifigan_model.cpu()
hifigan_model = None
@ -210,6 +212,7 @@ def clean_hifigan_model():
torch.cuda.empty_cache()
except:
pass
hifigan_version = None
def clean_bigvgan_model():
@ -240,7 +243,8 @@ def init_bigvgan():
bigvgan_model = bigvgan.BigVGAN.from_pretrained(
"%s/GPT_SoVITS/pretrained_models/models--nvidia--bigvgan_v2_24khz_100band_256x" % (now_dir,),
)
use_cuda_kernel=False,
) # if True, RuntimeError: Ninja is required to load C++ extensions
# remove weight norm in the model and set to eval mode
bigvgan_model.remove_weight_norm()
bigvgan_model = bigvgan_model.eval()
@ -251,8 +255,12 @@ def init_bigvgan():
bigvgan_model = bigvgan_model.to(device)
def init_hifigan():
global hifigan_model, bigvgan_model, sv_cn_model
def init_hifigan(version="v4"):
global hifigan_model, hifigan_version, bigvgan_model, sv_cn_model
vocoder_version = "v5" if version in V5_VERSIONS else version
if hifigan_model is not None and hifigan_version == vocoder_version:
return
clean_hifigan_model()
hifigan_model = Generator(
initial_channel=100,
resblock="1",
@ -267,7 +275,7 @@ def init_hifigan():
hifigan_model.eval()
hifigan_model.remove_weight_norm()
state_dict_g = torch.load(
"%s/GPT_SoVITS/pretrained_models/gsv-v4-pretrained/vocoder.pth" % (now_dir,),
"%s/GPT_SoVITS/pretrained_models/gsv-%s-pretrained/vocoder.pth" % (now_dir, vocoder_version),
map_location="cpu",
weights_only=False,
)
@ -276,6 +284,7 @@ def init_hifigan():
hifigan_model = hifigan_model.half().to(device)
else:
hifigan_model = hifigan_model.to(device)
hifigan_version = vocoder_version
from sv import SV
@ -380,14 +389,9 @@ from process_ckpt import get_sovits_version_from_path_fast, load_sovits_new
def get_sovits_weights(sovits_path):
from config import pretrained_sovits_name
path_sovits_v3 = pretrained_sovits_name["v3"]
path_sovits_v4 = pretrained_sovits_name["v4"]
is_exist_s2gv3 = os.path.exists(path_sovits_v3)
is_exist_s2gv4 = os.path.exists(path_sovits_v4)
version, model_version, if_lora_v3 = get_sovits_version_from_path_fast(sovits_path)
is_exist = is_exist_s2gv3 if model_version == "v3" else is_exist_s2gv4
path_sovits = path_sovits_v3 if model_version == "v3" else path_sovits_v4
path_sovits = pretrained_sovits_name[model_version]
is_exist = os.path.exists(path_sovits)
if if_lora_v3 == True and is_exist == False:
logger.info("SoVITS %s 底模缺失,无法加载相应 LoRA 权重" % model_version)
@ -404,7 +408,7 @@ def get_sovits_weights(sovits_path):
hps.model.version = "v2"
model_params_dict = vars(hps.model)
if model_version not in {"v3", "v4"}:
if model_version not in {"v3", "v4"} | V5_VERSIONS:
if "Pro" in model_version:
hps.model.version = model_version
if sv_cn_model == None:
@ -426,8 +430,8 @@ def get_sovits_weights(sovits_path):
)
if model_version == "v3":
init_bigvgan()
if model_version == "v4":
init_hifigan()
else:
init_hifigan(model_version)
model_version = hps.model.version
logger.info(f"模型版本: {model_version}")
@ -444,7 +448,6 @@ def get_sovits_weights(sovits_path):
if if_lora_v3 == False:
vq_model.load_state_dict(dict_s2["weight"], strict=False)
else:
path_sovits = path_sovits_v3 if model_version == "v3" else path_sovits_v4
vq_model.load_state_dict(load_sovits_new(path_sovits)["weight"], strict=False)
lora_rank = dict_s2["lora_rank"]
lora_config = LoraConfig(
@ -837,9 +840,10 @@ def get_tts_wav(
temperature=0.6,
speed=1,
inp_refs=None,
sample_steps=32,
sample_steps=None,
if_sr=False,
spk="default",
cfg_rate=None,
):
infer_sovits = speaker_list[spk].sovits
vq_model = infer_sovits.vq_model
@ -850,6 +854,12 @@ def get_tts_wav(
t2s_model = infer_gpt.t2s_model
max_sec = infer_gpt.max_sec
if sample_steps is None:
sample_steps = 4 if version == "v5turbo" else 32
if cfg_rate is None:
cfg_rate = 1.30 if version == "v5dev" else 0.0
sample_steps, cfg_rate = int(sample_steps), float(cfg_rate)
if version == "v3":
if sample_steps not in [4, 8, 16, 32, 64, 128]:
sample_steps = 32
@ -884,7 +894,7 @@ def get_tts_wav(
prompt = prompt_semantic.unsqueeze(0).to(device)
is_v2pro = version in {"v2Pro", "v2ProPlus"}
if version not in {"v3", "v4"}:
if version not in {"v3", "v4"} | V5_VERSIONS:
refers = []
if is_v2pro:
sv_emb = []
@ -945,7 +955,7 @@ def get_tts_wav(
pred_semantic = pred_semantic[:, -idx:].unsqueeze(0)
t3 = ttime()
if version not in {"v3", "v4"}:
if version not in {"v3", "v4"} | V5_VERSIONS:
if is_v2pro:
audio = (
vq_model.decode(
@ -983,41 +993,46 @@ def get_tts_wav(
ref_audio = resample(ref_audio, sr, tgt_sr, device)
mel2 = mel_fn(ref_audio) if version == "v3" else mel_fn_v4(ref_audio)
mel2 = norm_spec(mel2)
T_min = min(mel2.shape[2], fea_ref.shape[2])
mel2 = mel2[:, :, :T_min]
fea_ref = fea_ref[:, :, :T_min]
Tref = 468 if version == "v3" else 500
Tchunk = 934 if version == "v3" else 1000
if T_min > Tref:
mel2 = mel2[:, :, -Tref:]
fea_ref = fea_ref[:, :, -Tref:]
T_min = Tref
chunk_len = Tchunk - T_min
mel2 = mel2.to(dtype)
fea_todo, ge = vq_model.decode_encp(pred_semantic, phoneme_ids1, refer, ge, speed)
cfm_resss = []
idx = 0
while 1:
fea_todo_chunk = fea_todo[:, :, idx : idx + chunk_len]
if fea_todo_chunk.shape[-1] == 0:
break
idx += chunk_len
fea = torch.cat([fea_ref, fea_todo_chunk], 2).transpose(2, 1)
cfm_res = vq_model.cfm.inference(
fea, torch.LongTensor([fea.size(1)]).to(fea.device), mel2, sample_steps, inference_cfg_rate=0
)
cfm_res = cfm_res[:, :, mel2.shape[2] :]
mel2 = cfm_res[:, :, -T_min:]
fea_ref = fea_todo_chunk[:, :, -T_min:]
cfm_resss.append(cfm_res)
cfm_res = torch.cat(cfm_resss, 2)
if version in V5_VERSIONS:
fea_todo, ge = vq_model.decode_encp(pred_semantic, phoneme_ids1, refer, ge, speed)
cfm_res = synthesize_v5_mel(
vq_model, fea_ref, fea_todo, mel2.to(dtype), sample_steps, cfg_rate
).to(dtype)
else:
T_min = min(mel2.shape[2], fea_ref.shape[2])
mel2 = mel2[:, :, :T_min]
fea_ref = fea_ref[:, :, :T_min]
Tref = 468 if version == "v3" else 500
Tchunk = 934 if version == "v3" else 1000
if T_min > Tref:
mel2 = mel2[:, :, -Tref:]
fea_ref = fea_ref[:, :, -Tref:]
T_min = Tref
chunk_len = Tchunk - T_min
mel2 = mel2.to(dtype)
fea_todo, ge = vq_model.decode_encp(pred_semantic, phoneme_ids1, refer, ge, speed)
cfm_resss = []
idx = 0
while 1:
fea_todo_chunk = fea_todo[:, :, idx : idx + chunk_len]
if fea_todo_chunk.shape[-1] == 0:
break
idx += chunk_len
fea = torch.cat([fea_ref, fea_todo_chunk], 2).transpose(2, 1)
cfm_res = vq_model.cfm.inference(
fea, torch.LongTensor([fea.size(1)]).to(fea.device), mel2, sample_steps, inference_cfg_rate=0
)
cfm_res = cfm_res[:, :, mel2.shape[2] :]
mel2 = cfm_res[:, :, -T_min:]
fea_ref = fea_todo_chunk[:, :, -T_min:]
cfm_resss.append(cfm_res)
cfm_res = torch.cat(cfm_resss, 2)
cfm_res = denorm_spec(cfm_res)
if version == "v3":
if bigvgan_model == None:
init_bigvgan()
else: # v4
if hifigan_model == None:
init_hifigan()
else:
init_hifigan(version)
vocoder_model = bigvgan_model if version == "v3" else hifigan_model
with torch.inference_mode():
wav_gen = vocoder_model(cfm_res)
@ -1110,6 +1125,7 @@ def handle(
inp_refs,
sample_steps,
if_sr,
cfg_rate=None,
):
if (
refer_wav_path == ""
@ -1146,6 +1162,7 @@ def handle(
inp_refs,
sample_steps,
if_sr,
cfg_rate=cfg_rate,
),
media_type="audio/" + media_type,
)
@ -1352,8 +1369,9 @@ async def tts_endpoint(request: Request):
json_post_raw.get("temperature", 1.0),
json_post_raw.get("speed", 1.0),
json_post_raw.get("inp_refs", []),
json_post_raw.get("sample_steps", 32),
json_post_raw.get("sample_steps"),
json_post_raw.get("if_sr", False),
cfg_rate=json_post_raw.get("cfg_rate"),
)
@ -1370,8 +1388,9 @@ async def tts_endpoint(
temperature: float = 1.0,
speed: float = 1.0,
inp_refs: list = Query(default=[]),
sample_steps: int = 32,
sample_steps: int = None,
if_sr: bool = False,
cfg_rate: float = None,
):
return handle(
refer_wav_path,
@ -1387,6 +1406,7 @@ async def tts_endpoint(
inp_refs,
sample_steps,
if_sr,
cfg_rate=cfg_rate,
)

View File

@ -13,10 +13,9 @@ pretrained_sovits_name = {
"v1": "GPT_SoVITS/pretrained_models/s2G488k.pth",
"v2": "GPT_SoVITS/pretrained_models/gsv-v2final-pretrained/s2G2333k.pth",
"v3": "GPT_SoVITS/pretrained_models/s2Gv3.pth", ###v3v4还要检查vocoder,算了。。。
"v4": "GPT_SoVITS/pretrained_models/gsv-v4-pretrained/s2Gv4.pth",
"v5": "GPT_SoVITS/pretrained_models/gsv-v5-pretrained/s2Gv5dev.pth",
"v5dev": "GPT_SoVITS/pretrained_models/gsv-v5-pretrained/s2Gv5dev.pth",
"v5turbo": "GPT_SoVITS/pretrained_models/gsv-v5-pretrained/s2Gv5turbo.pth",
"v4": "GPT_SoVITS/pretrained_models/gsv-v4-pretrained/s2Gv4.pth",
"v5dev": "GPT_SoVITS/pretrained_models/gsv-v5-pretrained/s2Gv5dev.pth",
"v5turbo": "GPT_SoVITS/pretrained_models/gsv-v5-pretrained/s2Gv5turbo.pth",
"v2Pro": "GPT_SoVITS/pretrained_models/v2Pro/s2Gv2Pro.pth",
"v2ProPlus": "GPT_SoVITS/pretrained_models/v2Pro/s2Gv2ProPlus.pth",
}
@ -25,16 +24,15 @@ pretrained_gpt_name = {
"v1": "GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt",
"v2": "GPT_SoVITS/pretrained_models/gsv-v2final-pretrained/s1bert25hz-5kh-longer-epoch=12-step=369668.ckpt",
"v3": "GPT_SoVITS/pretrained_models/s1v3.ckpt",
"v4": "GPT_SoVITS/pretrained_models/s1v3.ckpt",
"v5": "GPT_SoVITS/pretrained_models/s1v3.ckpt",
"v5dev": "GPT_SoVITS/pretrained_models/s1v3.ckpt",
"v5turbo": "GPT_SoVITS/pretrained_models/s1v3.ckpt",
"v4": "GPT_SoVITS/pretrained_models/s1v3.ckpt",
"v5dev": "GPT_SoVITS/pretrained_models/s1v3.ckpt",
"v5turbo": "GPT_SoVITS/pretrained_models/s1v3.ckpt",
"v2Pro": "GPT_SoVITS/pretrained_models/s1v3.ckpt",
"v2ProPlus": "GPT_SoVITS/pretrained_models/s1v3.ckpt",
}
name2sovits_path = {
i18n("不训练直接推v5dev底模!"): pretrained_sovits_name["v5dev"],
i18n("不训练直接推v5turbo底模!"): pretrained_sovits_name["v5turbo"],
name2sovits_path = {
i18n("不训练直接推v5dev底模!"): pretrained_sovits_name["v5dev"],
i18n("不训练直接推v5turbo底模!"): pretrained_sovits_name["v5turbo"],
# i18n("不训练直接推v1底模!"): "GPT_SoVITS/pretrained_models/s2G488k.pth",
i18n("不训练直接推v2底模!"): "GPT_SoVITS/pretrained_models/gsv-v2final-pretrained/s2G2333k.pth",
# i18n("不训练直接推v3底模!"): "GPT_SoVITS/pretrained_models/s2Gv3.pth",
@ -54,6 +52,8 @@ SoVITS_weight_root = [
"SoVITS_weights_v2",
"SoVITS_weights_v3",
"SoVITS_weights_v4",
"SoVITS_weights_v5dev",
"SoVITS_weights_v5turbo",
"SoVITS_weights_v2Pro",
"SoVITS_weights_v2ProPlus",
]
@ -62,6 +62,8 @@ GPT_weight_root = [
"GPT_weights_v2",
"GPT_weights_v3",
"GPT_weights_v4",
"GPT_weights_v5dev",
"GPT_weights_v5turbo",
"GPT_weights_v2Pro",
"GPT_weights_v2ProPlus",
]
@ -70,6 +72,8 @@ SoVITS_weight_version2root = {
"v2": "SoVITS_weights_v2",
"v3": "SoVITS_weights_v3",
"v4": "SoVITS_weights_v4",
"v5dev": "SoVITS_weights_v5dev",
"v5turbo": "SoVITS_weights_v5turbo",
"v2Pro": "SoVITS_weights_v2Pro",
"v2ProPlus": "SoVITS_weights_v2ProPlus",
}
@ -78,6 +82,8 @@ GPT_weight_version2root = {
"v2": "GPT_weights_v2",
"v3": "GPT_weights_v3",
"v4": "GPT_weights_v4",
"v5dev": "GPT_weights_v5dev",
"v5turbo": "GPT_weights_v5turbo",
"v2Pro": "GPT_weights_v2Pro",
"v2ProPlus": "GPT_weights_v2ProPlus",
}

View File

@ -98,7 +98,7 @@ gpu_infos = GPU_INFOS
mem = memset
is_gpu_ok = IS_GPU
v3v4set = {"v3", "v4"}
v3v4set = {"v3", "v4", "v5dev", "v5turbo"}
def set_default():
@ -167,14 +167,15 @@ from config import pretrained_gpt_name, pretrained_sovits_name
def check_pretrained_is_exist(version):
pretrained_model_list = (
pretrained_sovits_name[version],
pretrained_sovits_name[version].replace("s2G", "s2D"),
pretrained_gpt_name[version],
"GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large",
"GPT_SoVITS/pretrained_models/chinese-hubert-base",
)
if version not in v3v4set:
pretrained_model_list += (pretrained_sovits_name[version].replace("s2G", "s2D"),)
_ = ""
for i in pretrained_model_list:
if "s2Dv3" not in i and "s2Dv4" not in i and os.path.exists(i) == False:
if os.path.exists(i) == False:
_ += f"\n {i}"
if _:
print("warning: ", i18n("以下模型不存在:") + _)
@ -1272,7 +1273,7 @@ def switch_version(version_):
set_default()
return (
{"__type__": "update", "value": pretrained_sovits_name[version]},
{"__type__": "update", "value": pretrained_sovits_name[version].replace("s2G", "s2D")},
{"__type__": "update", "value": pretrained_sovits_name[version].replace("s2G", "s2D") if version not in v3v4set else "", "visible": version not in v3v4set},
{"__type__": "update", "value": pretrained_gpt_name[version]},
{"__type__": "update", "value": pretrained_gpt_name[version]},
{"__type__": "update", "value": pretrained_sovits_name[version]},
@ -1495,7 +1496,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
version_checkbox = gr.Radio(
label=i18n("训练模型的版本"),
value=version,
choices=["v1", "v2", "v4", "v2Pro", "v2ProPlus"],
choices=["v2", "v2Pro", "v2ProPlus", "v4", "v5dev", "v5turbo"],
scale=5,
)
with gr.Accordion(label=i18n("预训练模型路径"), open=False):
@ -1519,7 +1520,8 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
)
pretrained_s2D = gr.Textbox(
label=i18n("预训练SoVITS-D模型路径"),
value=pretrained_sovits_name[version].replace("s2G", "s2D"),
value=pretrained_sovits_name[version].replace("s2G", "s2D") if version not in v3v4set else "",
visible=version not in v3v4set,
interactive=True,
lines=1,
max_lines=1,