mirror of
https://github.com/RVC-Boss/GPT-SoVITS.git
synced 2026-11-17 18:43:37 +08:00
Compare commits
5 Commits
18f3f4d211
...
e77aef2525
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e77aef2525 | ||
|
|
6d51ffceaf | ||
|
|
59c44686f5 | ||
|
|
04a44eb977 | ||
|
|
854fc555d6 |
@ -512,7 +512,6 @@ class TTS:
|
||||
|
||||
self.t2s_model: Text2SemanticLightningModule = None
|
||||
self.t2s_accel = None
|
||||
self.streaming_accel = None
|
||||
self.vits_model: Union[SynthesizerTrn, SynthesizerTrnV3] = None
|
||||
self.bert_tokenizer: AutoTokenizer = None
|
||||
self.bert_model: AutoModelForMaskedLM = None
|
||||
@ -679,9 +678,6 @@ class TTS:
|
||||
|
||||
|
||||
def init_t2s_weights(self, weights_path: str):
|
||||
if self.streaming_accel is not None:
|
||||
self.streaming_accel.close()
|
||||
self.streaming_accel = None
|
||||
if self.t2s_accel is not None:
|
||||
self.t2s_accel.close()
|
||||
self.t2s_accel = None
|
||||
@ -795,9 +791,6 @@ class TTS:
|
||||
print("Half precision is not supported on CPU.")
|
||||
return
|
||||
|
||||
if self.streaming_accel is not None:
|
||||
self.streaming_accel.close()
|
||||
self.streaming_accel = None
|
||||
if self.t2s_accel is not None:
|
||||
self.t2s_accel.close()
|
||||
self.t2s_accel = None
|
||||
@ -834,9 +827,6 @@ class TTS:
|
||||
Args:
|
||||
device: torch.device, the device to use for all models.
|
||||
"""
|
||||
if self.streaming_accel is not None:
|
||||
self.streaming_accel.close()
|
||||
self.streaming_accel = None
|
||||
if self.t2s_accel is not None:
|
||||
self.t2s_accel.close()
|
||||
self.t2s_accel = None
|
||||
@ -1104,6 +1094,41 @@ class TTS:
|
||||
"""
|
||||
self.stop_flag = True
|
||||
|
||||
def apply_cache_dit(self, inputs: dict = None, sample_steps: int = 32):
|
||||
"""cache-dit / DBCache switch for the DiT sampler. **Default: off.**
|
||||
|
||||
Reads the same field names as the HTTP API, so a request, an environment variable and
|
||||
a direct call give the same result:
|
||||
|
||||
cache_dit: off | auto | f4 | f8 | f16 (alias: dbcache)
|
||||
dbcache_fn, dbcache_threshold, dbcache_warmup, dbcache_downsample
|
||||
dbcache_warmup_interval, dbcache_max_cached, dbcache_max_cont, dbcache_max_accum
|
||||
|
||||
Env fallbacks: ``GSV_DBCACHE``, ``GSV_DBCACHE_THRESHOLD``, ``GSV_DBCACHE_WARMUP``,
|
||||
``GSV_DBCACHE_DOWNSAMPLE`` (and the other ``GSV_DBCACHE_*`` in module/cache_dit.py).
|
||||
``auto`` = F8 when ``sample_steps >= 16`` and off below that, so a 4-step run is never
|
||||
cached by accident (its warmup budget already covers the whole run, so there is nothing
|
||||
to gain and the quality risk is real).
|
||||
"""
|
||||
cfm = getattr(self.vits_model, "cfm", None)
|
||||
if cfm is None or not hasattr(cfm, "set_dbcache"):
|
||||
return None
|
||||
from module.cache_dit import config_from_params
|
||||
|
||||
cfg = config_from_params(dict(inputs or {}), int(sample_steps),
|
||||
getattr(cfm, "cache_depth", 0) or None)
|
||||
if cfg.enabled and str(getattr(self.configs, "version", "")) == "v5turbo" and not cfg.force:
|
||||
# 4-step turbo: nothing to cache, keep it off
|
||||
print(i18n("cache-dit: not supported for v5turbo (4 steps), staying off"))
|
||||
cfg.fn = 0
|
||||
cfm.set_dbcache(cfg)
|
||||
if cfg.enabled:
|
||||
if int(sample_steps) < 16:
|
||||
print(i18n("cache-dit: only %d sampling steps, expect a quality drop "
|
||||
"(4-step v5turbo should keep cache-dit off)" % int(sample_steps)))
|
||||
print(i18n("cache-dit: %s" % cfg))
|
||||
return cfg
|
||||
|
||||
@torch.no_grad()
|
||||
def run(self, inputs: dict):
|
||||
"""
|
||||
@ -1131,6 +1156,13 @@ class TTS:
|
||||
"parallel_infer": True, # bool. whether to use parallel inference.
|
||||
"repetition_penalty": 1.35, # float. repetition penalty for T2S model.
|
||||
"sample_steps": 32, # int. number of sampling steps for VITS model V3.
|
||||
"cfg_rate": 1.30, # float. classifier-free-guidance rate for V5dev/V5.
|
||||
"cache_dit": "off", # str. off|auto|f4|f8|f16. DBCache for the V3/V4/V5 DiT:
|
||||
# fewer DiT forwards, near-identical audio. off by default.
|
||||
"dbcache_fn": 8, # int. front blocks (auto = F8 when sample_steps >= 16).
|
||||
"dbcache_threshold": 0.02, # float. relative residual-diff threshold.
|
||||
"dbcache_warmup": 8, # int. forced full-compute steps at the start.
|
||||
"dbcache_downsample": 4, # int. time downsample of the cached Fn residual.
|
||||
"super_sampling": False, # bool. whether to use super-sampling for audio when using VITS model V3.
|
||||
"return_fragment": False, # bool. step by step return the audio fragment. (Best Quality, Slowest response speed. old version of streaming mode)
|
||||
"streaming_mode": False, # bool. return audio chunk by chunk. (Medium quality, Slow response speed)
|
||||
@ -1170,6 +1202,8 @@ class TTS:
|
||||
cfg_rate = inputs.get("cfg_rate")
|
||||
sample_steps = (4 if self.configs.version == "v5turbo" else 32) if sample_steps is None else int(sample_steps)
|
||||
cfg_rate = (1.30 if self.configs.version == "v5dev" else 0.0) if cfg_rate is None else float(cfg_rate)
|
||||
# cache-dit (DBCache): off unless the request asks for it, see module/cache_dit.py
|
||||
self.apply_cache_dit(inputs, sample_steps)
|
||||
super_sampling = inputs.get("super_sampling", False)
|
||||
streaming_mode = inputs.get("streaming_mode", False)
|
||||
overlap_length = inputs.get("overlap_length", 2)
|
||||
@ -1177,31 +1211,7 @@ class TTS:
|
||||
fixed_length_chunk = inputs.get("fixed_length_chunk", False)
|
||||
chunk_split_thershold = 0.0 # 该值代表语义token与mute token的余弦相似度阈值,若大于该阈值,则视为可切分点。
|
||||
|
||||
gpt_streaming = inputs.get("gpt_streaming", False)
|
||||
if gpt_streaming:
|
||||
from .streaming import (validate_streaming_options, PipelineStop, semantic_updates,
|
||||
v2_token_chunks, v5_reference, stream_v5_windows,
|
||||
StreamingWindowSchedule)
|
||||
validate_streaming_options(inputs, self.configs.version)
|
||||
stream_stop = PipelineStop(self)
|
||||
stream_options = dict(inputs, sample_steps=sample_steps, cfg_rate=cfg_rate,
|
||||
speed_factor=speed_factor,
|
||||
streaming_chunk_seconds=inputs.get("streaming_chunk_seconds", 2.0),
|
||||
streaming_lookahead_seconds=inputs.get("streaming_lookahead_seconds", 0.12),
|
||||
streaming_overlap_seconds=inputs.get("streaming_overlap_seconds", 0.04),
|
||||
gpt_streaming_lookahead_seconds=inputs.get("gpt_streaming_lookahead_seconds", 0.5))
|
||||
acoustic_rng = torch.Generator(device=self.configs.device).manual_seed(actual_seed)
|
||||
# Reference semantic codes depend on the active SoVITS model.
|
||||
if getattr(self, "_streaming_ref_model", None) != id(self.vits_model):
|
||||
ref_audio_path = ref_audio_path or self.prompt_cache["ref_audio_path"]
|
||||
if not ref_audio_path:
|
||||
raise ValueError("A reloadable reference audio is required after model changes")
|
||||
self.prompt_cache["ref_audio_path"] = None
|
||||
self._streaming_ref_model = id(self.vits_model)
|
||||
if self.prompt_cache["prompt_lang"] != prompt_lang:
|
||||
self.prompt_cache["prompt_text"] = None
|
||||
|
||||
if streaming_mode and self.configs.use_vocoder and not gpt_streaming:
|
||||
if streaming_mode and self.configs.use_vocoder:
|
||||
print(i18n("SoVits V3/4模型不支持流式推理模式,已自动回退到分段返回模式"))
|
||||
streaming_mode = False
|
||||
return_fragment = True
|
||||
@ -1253,7 +1263,7 @@ class TTS:
|
||||
assert prompt_lang in self.configs.languages
|
||||
|
||||
if no_prompt_text and self.configs.use_vocoder:
|
||||
raise NO_PROMPT_ERROR("prompt_text cannot be empty when using SoVITS_V3")
|
||||
raise NO_PROMPT_ERROR("SoVITS V3/V4/V5 do not support reference-text-free mode; prompt_text cannot be empty")
|
||||
|
||||
if ref_audio_path in [None, ""] and (
|
||||
(self.prompt_cache["prompt_semantic"] is None) or (self.prompt_cache["refer_spec"] in [None, []])
|
||||
@ -1321,15 +1331,7 @@ class TTS:
|
||||
)
|
||||
else:
|
||||
print(f"############ {i18n('切分文本')} ############")
|
||||
# Preserve cut0's single-session behavior; other methods use the
|
||||
# frontend's normal punctuation/newline and long-text handling.
|
||||
texts = self.text_preprocessor.pre_seg_text(
|
||||
text.replace("\n", " ") if gpt_streaming and text_split_method == "cut0" else text,
|
||||
text_lang,
|
||||
text_split_method,
|
||||
)
|
||||
if gpt_streaming and text_split_method == "cut0" and len(texts) > 1:
|
||||
raise ValueError("Text exceeds the single-session frontend limit; shorten the input")
|
||||
texts = self.text_preprocessor.pre_seg_text(text, text_lang, text_split_method)
|
||||
data = []
|
||||
for i in range(len(texts)):
|
||||
if i % batch_size == 0:
|
||||
@ -1365,8 +1367,6 @@ class TTS:
|
||||
return batch[0]
|
||||
|
||||
t2 = time.perf_counter()
|
||||
streaming_tokens = streaming_audio = semantic_token_generator = None
|
||||
streaming_reference = None
|
||||
try:
|
||||
print("############ 推理 ############")
|
||||
###### inference ######
|
||||
@ -1376,8 +1376,6 @@ class TTS:
|
||||
is_first_package = True
|
||||
output_sr = self.configs.sampling_rate if not self.configs.use_vocoder else self.vocoder_configs["sr"]
|
||||
for item in data:
|
||||
if gpt_streaming and stream_stop.is_set():
|
||||
break
|
||||
t3 = time.perf_counter()
|
||||
if return_fragment or streaming_mode:
|
||||
item = make_batch(item)
|
||||
@ -1536,59 +1534,27 @@ class TTS:
|
||||
)
|
||||
batch_audio_fragment.append(audio_fragment)
|
||||
|
||||
elif gpt_streaming and self.configs.version in V5_VERSIONS:
|
||||
if streaming_reference is None:
|
||||
streaming_reference = v5_reference(self)
|
||||
reference = streaming_reference
|
||||
# Validate the acoustic context before starting the AR owner thread.
|
||||
frames = round(stream_options["streaming_chunk_seconds"] * 100)
|
||||
future = round(stream_options["streaming_lookahead_seconds"] * 100)
|
||||
if reference[6].shape[-1] + frames + future > 1000:
|
||||
raise ValueError("Reference plus streaming window exceeds the V5 context")
|
||||
streaming_tokens = semantic_updates(self, all_phoneme_ids[0], prompt,
|
||||
all_bert_features[0], stream_options, stream_stop)
|
||||
streaming_audio = stream_v5_windows(self.vits_model, self.vocoder, denorm_spec,
|
||||
self.configs.device, streaming_tokens, batch_phones[0], reference,
|
||||
stream_options, stream_stop)
|
||||
for pcm in streaming_audio:
|
||||
yield output_sr, pcm
|
||||
streaming_audio.close()
|
||||
streaming_tokens.close()
|
||||
streaming_audio = streaming_tokens = None
|
||||
if not stream_stop.is_set() and fragment_interval:
|
||||
yield output_sr, np.zeros(int(output_sr * fragment_interval), dtype=np.int16)
|
||||
t4 = time.perf_counter()
|
||||
else:
|
||||
# refer_audio_spec: torch.Tensor = [
|
||||
# item.to(dtype=self.precision, device=self.configs.device)
|
||||
# for item in self.prompt_cache["refer_spec"]
|
||||
# ]
|
||||
window_schedule = None
|
||||
if gpt_streaming:
|
||||
if inputs.get("streaming_window_mode", "fixed") == "hybrid":
|
||||
window_schedule = StreamingWindowSchedule(min_chunk_length, min_chunk_length * 4)
|
||||
streaming_tokens = semantic_updates(self, all_phoneme_ids[0], prompt,
|
||||
all_bert_features[0], stream_options, stream_stop)
|
||||
semantic_token_generator = v2_token_chunks(streaming_tokens, min_chunk_length,
|
||||
self.configs.mute_emb_sim_matrix if not fixed_length_chunk else None,
|
||||
chunk_split_thershold, schedule=window_schedule)
|
||||
else:
|
||||
semantic_token_generator =self.t2s_model.model.infer_panel(
|
||||
all_phoneme_ids[0].unsqueeze(0),
|
||||
all_phoneme_lens,
|
||||
prompt,
|
||||
all_bert_features[0].unsqueeze(0),
|
||||
top_k=top_k,
|
||||
top_p=top_p,
|
||||
temperature=temperature,
|
||||
early_stop_num=self.configs.hz * self.configs.max_sec,
|
||||
max_len=max_len,
|
||||
repetition_penalty=repetition_penalty,
|
||||
streaming_mode=True,
|
||||
chunk_length=min_chunk_length,
|
||||
mute_emb_sim_matrix=self.configs.mute_emb_sim_matrix if not fixed_length_chunk else None,
|
||||
chunk_split_thershold=chunk_split_thershold,
|
||||
)
|
||||
semantic_token_generator =self.t2s_model.model.infer_panel(
|
||||
all_phoneme_ids[0].unsqueeze(0),
|
||||
all_phoneme_lens,
|
||||
prompt,
|
||||
all_bert_features[0].unsqueeze(0),
|
||||
top_k=top_k,
|
||||
top_p=top_p,
|
||||
temperature=temperature,
|
||||
early_stop_num=self.configs.hz * self.configs.max_sec,
|
||||
max_len=max_len,
|
||||
repetition_penalty=repetition_penalty,
|
||||
streaming_mode=True,
|
||||
chunk_length=min_chunk_length,
|
||||
mute_emb_sim_matrix=self.configs.mute_emb_sim_matrix if not fixed_length_chunk else None,
|
||||
chunk_split_thershold=chunk_split_thershold,
|
||||
)
|
||||
t4 = time.perf_counter()
|
||||
t_34 += t4 - t3
|
||||
phones = batch_phones[0].unsqueeze(0).to(self.configs.device)
|
||||
@ -1653,8 +1619,7 @@ class TTS:
|
||||
result_length=semantic_tokens.shape[-1]+overlap_len if not is_first_chunk else None,
|
||||
overlap_frames=last_latent[:,:,-overlap_len*(2 if self.vits_model.semantic_frame_rate == "25hz" else 1):] \
|
||||
if last_latent is not None else None,
|
||||
padding_length=token_padding_length,
|
||||
**({"generator": acoustic_rng} if gpt_streaming else {})
|
||||
padding_length=token_padding_length
|
||||
)
|
||||
audio_chunk=audio_chunk.detach()[0, 0, :]
|
||||
else:
|
||||
@ -1669,7 +1634,7 @@ class TTS:
|
||||
audio_chunk_ = audio_chunk_[:-overlap_size]
|
||||
elif is_first_chunk and is_final:
|
||||
is_first_chunk = False
|
||||
elif not is_first_chunk and (not is_final or gpt_streaming):
|
||||
elif not is_first_chunk and not is_final:
|
||||
audio_chunk_ = self.sola_algorithm([last_audio_chunk, audio_chunk_], overlap_size)
|
||||
audio_chunk_ = (
|
||||
audio_chunk_[last_audio_chunk.shape[0]-overlap_size:-overlap_size] if not is_final \
|
||||
@ -1678,7 +1643,7 @@ class TTS:
|
||||
|
||||
last_latent = latent
|
||||
last_audio_chunk = audio_chunk
|
||||
audio_result = self.audio_postprocess(
|
||||
yield self.audio_postprocess(
|
||||
[[audio_chunk_]],
|
||||
output_sr,
|
||||
None,
|
||||
@ -1687,21 +1652,13 @@ class TTS:
|
||||
0.0,
|
||||
super_sampling if self.configs.use_vocoder and self.configs.version == "v3" else False,
|
||||
)
|
||||
if window_schedule is not None:
|
||||
window_schedule.observe(semantic_tokens.numel(), audio_result[1].size, output_sr)
|
||||
yield audio_result
|
||||
|
||||
if is_first_package:
|
||||
print(f"first_package_delay: {time.perf_counter()-t0:.3f}")
|
||||
is_first_package = False
|
||||
|
||||
|
||||
if gpt_streaming:
|
||||
semantic_token_generator.close()
|
||||
streaming_tokens.close()
|
||||
semantic_token_generator = streaming_tokens = None
|
||||
if not gpt_streaming or not stream_stop.is_set():
|
||||
yield output_sr, np.zeros(int(output_sr*fragment_interval), dtype=np.int16)
|
||||
yield output_sr, np.zeros(int(output_sr*fragment_interval), dtype=np.int16)
|
||||
|
||||
t5 = time.perf_counter()
|
||||
t_45 += t5 - t4
|
||||
@ -1740,8 +1697,6 @@ class TTS:
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
if gpt_streaming:
|
||||
raise
|
||||
traceback.print_exc()
|
||||
# 必须返回一个空音频, 否则会导致显存不释放。
|
||||
yield 16000, np.zeros(int(16000), dtype=np.int16)
|
||||
@ -1754,11 +1709,6 @@ class TTS:
|
||||
self.init_vits_weights(self.configs.vits_weights_path)
|
||||
raise e
|
||||
finally:
|
||||
if gpt_streaming:
|
||||
stream_stop.set()
|
||||
for iterator in (streaming_audio, semantic_token_generator, streaming_tokens):
|
||||
if iterator is not None:
|
||||
iterator.close()
|
||||
self.empty_cache()
|
||||
|
||||
def empty_cache(self):
|
||||
|
||||
@ -171,7 +171,58 @@ class DiT(nn.Module):
|
||||
text_cache=None, # torch tensor as text_embed
|
||||
dt_cache=None, # torch tensor as dt
|
||||
static_cache=None,
|
||||
# cache-dit / DBCache (see module/cache_dit.py). `dbcache=None` => untouched path.
|
||||
dbcache=None, # module.cache_dit.DBCacheState
|
||||
dbcache_branch=0, # 0 = CFG positive, 1 = CFG negative
|
||||
):
|
||||
x, t, mask, rope, text_embed, dt = self.trunk(
|
||||
x0, cond0, x_lens, time, dt_base_bootstrap=dt_base_bootstrap, text0=text0,
|
||||
drop_audio_cond=drop_audio_cond, drop_text=drop_text, infer=infer,
|
||||
text_cache=text_cache, dt_cache=dt_cache, static_cache=static_cache,
|
||||
)
|
||||
|
||||
residual = x
|
||||
if self.long_skip_connection is None:
|
||||
residual = None
|
||||
|
||||
if dbcache is None or not dbcache.enabled:
|
||||
# ---- original path: all blocks, then head ------------------------------
|
||||
x = self.run_blocks(x, t, mask, rope, 0, self.depth, use_grad_ckpt)
|
||||
else:
|
||||
# ---- cache-dit: front -> (hit: head only | miss: mid -> head) -----------
|
||||
fn = max(0, min(int(dbcache.fn), self.depth - 1))
|
||||
x_f = self.run_blocks(x, t, mask, rope, 0, fn, use_grad_ckpt)
|
||||
fd = dbcache.front_update(dbcache_branch, x_f, x)
|
||||
if dbcache.can_cache(dbcache_branch, fd):
|
||||
x = x_f + dbcache.mid_residual(dbcache_branch)
|
||||
dbcache.on_hit(dbcache_branch)
|
||||
if dbcache.trace:
|
||||
print("[dbcache] step=%d HIT fd=%.5f (hits=%d misses=%d)"
|
||||
% (dbcache.step, fd, dbcache.branch(dbcache_branch).hits,
|
||||
dbcache.branch(dbcache_branch).misses), flush=True)
|
||||
else:
|
||||
x = self.run_blocks(x_f, t, mask, rope, fn, self.depth, use_grad_ckpt)
|
||||
dbcache.on_miss(dbcache_branch, x, x_f)
|
||||
if dbcache.trace:
|
||||
print("[dbcache] step=%d MISS fd=%.5f (hits=%d misses=%d)"
|
||||
% (dbcache.step, fd, dbcache.branch(dbcache_branch).hits,
|
||||
dbcache.branch(dbcache_branch).misses), flush=True)
|
||||
|
||||
output = self.head(x, t, residual)
|
||||
|
||||
if infer:
|
||||
return output, text_embed if static_cache is None else None, dt
|
||||
else:
|
||||
return output
|
||||
|
||||
# ---------------------------------------------------------------------------------
|
||||
# Three-segment split used by cache-dit; the weights are shared, nothing is copied.
|
||||
# ---------------------------------------------------------------------------------
|
||||
def trunk(self, x0, cond0, x_lens, time, dt_base_bootstrap=None, text0=None,
|
||||
drop_audio_cond=False, drop_text=False, infer=False, text_cache=None,
|
||||
dt_cache=None, static_cache=None):
|
||||
"""Everything before the transformer blocks: t/dt embedding + input embedding."""
|
||||
text_embed = None
|
||||
x = x0.transpose(2, 1)
|
||||
cond = cond0.transpose(2, 1)
|
||||
text = text0.transpose(2, 1)
|
||||
@ -207,22 +258,22 @@ class DiT(nn.Module):
|
||||
rope = (static_cache["rope"] if static_cache is not None
|
||||
else self.rotary_embed.forward_from_seq_len(seq_len))
|
||||
|
||||
if self.long_skip_connection is not None:
|
||||
residual = x
|
||||
return x, t, mask, rope, text_embed, dt
|
||||
|
||||
for block in self.transformer_blocks:
|
||||
def run_blocks(self, x, t, mask, rope, start=0, end=None, use_grad_ckpt=False):
|
||||
"""`front` is blocks[0:fn], `mid` is blocks[fn:depth]."""
|
||||
end = self.depth if end is None else end
|
||||
for block in self.transformer_blocks[start:end]:
|
||||
if use_grad_ckpt:
|
||||
x = checkpoint(self.ckpt_wrapper(block), x, t, mask, rope, use_reentrant=False)
|
||||
else:
|
||||
x = block(x, t, mask=mask, rope=rope)
|
||||
return x
|
||||
|
||||
def head(self, x, t, residual=None):
|
||||
"""`norm_out -> proj_out`, computed on every step, hits included."""
|
||||
if self.long_skip_connection is not None:
|
||||
x = self.long_skip_connection(torch.cat((x, residual), dim=-1))
|
||||
|
||||
x = self.norm_out(x, t)
|
||||
output = self.proj_out(x)
|
||||
return self.proj_out(x)
|
||||
|
||||
if infer:
|
||||
return output, text_embed if static_cache is None else None, dt
|
||||
else:
|
||||
return output
|
||||
|
||||
@ -3,6 +3,7 @@ import os
|
||||
import soundfile as sf
|
||||
|
||||
from tools.i18n.i18n import I18nAuto
|
||||
import GPT_SoVITS.inference_webui as webui_ui
|
||||
from GPT_SoVITS.inference_webui import change_gpt_weights, change_sovits_weights, get_tts_wav
|
||||
|
||||
i18n = I18nAuto()
|
||||
@ -67,9 +68,25 @@ def main():
|
||||
help="Language of the target text",
|
||||
)
|
||||
parser.add_argument("--output_path", required=True, help="Path to the output directory")
|
||||
parser.add_argument(
|
||||
"--cache_dit",
|
||||
default="off",
|
||||
choices=["off", "auto", "f4", "f8", "f16"],
|
||||
help="cache-dit (DBCache) for the V3/V4/V5dev DiT, off by default; "
|
||||
"fewer DiT forwards with near-identical audio",
|
||||
)
|
||||
parser.add_argument("--dbcache_fn", type=int, default=-1, help="front blocks (default 8)")
|
||||
parser.add_argument("--dbcache_threshold", type=float, default=0.02, help="residual-diff threshold")
|
||||
parser.add_argument("--dbcache_warmup", type=int, default=8, help="full-compute steps first")
|
||||
parser.add_argument("--dbcache_downsample", type=int, default=4, help="residual time downsample")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
webui_ui.update_cache_dit(
|
||||
args.cache_dit, args.dbcache_fn, args.dbcache_threshold,
|
||||
args.dbcache_warmup, args.dbcache_downsample,
|
||||
)
|
||||
|
||||
synthesize(
|
||||
args.gpt_model,
|
||||
args.sovits_model,
|
||||
|
||||
@ -100,8 +100,8 @@ cnhubert.cnhubert_base_path = cnhubert_base_path
|
||||
import random
|
||||
|
||||
from tools.acceleration import create_acceleration, cuda_graph_available, flash_attention_available
|
||||
from module.models import Generator, SynthesizerTrn, SynthesizerTrnV3
|
||||
from module.models_v5 import synthesize_v5_mel, V5_VERSIONS
|
||||
from module.models import CFM, Generator, SynthesizerTrn, SynthesizerTrnV3
|
||||
from module.models_v5 import V5_VERSIONS
|
||||
|
||||
|
||||
def set_seed(seed):
|
||||
@ -283,14 +283,14 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None)
|
||||
{
|
||||
"__type__": "update",
|
||||
"visible": visible_sample_steps,
|
||||
"value": (4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8),
|
||||
"value": (32 if model_version == "v3" else 8),
|
||||
"choices": [4, 8, 16, 32],
|
||||
},
|
||||
{"__type__": "update", "visible": visible_inp_refs},
|
||||
{"__type__": "update", "value": False, "interactive": True if model_version not in v3v4set else False},
|
||||
{"__type__": "update", "visible": True if model_version == "v3" else False},
|
||||
{"__type__": "update", "value": i18n("模型加载中,请等待"), "interactive": False},
|
||||
{"__type__": "update", "visible": model_version in V5_VERSIONS, "value": (1.30 if model_version == "v5dev" else 0.0)},
|
||||
{"__type__": "update", "visible": False, "value": 0.0, "interactive": False},
|
||||
)
|
||||
|
||||
dict_s2 = load_sovits_new(sovits_path)
|
||||
@ -355,6 +355,10 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None)
|
||||
# torch.save(vq_model.state_dict(),"merge_win.pth")
|
||||
vq_model.eval()
|
||||
|
||||
if model_version in V5_VERSIONS:
|
||||
vq_model.cfm.noise_temperature = None
|
||||
vq_model.cfm.use_static_cache = False
|
||||
|
||||
yield (
|
||||
{"__type__": "update", "choices": list(dict_language.keys())},
|
||||
{"__type__": "update", "choices": list(dict_language.keys())},
|
||||
@ -365,14 +369,14 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None)
|
||||
{
|
||||
"__type__": "update",
|
||||
"visible": visible_sample_steps,
|
||||
"value": (4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8),
|
||||
"value": (32 if model_version == "v3" else 8),
|
||||
"choices": [4, 8, 16, 32],
|
||||
},
|
||||
{"__type__": "update", "visible": visible_inp_refs},
|
||||
{"__type__": "update", "value": False, "interactive": True if model_version not in v3v4set else False},
|
||||
{"__type__": "update", "visible": True if model_version == "v3" else False},
|
||||
{"__type__": "update", "value": i18n("合成语音"), "interactive": True},
|
||||
{"__type__": "update", "visible": model_version in V5_VERSIONS, "value": (1.30 if model_version == "v5dev" else 0.0)},
|
||||
{"__type__": "update", "visible": False, "value": 0.0, "interactive": False},
|
||||
)
|
||||
with open("./weight.json") as f:
|
||||
data = f.read()
|
||||
@ -492,13 +496,12 @@ def init_hifigan():
|
||||
hifigan_model.eval()
|
||||
hifigan_model.remove_weight_norm()
|
||||
state_dict_g = torch.load(
|
||||
"%s/GPT_SoVITS/pretrained_models/gsv-%s-pretrained/vocoder.pth" %
|
||||
(now_dir, "v5" if model_version in V5_VERSIONS else model_version),
|
||||
"%s/GPT_SoVITS/pretrained_models/gsv-v4-pretrained/vocoder.pth" % (now_dir,),
|
||||
map_location="cpu",
|
||||
weights_only=False,
|
||||
)
|
||||
print("loading vocoder", hifigan_model.load_state_dict(state_dict_g))
|
||||
hifigan_version = model_version
|
||||
hifigan_version = "v4"
|
||||
clean_bigvgan_model()
|
||||
clean_sv_cn_model()
|
||||
if is_half == True:
|
||||
@ -841,6 +844,36 @@ def audio_sr(audio, sr):
|
||||
# cache_tokens={}#暂未实现清理机制
|
||||
cache = {}
|
||||
|
||||
# cache-dit (DBCache) for the DiT sampler: off by default. The UI writes here, get_tts_wav
|
||||
# reads it; the GSV_DBCACHE* environment variables work as well.
|
||||
CACHE_DIT = {"cache_dit": "off", "dbcache_fn": None, "dbcache_threshold": 0.02,
|
||||
"dbcache_warmup": 8, "dbcache_downsample": 4}
|
||||
|
||||
|
||||
def update_cache_dit(mode, fn, thr, warmup, ds):
|
||||
CACHE_DIT["cache_dit"] = str(mode or "off")
|
||||
CACHE_DIT["dbcache_fn"] = None if fn in (None, "") or int(fn) < 0 else int(fn)
|
||||
CACHE_DIT["dbcache_threshold"] = float(thr) if thr not in (None, "") else 0.02
|
||||
CACHE_DIT["dbcache_warmup"] = int(warmup) if warmup not in (None, "") else 8
|
||||
CACHE_DIT["dbcache_downsample"] = int(ds) if ds not in (None, "") else 4
|
||||
|
||||
|
||||
def apply_cache_dit(vq_model, sample_steps, params=None):
|
||||
"""cache-dit (DBCache) switch; off unless configured (UI or GSV_DBCACHE* env)."""
|
||||
cfm = getattr(vq_model, "cfm", None)
|
||||
if cfm is None or not hasattr(cfm, "set_dbcache"):
|
||||
return None
|
||||
from module.cache_dit import config_from_params
|
||||
|
||||
cfg = config_from_params(dict(params if params is not None else CACHE_DIT),
|
||||
int(sample_steps), getattr(cfm, "cache_depth", 0) or None)
|
||||
if cfg.enabled and str(getattr(vq_model, "version", "")) == "v5turbo" and not cfg.force:
|
||||
cfg.fn = 0 # 4-step turbo: no gain and unstable quality, keep it off
|
||||
cfm.set_dbcache(cfg)
|
||||
if cfg.enabled:
|
||||
print("cache-dit:", cfg)
|
||||
return cfg
|
||||
|
||||
|
||||
def get_tts_wav(
|
||||
ref_wav_path,
|
||||
@ -865,10 +898,11 @@ def get_tts_wav(
|
||||
):
|
||||
global cache, t2s_model_cudagraph
|
||||
if sample_steps is None:
|
||||
sample_steps = 4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8
|
||||
sample_steps = 32 if model_version == "v3" else 8
|
||||
if cfg_rate is None:
|
||||
cfg_rate = 1.30 if model_version == "v5dev" else 0.0
|
||||
cfg_rate = 0.0
|
||||
sample_steps, cfg_rate = int(sample_steps), float(cfg_rate)
|
||||
apply_cache_dit(globals().get("vq_model"), sample_steps)
|
||||
if ref_wav_path:
|
||||
pass
|
||||
else:
|
||||
@ -1081,30 +1115,27 @@ def get_tts_wav(
|
||||
fea_todo, ge = vq_model.decode_encp(pred_semantic, phoneme_ids1, refer, ge, speed)
|
||||
cfm_resss = []
|
||||
idx = 0
|
||||
if model_version in V5_VERSIONS:
|
||||
cfm_res = synthesize_v5_mel(vq_model, fea_ref, fea_todo, mel2, sample_steps, cfg_rate)
|
||||
cfm_res = cfm_res.to(fea_todo.dtype)
|
||||
else:
|
||||
while 1:
|
||||
fea_todo_chunk = fea_todo[:, :, idx : idx + chunk_len]
|
||||
if fea_todo_chunk.shape[-1] == 0:
|
||||
break
|
||||
idx += chunk_len
|
||||
fea = torch.cat([fea_ref, fea_todo_chunk], 2).transpose(2, 1)
|
||||
cfm_res = vq_model.cfm.inference(
|
||||
fea, torch.LongTensor([fea.size(1)]).to(fea.device), mel2, sample_steps, inference_cfg_rate=0
|
||||
)
|
||||
cfm_res = cfm_res[:, :, mel2.shape[2] :]
|
||||
mel2 = cfm_res[:, :, -T_min:]
|
||||
fea_ref = fea_todo_chunk[:, :, -T_min:]
|
||||
cfm_resss.append(cfm_res)
|
||||
cfm_res = torch.cat(cfm_resss, 2)
|
||||
while 1:
|
||||
fea_todo_chunk = fea_todo[:, :, idx : idx + chunk_len]
|
||||
if fea_todo_chunk.shape[-1] == 0:
|
||||
break
|
||||
idx += chunk_len
|
||||
fea = torch.cat([fea_ref, fea_todo_chunk], 2).transpose(2, 1)
|
||||
cfm_res = CFM.inference(
|
||||
vq_model.cfm, fea, torch.LongTensor([fea.size(1)]).to(fea.device), mel2, sample_steps,
|
||||
temperature=1.0, inference_cfg_rate=0,
|
||||
)
|
||||
cfm_res = cfm_res[:, :, mel2.shape[2] :]
|
||||
mel2 = cfm_res[:, :, -T_min:]
|
||||
fea_ref = fea_todo_chunk[:, :, -T_min:]
|
||||
cfm_resss.append(cfm_res)
|
||||
cfm_res = torch.cat(cfm_resss, 2)
|
||||
cfm_res = denorm_spec(cfm_res)
|
||||
if model_version == "v3":
|
||||
if bigvgan_model == None:
|
||||
init_bigvgan()
|
||||
else: # v4
|
||||
if hifigan_model is None or hifigan_version != model_version:
|
||||
if hifigan_model is None or hifigan_version != "v4":
|
||||
init_hifigan()
|
||||
vocoder_model = bigvgan_model if model_version == "v3" else hifigan_model
|
||||
with torch.inference_mode():
|
||||
@ -1299,7 +1330,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
|
||||
with gr.Column(scale=13):
|
||||
ref_text_free = gr.Checkbox(
|
||||
label=i18n("开启无参考文本模式。不填参考文本亦相当于开启。")
|
||||
+ i18n("v3暂不支持该模式,使用了会报错。"),
|
||||
+ i18n("v3、v4、v5 均不支持无参考文本模式,请填写参考文本!"),
|
||||
value=False,
|
||||
interactive=True if model_version not in v3v4set else False,
|
||||
show_label=True,
|
||||
@ -1322,8 +1353,8 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
|
||||
)
|
||||
cfg_rate = gr.Slider(
|
||||
minimum=0, maximum=2, step=0.05,
|
||||
value=(1.30 if model_version == "v5dev" else 0.0), label="CFG",
|
||||
visible=model_version in V5_VERSIONS, interactive=True, scale=1,
|
||||
value=0.0, label="CFG",
|
||||
visible=False, interactive=False, scale=1,
|
||||
)
|
||||
inp_refs = (
|
||||
gr.File(
|
||||
@ -1344,7 +1375,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
|
||||
sample_steps = (
|
||||
gr.Radio(
|
||||
label=i18n("采样步数,如果觉得电,提高试试,如果觉得慢,降低试试"),
|
||||
value=(4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8),
|
||||
value=(32 if model_version == "v3" else 8),
|
||||
choices=[4, 8, 16, 32],
|
||||
visible=True,
|
||||
)
|
||||
@ -1353,9 +1384,29 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
|
||||
label=i18n("采样步数,如果觉得电,提高试试,如果觉得慢,降低试试"),
|
||||
choices=[4, 8, 16, 32],
|
||||
visible=False,
|
||||
value=(4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8),
|
||||
value=(32 if model_version == "v3" else 8),
|
||||
)
|
||||
)
|
||||
with gr.Accordion(i18n("DiT 加速 cache-dit(默认关闭)"), open=False):
|
||||
with gr.Row():
|
||||
cache_dit_mode = gr.Dropdown(
|
||||
label=i18n("cache-dit"),
|
||||
choices=["off", "auto", "f4", "f8", "f16"],
|
||||
value="off", scale=2,
|
||||
)
|
||||
dbcache_fn = gr.Number(label="fn", value=-1, precision=0, scale=1)
|
||||
dbcache_threshold = gr.Number(label="thr", value=0.02, scale=1)
|
||||
dbcache_warmup = gr.Number(label="warmup", value=8, precision=0, scale=1)
|
||||
dbcache_downsample = gr.Number(label="ds", value=4, precision=0, scale=1)
|
||||
gr.Markdown(i18n("DBCache:只算前 fn 个 block,残差变化小于 thr 时跳过中间 block。"
|
||||
"fn=-1 表示跟随模式(auto = 步数>=16 时 F8)。4 步的 v5turbo 不支持。"))
|
||||
for _cd in (cache_dit_mode, dbcache_fn, dbcache_threshold,
|
||||
dbcache_warmup, dbcache_downsample):
|
||||
_cd.change(
|
||||
update_cache_dit,
|
||||
[cache_dit_mode, dbcache_fn, dbcache_threshold,
|
||||
dbcache_warmup, dbcache_downsample],
|
||||
)
|
||||
if_sr_Checkbox = gr.Checkbox(
|
||||
label=i18n("v3输出如果觉得闷可以试试开超分"),
|
||||
value=False,
|
||||
|
||||
@ -73,6 +73,19 @@ language = os.environ.get("language", "Auto")
|
||||
language = sys.argv[-1] if sys.argv[-1] in scan_language_list() else language
|
||||
i18n = I18nAuto(language=language)
|
||||
|
||||
# cache-dit (DBCache) for the DiT sampler: off by default. The UI writes here, tts_infer
|
||||
# forwards it to TTS_infer_pack; the GSV_DBCACHE* environment variables work too.
|
||||
CACHE_DIT = {"cache_dit": "off", "dbcache_fn": None, "dbcache_threshold": 0.02,
|
||||
"dbcache_warmup": 8, "dbcache_downsample": 4}
|
||||
|
||||
|
||||
def update_cache_dit(mode, fn, thr, warmup, ds):
|
||||
CACHE_DIT["cache_dit"] = str(mode or "off")
|
||||
CACHE_DIT["dbcache_fn"] = None if fn in (None, "") or int(fn) < 0 else int(fn)
|
||||
CACHE_DIT["dbcache_threshold"] = float(thr) if thr not in (None, "") else 0.02
|
||||
CACHE_DIT["dbcache_warmup"] = int(warmup) if warmup not in (None, "") else 8
|
||||
CACHE_DIT["dbcache_downsample"] = int(ds) if ds not in (None, "") else 4
|
||||
|
||||
|
||||
# os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1' # 确保直接启动推理UI时也能够设置。
|
||||
|
||||
@ -209,11 +222,12 @@ def inference(
|
||||
"cfg_rate": float(cfg_rate),
|
||||
"super_sampling": super_sampling,
|
||||
}
|
||||
inputs.update(CACHE_DIT) # cache-dit (DBCache), off unless the UI / env enables it
|
||||
try:
|
||||
for item in tts_pipeline.run(inputs):
|
||||
yield item, actual_seed
|
||||
except NO_PROMPT_ERROR:
|
||||
gr.Warning(i18n("V3不支持无参考文本模式,请填写参考文本!"))
|
||||
gr.Warning(i18n("v3、v4、v5 均不支持无参考文本模式,请填写参考文本!"))
|
||||
|
||||
|
||||
def custom_sort_key(s):
|
||||
@ -373,7 +387,8 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
|
||||
)
|
||||
with gr.Column():
|
||||
ref_text_free = gr.Checkbox(
|
||||
label=i18n("开启无参考文本模式。不填参考文本亦相当于开启。"),
|
||||
label=i18n("开启无参考文本模式。不填参考文本亦相当于开启。")
|
||||
+ i18n("v3、v4、v5 均不支持无参考文本模式,请填写参考文本!"),
|
||||
value=False,
|
||||
interactive=True if model_version != "v3" else False,
|
||||
show_label=True,
|
||||
@ -420,6 +435,25 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
|
||||
repetition_penalty = gr.Slider(
|
||||
minimum=0, maximum=2, step=0.05, label=i18n("重复惩罚"), value=1.35, interactive=True
|
||||
)
|
||||
with gr.Accordion(i18n("DiT 加速 cache-dit(默认关闭)"), open=False):
|
||||
with gr.Row():
|
||||
cache_dit_mode = gr.Dropdown(
|
||||
label=i18n("cache-dit"), choices=["off", "auto", "f4", "f8", "f16"],
|
||||
value="off", interactive=True, scale=2,
|
||||
)
|
||||
dbcache_fn = gr.Number(label="fn", value=-1, precision=0, scale=1)
|
||||
dbcache_threshold = gr.Number(label="thr", value=0.02, scale=1)
|
||||
dbcache_warmup = gr.Number(label="warmup", value=8, precision=0, scale=1)
|
||||
dbcache_downsample = gr.Number(label="ds", value=4, precision=0, scale=1)
|
||||
gr.Markdown(i18n("DBCache:只算前 fn 个 block,残差变化小于 thr 时跳过中间 block。"
|
||||
"fn=-1 表示跟随模式(auto = 步数>=16 时 F8)。v5turbo 4 步不支持。"))
|
||||
for _cd in (cache_dit_mode, dbcache_fn, dbcache_threshold,
|
||||
dbcache_warmup, dbcache_downsample):
|
||||
_cd.change(
|
||||
update_cache_dit,
|
||||
[cache_dit_mode, dbcache_fn, dbcache_threshold,
|
||||
dbcache_warmup, dbcache_downsample],
|
||||
)
|
||||
|
||||
with gr.Column():
|
||||
with gr.Row():
|
||||
|
||||
437
GPT_SoVITS/module/cache_dit.py
Normal file
437
GPT_SoVITS/module/cache_dit.py
Normal file
@ -0,0 +1,437 @@
|
||||
"""cache-dit / DBCache (three-segment block caching) for the DiT/CFM sampler.
|
||||
|
||||
Follows vipshop/cache-dit (`CachedBlocks_Pattern_Base.forward`), adapted to the way this repo
|
||||
samples: the block stack is split into three segments, and the middle one is skipped when the
|
||||
front residual barely moved.
|
||||
|
||||
front : trunk(x_in) -> blocks[0:Fn] -> x_F (every step)
|
||||
R = x_F - trunk(x_in) Fn residual, the decision signal
|
||||
fd = sum|R - R_prev| / (sum|R_prev| + eps) (relative -> scale free)
|
||||
hit : x_mid = x_F + MnResidual_prev, then head only (norm_out -> proj_out)
|
||||
miss : x_F -> blocks[Fn:DEPTH] -> x_M; MnResidual = x_M - x_F; R -> R_prev
|
||||
head : x_mid -> norm_out -> proj_out -> vel (every step, hit included)
|
||||
|
||||
A hit additionally requires `can_cache`: not in warmup, `fd < threshold`, and the three gates
|
||||
`max_cached` (cached steps per sampling run), `max_cont` (consecutive hits) and `max_accum`
|
||||
(accumulated residual drift).
|
||||
|
||||
CFG. This repo runs the conditional and the unconditional branch as **two separate forward
|
||||
passes** (`CFM.inference`), so each branch owns its own `R_prev`, `MnResidual`, hit/miss
|
||||
counters and gate counters -- the equivalent of cache-dit's `enable_separate_cfg` plus
|
||||
`cfg_diff_compute_separate`. Nothing is shared between the branches.
|
||||
|
||||
Everything here is inert unless a `DBCacheConfig` with `fn > 0` is handed to `DiT.forward`;
|
||||
`cache_dit="off"` keeps the sampler on the identical code path it had before this change.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import torch
|
||||
|
||||
__all__ = [
|
||||
"DBCacheConfig",
|
||||
"DBCacheState",
|
||||
"resolve_fn",
|
||||
"config_from_params",
|
||||
"MODES",
|
||||
]
|
||||
|
||||
MODES = ("off", "auto", "f4", "f8", "f16")
|
||||
|
||||
# Defaults of the cache-dit reference recipe, and the `auto` policy used by the HTTP API
|
||||
# (F8 from 16 sampling steps on, off below that).
|
||||
DEFAULT_FN = 8
|
||||
DEFAULT_THRESHOLD = 0.02
|
||||
DEFAULT_WARMUP = 8
|
||||
DEFAULT_DOWNSAMPLE = 4
|
||||
AUTO_FN = 8
|
||||
AUTO_MIN_STEPS = 16
|
||||
|
||||
_EPS = 1e-6
|
||||
|
||||
|
||||
def resolve_fn(mode, steps=32, depth=None):
|
||||
"""`off|auto|f4|f8|f16` (or a plain integer) -> number of front blocks."""
|
||||
if mode is None:
|
||||
return 0
|
||||
if isinstance(mode, bool):
|
||||
mode = int(mode)
|
||||
if isinstance(mode, (int, float)):
|
||||
fn = int(mode)
|
||||
else:
|
||||
text = str(mode).strip().lower()
|
||||
if text in ("", "off", "none", "0", "false", "no"):
|
||||
fn = 0
|
||||
elif text in ("on", "true", "yes"):
|
||||
fn = DEFAULT_FN
|
||||
elif text == "auto":
|
||||
fn = AUTO_FN if int(steps) >= AUTO_MIN_STEPS else 0
|
||||
elif text.startswith("f") and text[1:].isdigit():
|
||||
fn = int(text[1:])
|
||||
elif text.isdigit():
|
||||
fn = int(text)
|
||||
else:
|
||||
raise ValueError("unknown cache_dit mode %r (expected off|auto|f4|f8|f16)" % (mode,))
|
||||
if depth is not None:
|
||||
fn = max(0, min(fn, int(depth) - 1))
|
||||
return fn
|
||||
|
||||
|
||||
@dataclass
|
||||
class DBCacheConfig:
|
||||
"""cache-dit knobs. `fn == 0` means off (the default)."""
|
||||
|
||||
mode: str = "off"
|
||||
fn: int = 0
|
||||
threshold: float = DEFAULT_THRESHOLD
|
||||
warmup: int = DEFAULT_WARMUP
|
||||
downsample: int = DEFAULT_DOWNSAMPLE
|
||||
warmup_interval: int = 1
|
||||
max_cached: int = -1
|
||||
max_cont: int = -1
|
||||
max_accum: float = 0.0
|
||||
# fd is reduced over the whole sequence by default: the prompt/reference frames carry a
|
||||
# nearly constant residual, so they dilute the ratio and threshold 0.02 stays usable. Set
|
||||
# to True to reduce over the target frames only -- a ~3x stricter signal on this model, so
|
||||
# thresholds are not transferable between the two conventions.
|
||||
fd_ignore_prompt: bool = False
|
||||
trace: bool = False
|
||||
# expert override: allow cache-dit below 16 sampling steps (see docs in
|
||||
# `DBCacheMixin.dbcache_config`: too few steps gain nothing and the quality risk is real)
|
||||
force: bool = False
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self.fn > 0
|
||||
|
||||
def clamped(self, depth: int | None = None) -> "DBCacheConfig":
|
||||
if depth is None:
|
||||
return self
|
||||
return replace(self, fn=max(0, min(int(self.fn), int(depth) - 1)))
|
||||
|
||||
def __str__(self) -> str: # for logs
|
||||
if not self.enabled:
|
||||
return "off"
|
||||
return ("%s(fn=%d thr=%.3g warmup=%d ds=%d mc=%d mcont=%d macc=%.3g)"
|
||||
% (self.mode, self.fn, self.threshold, self.warmup, self.downsample,
|
||||
self.max_cached, self.max_cont, self.max_accum))
|
||||
|
||||
|
||||
def _get(params, key, env, default, cast):
|
||||
"""request dict first (API field names), then env, then default."""
|
||||
if params is not None:
|
||||
for k in (key,):
|
||||
if k in params and params[k] is not None and params[k] != "":
|
||||
try:
|
||||
return cast(params[k])
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
val = os.environ.get(env)
|
||||
if val is not None and val != "":
|
||||
try:
|
||||
return cast(val)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return default
|
||||
|
||||
|
||||
def config_from_params(params=None, steps=32, depth=None) -> DBCacheConfig:
|
||||
"""Build the config from a request dict / env vars.
|
||||
|
||||
Field names match the HTTP API of our C++ server:
|
||||
cache_dit (alias dbcache) = off|auto|f4|f8|f16
|
||||
dbcache_fn, dbcache_threshold, dbcache_warmup, dbcache_downsample
|
||||
dbcache_warmup_interval, dbcache_max_cached, dbcache_max_cont, dbcache_max_accum
|
||||
Env fallbacks: GSV_DBCACHE / GSV_DBCACHE_FN / _THRESHOLD / _WARMUP / _DOWNSAMPLE / ...
|
||||
"""
|
||||
p = dict(params or {})
|
||||
mode = p.get("cache_dit", p.get("dbcache", os.environ.get("GSV_DBCACHE", "off")))
|
||||
mode = "off" if mode is None else str(mode).strip().lower()
|
||||
|
||||
cfg = DBCacheConfig(mode=mode)
|
||||
cfg.threshold = _get(p, "dbcache_threshold", "GSV_DBCACHE_THRESHOLD", cfg.threshold, float)
|
||||
cfg.warmup = _get(p, "dbcache_warmup", "GSV_DBCACHE_WARMUP", cfg.warmup, int)
|
||||
cfg.downsample = max(1, _get(p, "dbcache_downsample", "GSV_DBCACHE_DOWNSAMPLE", cfg.downsample, int))
|
||||
cfg.warmup_interval = max(1, _get(p, "dbcache_warmup_interval", "GSV_DBCACHE_WARMUP_INTERVAL",
|
||||
cfg.warmup_interval, int))
|
||||
cfg.max_cached = _get(p, "dbcache_max_cached", "GSV_DBCACHE_MAX_CACHED", cfg.max_cached, int)
|
||||
cfg.max_cont = _get(p, "dbcache_max_cont", "GSV_DBCACHE_MAX_CONT", cfg.max_cont, int)
|
||||
cfg.max_accum = _get(p, "dbcache_max_accum", "GSV_DBCACHE_MAX_ACCUM", cfg.max_accum, float)
|
||||
cfg.fd_ignore_prompt = bool(_get(p, "dbcache_fd_ignore_prompt", "GSV_DBCACHE_FD_IGNORE_PROMPT",
|
||||
int(cfg.fd_ignore_prompt), int))
|
||||
cfg.trace = bool(_get(p, "dbcache_trace", "GSV_DBCACHE_TRACE", 0, int))
|
||||
cfg.force = bool(_get(p, "dbcache_force", "GSV_DBCACHE_FORCE", 0, int))
|
||||
|
||||
# an explicit fn wins over the mode
|
||||
if p.get("dbcache_fn") not in (None, ""):
|
||||
cfg.fn = int(p["dbcache_fn"])
|
||||
cfg.mode = "custom"
|
||||
elif os.environ.get("GSV_DBCACHE_FN"):
|
||||
cfg.fn = int(os.environ["GSV_DBCACHE_FN"])
|
||||
cfg.mode = "custom"
|
||||
else:
|
||||
cfg.fn = resolve_fn(mode, steps)
|
||||
if depth is not None:
|
||||
cfg.fn = max(0, min(cfg.fn, int(depth) - 1))
|
||||
return cfg
|
||||
|
||||
|
||||
class DBCacheMixin:
|
||||
"""The CFM-side plumbing (`set_dbcache` / per-run state / hit counters).
|
||||
|
||||
Mixed into `module.models.CFM` and `module.v5_inference.CFMV5`; both expose the DiT as
|
||||
`self.estimator`. Defaults are class attributes, so `cache off` needs no `__init__` change
|
||||
and the object layout of an existing checkpoint-compatible model is untouched.
|
||||
"""
|
||||
|
||||
dbcache_spec = None # DBCacheConfig or request dict of dbcache_* fields
|
||||
last_dbcache_stats = None # stats of the most recent inference() call
|
||||
_dbcache_cum = None # accumulated over all calls (reset by dbcache_stats)
|
||||
|
||||
# ------------------------------------------------------------------ config
|
||||
@property
|
||||
def cache_depth(self):
|
||||
return int(getattr(self.estimator, "depth", 0) or 0)
|
||||
|
||||
def set_dbcache(self, spec=None, **kw):
|
||||
"""Enable/disable cache-dit for the next sampling runs.
|
||||
|
||||
`spec` is either a `module.cache_dit.DBCacheConfig` or a dict using the C++ server's
|
||||
field names (`cache_dit`/`dbcache` + `dbcache_fn|threshold|warmup|downsample|...`).
|
||||
Nothing is applied retroactively: `inference()` resolves it per call.
|
||||
"""
|
||||
if isinstance(spec, DBCacheConfig):
|
||||
self.dbcache_spec = spec
|
||||
else:
|
||||
merged = dict(spec or {})
|
||||
merged.update(kw)
|
||||
cfg = merged.pop("config", None)
|
||||
self.dbcache_spec = cfg if isinstance(cfg, DBCacheConfig) else merged
|
||||
return self.dbcache_spec
|
||||
|
||||
def dbcache_config(self, steps):
|
||||
"""Resolve the spec against the step count of the run that is about to start.
|
||||
|
||||
With no explicit spec the env switch (`GSV_DBCACHE=off|auto|f4|f8|f16`) is honoured,
|
||||
so every entry point (WebUI, api.py, api_v2.py, CLI, Colab) can enable cache-dit
|
||||
without extra plumbing. Below 16 sampling steps (the 4-step v5turbo) it stays off:
|
||||
the measured gain is negative and the quality is unstable there.
|
||||
"""
|
||||
depth = self.cache_depth or None
|
||||
spec = self.dbcache_spec
|
||||
if isinstance(spec, DBCacheConfig):
|
||||
fn = spec.fn if spec.mode in ("custom", None) else resolve_fn(spec.mode, steps, depth)
|
||||
cfg = DBCacheConfig(**{**spec.__dict__,
|
||||
"fn": fn if depth is None else min(fn, depth - 1)})
|
||||
else:
|
||||
cfg = config_from_params(spec, steps, depth)
|
||||
if cfg.enabled and int(steps) < AUTO_MIN_STEPS and not cfg.force:
|
||||
cfg.fn = 0
|
||||
return cfg
|
||||
|
||||
# ------------------------------------------------------------------ per run
|
||||
def _dbcache_begin(self, steps, seq_len, prompt_len, x_lens, device):
|
||||
"""Fresh state per sampling run (new request / new chunk)."""
|
||||
cfg = self.dbcache_config(steps)
|
||||
if not cfg.enabled:
|
||||
self.last_dbcache_stats = None
|
||||
return None
|
||||
return DBCacheState(cfg, self.cache_depth).begin(seq_len, prompt_len, x_lens, device)
|
||||
|
||||
def _dbcache_end(self, state):
|
||||
if state is None:
|
||||
return
|
||||
st = state.stats()
|
||||
self.last_dbcache_stats = st
|
||||
cum = self._dbcache_cum
|
||||
if cum is None:
|
||||
cum = self._dbcache_cum = {"calls": 0, "steps": 0, "hits_pos": 0, "hits_neg": 0,
|
||||
"misses_pos": 0, "misses_neg": 0}
|
||||
cum["calls"] += 1
|
||||
for k in ("steps", "hits_pos", "hits_neg", "misses_pos", "misses_neg"):
|
||||
cum[k] = cum.get(k, 0) + st[k]
|
||||
cum["hits"] = cum["hits_pos"] + cum["hits_neg"]
|
||||
cum["misses"] = cum["misses_pos"] + cum["misses_neg"]
|
||||
cum["total"] = cum["hits"] + cum["misses"]
|
||||
cum["hit_rate"] = cum["hits"] / cum["total"] if cum["total"] else 0.0
|
||||
cum["fn"] = st.get("fn")
|
||||
cum["threshold"] = st.get("threshold")
|
||||
cum["last"] = st
|
||||
|
||||
def dbcache_stats(self, reset=True):
|
||||
"""Hit/miss counters accumulated since the last reset (None when cache-dit is off)."""
|
||||
st = self._dbcache_cum
|
||||
if reset:
|
||||
self._dbcache_cum = None
|
||||
return st
|
||||
|
||||
|
||||
class _Branch:
|
||||
"""Per-CFG-branch bookkeeping."""
|
||||
|
||||
__slots__ = ("valid", "r_prev", "r_cur", "m_res", "hits", "misses",
|
||||
"cached_steps", "cont", "acc", "last_fd")
|
||||
|
||||
def __init__(self):
|
||||
self.valid = False
|
||||
self.r_prev = None
|
||||
self.r_cur = None
|
||||
self.m_res = None
|
||||
self.hits = 0
|
||||
self.misses = 0
|
||||
self.cached_steps = 0
|
||||
self.cont = 0
|
||||
self.acc = 0.0
|
||||
self.last_fd = 0.0
|
||||
|
||||
def reset(self):
|
||||
self.__init__()
|
||||
|
||||
|
||||
class DBCacheState:
|
||||
"""State of one sampling run (`CFM.inference` = one run = one new request / chunk)."""
|
||||
|
||||
def __init__(self, config: DBCacheConfig, depth: int, trace: bool | None = None):
|
||||
self.cfg = config
|
||||
self.depth = int(depth)
|
||||
self.trace = config.trace if trace is None else trace
|
||||
self.step = 0
|
||||
self.n_steps = 0
|
||||
self.branches = [_Branch(), _Branch()]
|
||||
self.ds = 1
|
||||
self.note = "" # free-form tag for logs
|
||||
|
||||
# ------------------------------------------------------------------ lifecycle
|
||||
def begin(self, seq_len: int, prompt_len: int, x_lens=None, device=None):
|
||||
"""Reset for a new sampling run. Called once per `CFM.inference`."""
|
||||
for b in self.branches:
|
||||
b.reset()
|
||||
self.step = 0
|
||||
self.n_steps = 0
|
||||
ds = int(self.cfg.downsample) if self.cfg.downsample else 1
|
||||
self.ds = ds if ds >= 1 and seq_len % ds == 0 else 1
|
||||
self._build_frame_mask(seq_len, prompt_len, x_lens, device)
|
||||
return self
|
||||
|
||||
def _build_frame_mask(self, seq_len, prompt_len, x_lens, device):
|
||||
"""Frames used for fd: skip the prompt/reference rows and the padding."""
|
||||
self.mask = None
|
||||
if not self.cfg.fd_ignore_prompt:
|
||||
return
|
||||
idx = torch.arange(seq_len, device=device)
|
||||
mask = idx.unsqueeze(0) >= int(max(0, prompt_len))
|
||||
if x_lens is not None:
|
||||
lens = torch.as_tensor(x_lens, device=device).reshape(-1, 1)
|
||||
if lens.shape[0] == 1 and seq_len >= int(lens.max()):
|
||||
mask = mask & (idx.unsqueeze(0) < lens)
|
||||
mask = mask.to(torch.float32)
|
||||
if self.ds > 1:
|
||||
mask = mask[:, :: self.ds]
|
||||
mask = mask.unsqueeze(-1) # (b, n/ds, 1)
|
||||
if float(mask.sum()) <= 0.0: # degenerate (all-prompt) -> fall back to everything
|
||||
mask = None
|
||||
self.mask = mask
|
||||
|
||||
def step_begin(self, step: int):
|
||||
"""`db_set_step(j)`: current diffusion step, drives warmup and the trace."""
|
||||
self.step = int(step)
|
||||
self.n_steps += 1
|
||||
|
||||
def in_warmup(self) -> bool:
|
||||
if self.step >= self.cfg.warmup:
|
||||
return False
|
||||
return (self.step % max(1, self.cfg.warmup_interval)) == 0
|
||||
|
||||
# ------------------------------------------------------------------ per-step ops
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self.cfg.enabled
|
||||
|
||||
@property
|
||||
def fn(self) -> int:
|
||||
return self.cfg.fn
|
||||
|
||||
def branch(self, index: int) -> _Branch:
|
||||
return self.branches[0 if index == 0 else 1]
|
||||
|
||||
def front_update(self, index: int, x_f, trunk) -> float:
|
||||
"""Store x_F / R, update R_prev-based similarity, return `fd` (one host sync)."""
|
||||
br = self.branch(index)
|
||||
r = x_f.float() - trunk.float() # Fn residual; fp32 keeps fd stable
|
||||
if self.ds > 1:
|
||||
r = r[:, :: self.ds, :]
|
||||
r_prev = br.r_prev
|
||||
if r_prev is None or r_prev.shape != r.shape:
|
||||
r_prev = torch.zeros_like(r)
|
||||
num = (r - r_prev).abs()
|
||||
den = r_prev.abs()
|
||||
if self.mask is not None and self.mask.shape[0] == num.shape[0] \
|
||||
and self.mask.shape[1] == num.shape[1]:
|
||||
num = num * self.mask
|
||||
den = den * self.mask
|
||||
num = num.sum(dim=(1, 2))
|
||||
den = den.sum(dim=(1, 2)) + _EPS
|
||||
fd = float((num / den).max().item()) # the only device->host scalar per step
|
||||
br.r_cur = r
|
||||
br.last_fd = fd
|
||||
return fd
|
||||
|
||||
def can_cache(self, index: int, fd: float) -> bool:
|
||||
br = self.branch(index)
|
||||
cfg = self.cfg
|
||||
hit = False
|
||||
if br.valid and cfg.threshold > 0.0 and not self.in_warmup():
|
||||
ok = fd < cfg.threshold
|
||||
if ok and cfg.max_cached >= 0 and br.cached_steps >= cfg.max_cached:
|
||||
ok = False
|
||||
if ok and cfg.max_cont >= 0 and br.cont >= cfg.max_cont:
|
||||
ok = False
|
||||
br.cont = 0
|
||||
if ok and cfg.max_accum > 0.0 and br.acc >= cfg.max_accum:
|
||||
ok = False
|
||||
hit = ok
|
||||
if fd > 0.0:
|
||||
br.acc += fd
|
||||
return hit
|
||||
|
||||
def mid_residual(self, index: int):
|
||||
return self.branch(index).m_res
|
||||
|
||||
def on_hit(self, index: int):
|
||||
br = self.branch(index)
|
||||
br.hits += 1
|
||||
br.cached_steps += 1
|
||||
br.cont += 1
|
||||
|
||||
def on_miss(self, index: int, x_m, x_f):
|
||||
br = self.branch(index)
|
||||
br.misses += 1
|
||||
br.valid = True
|
||||
br.cont = 0
|
||||
br.m_res = (x_m - x_f)
|
||||
if br.r_cur is not None:
|
||||
if br.r_prev is None or br.r_prev.shape != br.r_cur.shape:
|
||||
br.r_prev = br.r_cur.clone()
|
||||
else:
|
||||
br.r_prev.copy_(br.r_cur)
|
||||
|
||||
# ------------------------------------------------------------------ reporting
|
||||
def stats(self) -> dict:
|
||||
out = {"steps": self.n_steps, "last_step": self.step, "fn": self.cfg.fn,
|
||||
"threshold": self.cfg.threshold, "warmup": self.cfg.warmup,
|
||||
"downsample": self.ds, "mode": self.cfg.mode}
|
||||
for i, name in enumerate(("pos", "neg")):
|
||||
br = self.branches[i]
|
||||
out["hits_" + name] = br.hits
|
||||
out["misses_" + name] = br.misses
|
||||
out["fd_" + name] = br.last_fd
|
||||
out["acc_" + name] = br.acc
|
||||
out["hits"] = out["hits_pos"] + out["hits_neg"]
|
||||
out["misses"] = out["misses_pos"] + out["misses_neg"]
|
||||
total = out["hits"] + out["misses"]
|
||||
out["total"] = total
|
||||
out["hit_rate"] = (out["hits"] / total) if total else 0.0
|
||||
return out
|
||||
@ -10,6 +10,7 @@ from torch.nn import functional as F
|
||||
from module import commons
|
||||
from module import modules
|
||||
from module import attentions
|
||||
from module.cache_dit import DBCacheMixin
|
||||
from f5_tts.model import DiT
|
||||
from torch.nn import Conv1d, ConvTranspose1d, Conv2d
|
||||
from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
|
||||
@ -1099,7 +1100,7 @@ class SynthesizerTrn(nn.Module):
|
||||
return codes.transpose(0, 1)
|
||||
|
||||
|
||||
class CFM(torch.nn.Module):
|
||||
class CFM(DBCacheMixin, torch.nn.Module):
|
||||
def __init__(self, in_channels, dit):
|
||||
super().__init__()
|
||||
self.sigma_min = 1e-6
|
||||
@ -1138,7 +1139,11 @@ class CFM(torch.nn.Module):
|
||||
if self.use_conditioner_cache and self.use_static_cache else None)
|
||||
d_tensor = (torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * d
|
||||
if self.use_step_embedding else None)
|
||||
# cache-dit: one fresh state per sampling run; None when the switch is off
|
||||
dbcache = self._dbcache_begin(n_timesteps, T, prompt_len, x_lens, x.device)
|
||||
for j in range(n_timesteps):
|
||||
if dbcache is not None:
|
||||
dbcache.step_begin(j)
|
||||
t_tensor = torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * (t if self.use_step_embedding else j * d)
|
||||
step_condition = {"dt_base_bootstrap": d_tensor, "dt_cache": dt_cache} if self.use_step_embedding else {}
|
||||
v_pred, text_emb, dt = self.estimator(
|
||||
@ -1153,6 +1158,8 @@ class CFM(torch.nn.Module):
|
||||
infer=True,
|
||||
text_cache=text_cache,
|
||||
static_cache=static_cache,
|
||||
dbcache=dbcache,
|
||||
dbcache_branch=0,
|
||||
**step_condition,
|
||||
)
|
||||
v_pred = v_pred.transpose(2, 1)
|
||||
@ -1174,6 +1181,8 @@ class CFM(torch.nn.Module):
|
||||
infer=True,
|
||||
text_cache=text_cfg_cache if self.cfg_drop_text else text_cache,
|
||||
static_cache=static_cache,
|
||||
dbcache=dbcache,
|
||||
dbcache_branch=1,
|
||||
**step_condition,
|
||||
)
|
||||
neg = neg.transpose(2, 1)
|
||||
@ -1183,6 +1192,7 @@ class CFM(torch.nn.Module):
|
||||
x = x + d * v_pred
|
||||
t = t + d
|
||||
x[:, :, :prompt_len] = 0
|
||||
self._dbcache_end(dbcache)
|
||||
return x
|
||||
|
||||
def forward(self, x1, x_lens, prompt_lens, mu, use_grad_ckpt):
|
||||
|
||||
@ -1,133 +0,0 @@
|
||||
import math
|
||||
import os
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
|
||||
V5_TEMPERATURE = 0.875
|
||||
V5_REFERENCE_ANCHOR = 1.0
|
||||
V5_ROLLING_TAIL_FRAMES = 32
|
||||
V5_MAX_TARGET_CHUNK_FRAMES = 640
|
||||
V5_REFERENCE_FRAMES = 500
|
||||
V5_TOTAL_CHUNK_FRAMES = 1000
|
||||
V5_DEFAULT_CFG = 1.30
|
||||
V5_REQUEST_DEFAULT_CFG = 0.0
|
||||
V5_VERSIONS = {"v5", "v5dev", "v5turbo"}
|
||||
|
||||
|
||||
def sampling_defaults(version, legacy_steps=32):
|
||||
if version == "v5turbo":
|
||||
return 4, 0.0
|
||||
if version in V5_VERSIONS:
|
||||
return 32, 1.30
|
||||
return legacy_steps, 0.0
|
||||
|
||||
|
||||
def resolve_sampling(version, sample_steps=None, cfg_rate=None, legacy_steps=32):
|
||||
steps, cfg = sampling_defaults(version, legacy_steps)
|
||||
return (steps if sample_steps is None else int(sample_steps),
|
||||
cfg if cfg_rate is None else float(cfg_rate))
|
||||
|
||||
|
||||
def validate_v5_sampling(sample_steps, cfg_rate):
|
||||
steps = int(sample_steps)
|
||||
cfg = float(cfg_rate)
|
||||
if steps not in (4, 8, 16, 32):
|
||||
raise ValueError("Euler steps must be one of 4, 8, 16, 32")
|
||||
if not math.isfinite(cfg) or cfg < 0 or cfg > 2:
|
||||
raise ValueError("CFG must be a finite value between 0 and 2")
|
||||
return steps, cfg
|
||||
|
||||
|
||||
class CFMV5(nn.Module):
|
||||
def __init__(self, in_channels, dit):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
self.estimator = dit
|
||||
self.use_conditioner_cache = True
|
||||
self.use_static_cache = os.environ.get(
|
||||
"GSVV5_CFM_STATIC_CACHE", os.environ.get("GSVV4_CFM_STATIC_CACHE", "1")
|
||||
) != "0"
|
||||
|
||||
@torch.inference_mode()
|
||||
def inference(self, mu, x_lens, prompt, n_timesteps, temperature=V5_TEMPERATURE,
|
||||
inference_cfg_rate=V5_DEFAULT_CFG):
|
||||
steps, cfg = validate_v5_sampling(n_timesteps, inference_cfg_rate)
|
||||
batch, frames = mu.shape[:2]
|
||||
prompt_len = prompt.shape[-1]
|
||||
# Match v4: noise, time and model inputs follow mu/model dtype.
|
||||
state_dtype = mu.dtype
|
||||
model_dtype = mu.dtype
|
||||
x = torch.randn(batch, self.in_channels, frames, device=mu.device, dtype=state_dtype) * V5_TEMPERATURE
|
||||
x[..., :prompt_len] = 0
|
||||
prompt_x = torch.zeros_like(x, dtype=model_dtype)
|
||||
prompt_x[..., :prompt_len] = prompt.to(model_dtype)
|
||||
condition = mu.transpose(2, 1)
|
||||
cache_enabled = bool(self.use_conditioner_cache and self.use_static_cache)
|
||||
cache = (self.estimator.prepare_static_cache(prompt_x, x_lens, condition)
|
||||
if cache_enabled else None)
|
||||
text_cache = None
|
||||
step = 1.0 / steps
|
||||
for index in range(steps):
|
||||
time = torch.full((batch,), index * step, device=mu.device, dtype=state_dtype)
|
||||
velocity, text_embedding, _ = self.estimator(
|
||||
x, prompt_x, x_lens, time, None, condition,
|
||||
drop_audio_cond=False, drop_text=False, static_cache=cache,
|
||||
infer=True, text_cache=text_cache,
|
||||
)
|
||||
if self.use_conditioner_cache and cache is None:
|
||||
text_cache = text_embedding
|
||||
velocity = velocity.transpose(2, 1)
|
||||
if cfg > 1e-5:
|
||||
negative, _, _ = self.estimator(
|
||||
x, prompt_x, x_lens, time, None, condition,
|
||||
drop_audio_cond=True, drop_text=False, static_cache=cache,
|
||||
infer=True, text_cache=text_cache,
|
||||
)
|
||||
negative = negative.transpose(2, 1)
|
||||
velocity = velocity + cfg * (velocity - negative)
|
||||
x = x + step * velocity
|
||||
x[..., :prompt_len] = 0
|
||||
return x
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def synthesize_v5_mel(model, reference_features, target_features, reference_mel,
|
||||
sample_steps=None, cfg_rate=None):
|
||||
sample_steps, cfg_rate = resolve_sampling(model.version, sample_steps, cfg_rate)
|
||||
steps, cfg = validate_v5_sampling(sample_steps, cfg_rate)
|
||||
reference_frames = min(reference_mel.shape[-1], reference_features.shape[-1])
|
||||
if reference_frames < 1:
|
||||
raise ValueError("V5 requires nonempty reference mel and semantic features")
|
||||
reference_mel = reference_mel[..., :reference_frames][..., -V5_REFERENCE_FRAMES:]
|
||||
reference_features = reference_features[..., :reference_frames][..., -V5_REFERENCE_FRAMES:]
|
||||
reference_frames = reference_mel.shape[-1]
|
||||
chunk_frames = min(V5_TOTAL_CHUNK_FRAMES - reference_frames, V5_MAX_TARGET_CHUNK_FRAMES)
|
||||
original_mel = reference_mel
|
||||
original_features = reference_features
|
||||
rolling_mel = reference_mel
|
||||
rolling_features = reference_features
|
||||
results = []
|
||||
for index, start in enumerate(range(0, target_features.shape[-1], chunk_frames)):
|
||||
target = target_features[..., start:start + chunk_frames]
|
||||
if index == 0:
|
||||
prompt = original_mel
|
||||
features = original_features
|
||||
else:
|
||||
tail = min(V5_ROLLING_TAIL_FRAMES, reference_frames,
|
||||
rolling_mel.shape[-1], rolling_features.shape[-1])
|
||||
prefix = reference_frames - tail
|
||||
prompt = torch.cat((original_mel[..., :prefix], rolling_mel[..., -tail:]), dim=-1)
|
||||
features = torch.cat((original_features[..., :prefix], rolling_features[..., -tail:]), dim=-1)
|
||||
mu = torch.cat((features, target), dim=-1).transpose(2, 1)
|
||||
lengths = torch.full((mu.shape[0],), mu.shape[1], dtype=torch.long, device=mu.device)
|
||||
generated = model.cfm.inference(
|
||||
mu, lengths, prompt, steps, inference_cfg_rate=cfg,
|
||||
)[..., prompt.shape[-1]:]
|
||||
results.append(generated)
|
||||
rolling_mel = generated[..., -reference_frames:].to(reference_mel.dtype)
|
||||
rolling_features = target[..., -reference_frames:]
|
||||
if not results:
|
||||
raise ValueError("V5 requires nonempty target semantic features")
|
||||
return torch.cat(results, dim=-1)
|
||||
@ -86,7 +86,10 @@ def _expand_time(m):
|
||||
"""
|
||||
hours, minutes = map(int, m.group(1, 2))
|
||||
period = "a.m." if hours < 12 else "p.m."
|
||||
if hours > 12:
|
||||
# 00:00 is midnight. On a 12-hour clock that hour is twelve, not zero.
|
||||
if hours == 0:
|
||||
hours = 12
|
||||
elif hours > 12:
|
||||
hours -= 12
|
||||
|
||||
hour_word = _inflect.number_to_words(hours)
|
||||
|
||||
22
GPT_SoVITS/text/en_normalization/test_expend.py
Normal file
22
GPT_SoVITS/text/en_normalization/test_expend.py
Normal file
@ -0,0 +1,22 @@
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from expend import normalize
|
||||
|
||||
|
||||
class TimeTests(unittest.TestCase):
|
||||
def test_midnight_is_twelve_am(self):
|
||||
self.assertEqual(normalize("0:00"), "twelve o'clock a.m.")
|
||||
self.assertEqual(normalize("00:30"), "twelve thirty a.m.")
|
||||
|
||||
def test_noon_and_afternoon_stay(self):
|
||||
self.assertEqual(normalize("12:00"), "twelve o'clock p.m.")
|
||||
self.assertEqual(normalize("4:00"), "four o'clock a.m.")
|
||||
self.assertEqual(normalize("13:00"), "one o'clock p.m.")
|
||||
self.assertEqual(normalize("13:30"), "one thirty p.m.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
58
api.py
58
api.py
@ -829,6 +829,28 @@ splits = {
|
||||
}
|
||||
|
||||
|
||||
def apply_cache_dit(vq_model, params: dict, sample_steps: int):
|
||||
"""cache-dit (DBCache) switch for the DiT sampler; off unless configured.
|
||||
|
||||
Field names match api_v2.py:
|
||||
``cache_dit`` = off|auto|f4|f8|f16, ``dbcache_fn``, ``dbcache_threshold``,
|
||||
``dbcache_warmup``, ``dbcache_downsample``. The ``GSV_DBCACHE*`` env vars work too,
|
||||
also on the paths that never go through these request fields.
|
||||
"""
|
||||
cfm = getattr(vq_model, "cfm", None)
|
||||
if cfm is None or not hasattr(cfm, "set_dbcache"):
|
||||
return None
|
||||
from module.cache_dit import config_from_params
|
||||
|
||||
cfg = config_from_params(params, int(sample_steps), getattr(cfm, "cache_depth", 0) or None)
|
||||
if cfg.enabled and str(getattr(vq_model, "version", "")) == "v5turbo" and not cfg.force:
|
||||
cfg.fn = 0 # 4-step turbo: no gain and unstable quality, keep it off
|
||||
cfm.set_dbcache(cfg)
|
||||
if cfg.enabled:
|
||||
print("cache-dit:", cfg)
|
||||
return cfg
|
||||
|
||||
|
||||
def get_tts_wav(
|
||||
ref_wav_path,
|
||||
prompt_text,
|
||||
@ -844,6 +866,11 @@ def get_tts_wav(
|
||||
if_sr=False,
|
||||
spk="default",
|
||||
cfg_rate=None,
|
||||
cache_dit=None,
|
||||
dbcache_fn=None,
|
||||
dbcache_threshold=None,
|
||||
dbcache_warmup=None,
|
||||
dbcache_downsample=None,
|
||||
):
|
||||
infer_sovits = speaker_list[spk].sovits
|
||||
vq_model = infer_sovits.vq_model
|
||||
@ -867,6 +894,12 @@ def get_tts_wav(
|
||||
if sample_steps not in [4, 8, 16, 32]:
|
||||
sample_steps = 8
|
||||
|
||||
# cache-dit (DBCache) for the DiT sampler: off unless asked for
|
||||
apply_cache_dit(vq_model, dict(
|
||||
cache_dit=cache_dit, dbcache_fn=dbcache_fn, dbcache_threshold=dbcache_threshold,
|
||||
dbcache_warmup=dbcache_warmup, dbcache_downsample=dbcache_downsample,
|
||||
), sample_steps)
|
||||
|
||||
if if_sr and version != "v3":
|
||||
if_sr = False
|
||||
|
||||
@ -1126,6 +1159,11 @@ def handle(
|
||||
sample_steps,
|
||||
if_sr,
|
||||
cfg_rate=None,
|
||||
cache_dit=None,
|
||||
dbcache_fn=None,
|
||||
dbcache_threshold=None,
|
||||
dbcache_warmup=None,
|
||||
dbcache_downsample=None,
|
||||
):
|
||||
if (
|
||||
refer_wav_path == ""
|
||||
@ -1163,6 +1201,11 @@ def handle(
|
||||
sample_steps,
|
||||
if_sr,
|
||||
cfg_rate=cfg_rate,
|
||||
cache_dit=cache_dit,
|
||||
dbcache_fn=dbcache_fn,
|
||||
dbcache_threshold=dbcache_threshold,
|
||||
dbcache_warmup=dbcache_warmup,
|
||||
dbcache_downsample=dbcache_downsample,
|
||||
),
|
||||
media_type="audio/" + media_type,
|
||||
)
|
||||
@ -1372,6 +1415,11 @@ async def tts_endpoint(request: Request):
|
||||
json_post_raw.get("sample_steps"),
|
||||
json_post_raw.get("if_sr", False),
|
||||
cfg_rate=json_post_raw.get("cfg_rate"),
|
||||
cache_dit=json_post_raw.get("cache_dit"),
|
||||
dbcache_fn=json_post_raw.get("dbcache_fn"),
|
||||
dbcache_threshold=json_post_raw.get("dbcache_threshold"),
|
||||
dbcache_warmup=json_post_raw.get("dbcache_warmup"),
|
||||
dbcache_downsample=json_post_raw.get("dbcache_downsample"),
|
||||
)
|
||||
|
||||
|
||||
@ -1391,6 +1439,11 @@ async def tts_endpoint(
|
||||
sample_steps: int = None,
|
||||
if_sr: bool = False,
|
||||
cfg_rate: float = None,
|
||||
cache_dit: str = None,
|
||||
dbcache_fn: int = None,
|
||||
dbcache_threshold: float = None,
|
||||
dbcache_warmup: int = None,
|
||||
dbcache_downsample: int = None,
|
||||
):
|
||||
return handle(
|
||||
refer_wav_path,
|
||||
@ -1407,6 +1460,11 @@ async def tts_endpoint(
|
||||
sample_steps,
|
||||
if_sr,
|
||||
cfg_rate=cfg_rate,
|
||||
cache_dit=cache_dit,
|
||||
dbcache_fn=dbcache_fn,
|
||||
dbcache_threshold=dbcache_threshold,
|
||||
dbcache_warmup=dbcache_warmup,
|
||||
dbcache_downsample=dbcache_downsample,
|
||||
)
|
||||
|
||||
|
||||
|
||||
22
api_v2.py
22
api_v2.py
@ -44,6 +44,11 @@ POST:
|
||||
"streaming_mode": False, # bool or int. return audio chunk by chunk.T he available options are: 0,1,2,3 or True/False (0/False: Disabled | 1/True: Best Quality, Slowest response speed (old version streaming_mode) | 2: Medium Quality, Slow response speed | 3: Lower Quality, Faster response speed )
|
||||
"overlap_length": 2, # int. overlap length of semantic tokens for streaming mode.
|
||||
"min_chunk_length": 16, # int. The minimum chunk length of semantic tokens for streaming mode. (affects audio chunk size)
|
||||
"cache_dit": "off", # str.(optional) off|auto|f4|f8|f16. cache-dit/DBCache for the V3/V4/V5 DiT: fewer DiT forwards, near-identical audio. Off by default; 4-step runs are refused.
|
||||
"dbcache_fn": 8, # int.(optional) front blocks (default 8; auto = F8 when sample_steps >= 16).
|
||||
"dbcache_threshold": 0.02, # float.(optional) relative residual-diff threshold (default 0.02).
|
||||
"dbcache_warmup": 8, # int.(optional) forced full-compute steps at the start (default 8).
|
||||
"dbcache_downsample": 4, # int.(optional) time downsample of the cached Fn residual (default 4).
|
||||
"gpt_streaming": false, # bool. opt in to semantic-token streaming.
|
||||
"streaming_chunk_seconds": 2.0,
|
||||
"streaming_window_mode": "fixed",
|
||||
@ -201,6 +206,13 @@ class TTS_Request(BaseModel):
|
||||
repetition_penalty: float = 1.35
|
||||
sample_steps: Optional[int] = None
|
||||
cfg_rate: Optional[float] = None
|
||||
# cache-dit (DBCache) for the V3/V4/V5 DiT sampler; off by default. Field names match
|
||||
# the ones the TTS pipeline and the CLI use.
|
||||
cache_dit: str = "off" # off | auto | f4 | f8 | f16
|
||||
dbcache_fn: Optional[int] = None # front blocks (overrides cache_dit)
|
||||
dbcache_threshold: Optional[float] = None
|
||||
dbcache_warmup: Optional[int] = None
|
||||
dbcache_downsample: Optional[int] = None
|
||||
super_sampling: bool = False
|
||||
overlap_length: int = 2
|
||||
min_chunk_length: int = 16
|
||||
@ -613,6 +625,11 @@ async def tts_get_endpoint(
|
||||
repetition_penalty: float = 1.35,
|
||||
sample_steps: Optional[int] = None,
|
||||
cfg_rate: Optional[float] = None,
|
||||
cache_dit: str = "off",
|
||||
dbcache_fn: Optional[int] = None,
|
||||
dbcache_threshold: Optional[float] = None,
|
||||
dbcache_warmup: Optional[int] = None,
|
||||
dbcache_downsample: Optional[int] = None,
|
||||
super_sampling: bool = False,
|
||||
streaming_mode: Union[bool, int] = False,
|
||||
overlap_length: int = 2,
|
||||
@ -649,6 +666,11 @@ async def tts_get_endpoint(
|
||||
"repetition_penalty": float(repetition_penalty),
|
||||
"sample_steps": None if sample_steps is None else int(sample_steps),
|
||||
"cfg_rate": None if cfg_rate is None else float(cfg_rate),
|
||||
"cache_dit": cache_dit,
|
||||
"dbcache_fn": dbcache_fn,
|
||||
"dbcache_threshold": dbcache_threshold,
|
||||
"dbcache_warmup": dbcache_warmup,
|
||||
"dbcache_downsample": dbcache_downsample,
|
||||
"super_sampling": super_sampling,
|
||||
"overlap_length": int(overlap_length),
|
||||
"min_chunk_length": int(min_chunk_length),
|
||||
|
||||
@ -10,9 +10,20 @@ def get_models():
|
||||
|
||||
|
||||
asr_dict = {
|
||||
"Fun-ASR-Nano (31语种+方言, 推荐)": {"lang": ["zh", "en", "ja", "ko", "yue", "auto"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
|
||||
"SenseVoice (极速, 5语种)": {"lang": ["zh", "en", "ja", "ko", "yue", "auto"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
|
||||
"达摩 ASR (中文经典)": {"lang": ["zh", "yue"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
|
||||
"达摩 ASR (中文经典)": {
|
||||
"lang": ["zh"],
|
||||
"size": ["large"],
|
||||
"path": "funasr_asr.py",
|
||||
"backend": "damo",
|
||||
"precision": ["float32"],
|
||||
},
|
||||
"Fun-ASR-Nano(速度更慢,更准)": {
|
||||
"lang": ["zh", "en", "ja", "ko", "yue", "auto"],
|
||||
"size": ["large"],
|
||||
"path": "funasr_asr.py",
|
||||
"backend": "fun-asr-nano",
|
||||
"precision": ["float32"],
|
||||
},
|
||||
"Faster Whisper (多语种)": {
|
||||
"lang": ["auto", "en", "ja", "ko"],
|
||||
"size": get_models(),
|
||||
|
||||
@ -40,6 +40,26 @@ language_code_list = [
|
||||
|
||||
|
||||
def download_model(model_size: str):
|
||||
model_path = ""
|
||||
if "distil" in model_size:
|
||||
if "3.5" in model_size:
|
||||
repo_id = "distil-whisper/distil-large-v3.5-ct2"
|
||||
model_path = "tools/asr/models/faster-distil-whisper-large-v3.5"
|
||||
else:
|
||||
repo_id = "Systran/faster-{}-whisper-{}".format(*model_size.split("-", maxsplit=1))
|
||||
elif model_size == "large-v3-turbo":
|
||||
repo_id = "mobiuslabsgmbh/faster-whisper-large-v3-turbo"
|
||||
model_path = "tools/asr/models/faster-whisper-large-v3-turbo"
|
||||
else:
|
||||
repo_id = f"Systran/faster-whisper-{model_size}"
|
||||
model_path = model_path or f"tools/asr/models/{repo_id.replace('Systran/', '').replace('distil-whisper/', '', 1)}"
|
||||
|
||||
if os.path.isfile(os.path.join(model_path, "config.json")) and os.path.isfile(
|
||||
os.path.join(model_path, "model.bin")
|
||||
):
|
||||
print(f"Using local Faster Whisper model: {model_path}")
|
||||
return model_path
|
||||
|
||||
url = "https://huggingface.co/api/models/gpt2"
|
||||
try:
|
||||
requests.get(url, timeout=3)
|
||||
@ -47,26 +67,6 @@ def download_model(model_size: str):
|
||||
except Exception:
|
||||
source = "ModelScope"
|
||||
|
||||
model_path = ""
|
||||
if source == "HF":
|
||||
if "distil" in model_size:
|
||||
if "3.5" in model_size:
|
||||
repo_id = "distil-whisper/distil-large-v3.5-ct2"
|
||||
model_path = "tools/asr/models/faster-distil-whisper-large-v3.5"
|
||||
else:
|
||||
repo_id = "Systran/faster-{}-whisper-{}".format(*model_size.split("-", maxsplit=1))
|
||||
elif model_size == "large-v3-turbo":
|
||||
repo_id = "mobiuslabsgmbh/faster-whisper-large-v3-turbo"
|
||||
model_path = "tools/asr/models/faster-whisper-large-v3-turbo"
|
||||
else:
|
||||
repo_id = f"Systran/faster-whisper-{model_size}"
|
||||
model_path = (
|
||||
model_path or f"tools/asr/models/{repo_id.replace('Systran/', '').replace('distil-whisper/', '', 1)}"
|
||||
)
|
||||
else:
|
||||
repo_id = "XXXXRT/faster-whisper"
|
||||
model_path = "tools/asr/models"
|
||||
|
||||
files: list[str] = [
|
||||
"config.json",
|
||||
"model.bin",
|
||||
@ -76,12 +76,8 @@ def download_model(model_size: str):
|
||||
if "large-v3" in model_size or "distil" in model_size:
|
||||
files.append("preprocessor_config.json")
|
||||
files.append("vocabulary.json")
|
||||
|
||||
files.remove("vocabulary.txt")
|
||||
|
||||
if source == "ModelScope":
|
||||
files = [f"faster-whisper-{model_size}/{file}".replace("whisper-distil", "distil-whisper") for file in files]
|
||||
|
||||
if source == "HF":
|
||||
print(f"Downloading model from HuggingFace: {repo_id} to {model_path}")
|
||||
snapshot_download_hf(
|
||||
@ -91,13 +87,14 @@ def download_model(model_size: str):
|
||||
allow_patterns=files,
|
||||
)
|
||||
else:
|
||||
print(f"Downloading model from ModelScope: {repo_id} to {model_path}")
|
||||
repo_id = "XXXXRT/faster-whisper"
|
||||
files = [f"faster-whisper-{model_size}/{file}".replace("whisper-distil", "distil-whisper") for file in files]
|
||||
print(f"Downloading model from ModelScope: {repo_id} to tools/asr/models")
|
||||
snapshot_download_ms(
|
||||
repo_id,
|
||||
local_dir=model_path,
|
||||
local_dir="tools/asr/models",
|
||||
allow_patterns=files,
|
||||
)
|
||||
return model_path + f"/faster-whisper-{model_size}".replace("whisper-distil", "distil-whisper")
|
||||
return model_path
|
||||
|
||||
|
||||
|
||||
@ -2,18 +2,45 @@
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import setuptools._distutils as _distutils
|
||||
import setuptools._distutils.util as _distutils_util
|
||||
import setuptools._distutils.version as _distutils_version
|
||||
|
||||
sys.modules.setdefault("distutils", _distutils)
|
||||
sys.modules.setdefault("distutils.util", _distutils_util)
|
||||
sys.modules.setdefault("distutils.version", _distutils_version)
|
||||
|
||||
from funasr import AutoModel
|
||||
from modelscope import snapshot_download
|
||||
from tqdm import tqdm
|
||||
|
||||
funasr_models = {} # 存储模型避免重复加载
|
||||
ASR_MODEL_ROOT = "tools/asr/models"
|
||||
FUN_ASR_NANO_MODEL_ID = "FunAudioLLM/Fun-ASR-Nano-2512"
|
||||
FUN_ASR_NANO_MODEL_SOURCES = (
|
||||
{"model": FUN_ASR_NANO_MODEL_ID, "hub": "hf", "trust_remote_code": True},
|
||||
{"model": FUN_ASR_NANO_MODEL_ID, "hub": "ms", "trust_remote_code": False},
|
||||
)
|
||||
VAD_MODEL_ID = "iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"
|
||||
PUNC_MODEL_ID = "iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
|
||||
PARAFORMER_MODEL_ID = "iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
|
||||
UNIASR_MODEL_ID = "iic/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
|
||||
FUN_ASR_NANO_MODEL_PATH = f"{ASR_MODEL_ROOT}/Fun-ASR-Nano-2512"
|
||||
VAD_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_fsmn_vad_zh-cn-16k-common-pytorch"
|
||||
PUNC_MODEL_PATH = f"{ASR_MODEL_ROOT}/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
|
||||
PARAFORMER_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
|
||||
UNIASR_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
|
||||
|
||||
|
||||
def ensure_model_dir(model_id, model_path, revision=None):
|
||||
if os.path.isfile(os.path.join(model_path, "model.pt")):
|
||||
return model_path
|
||||
|
||||
download_kwargs = {"local_dir": model_path}
|
||||
if revision is not None:
|
||||
download_kwargs["revision"] = revision
|
||||
print(f"Downloading FunASR model: {model_id} -> {model_path}")
|
||||
snapshot_download(model_id, **download_kwargs)
|
||||
return model_path
|
||||
|
||||
|
||||
def only_asr(input_file, language, backend="fun-asr-nano"):
|
||||
@ -26,100 +53,86 @@ def only_asr(input_file, language, backend="fun-asr-nano"):
|
||||
return text
|
||||
|
||||
|
||||
def is_unregistered_model_error(exc):
|
||||
message = str(exc).lower()
|
||||
return "model" in message and "not registered" in message
|
||||
def disable_flash_attention_probe():
|
||||
import transformers.utils
|
||||
import transformers.utils.import_utils
|
||||
|
||||
transformers.utils.import_utils.is_flash_attn_2_available = lambda: False
|
||||
transformers.utils.is_flash_attn_2_available = lambda: False
|
||||
|
||||
|
||||
def create_fun_asr_nano_model(device):
|
||||
common_kwargs = dict(
|
||||
vad_model="fsmn-vad",
|
||||
disable_flash_attention_probe()
|
||||
ensure_model_dir(FUN_ASR_NANO_MODEL_ID, FUN_ASR_NANO_MODEL_PATH)
|
||||
ensure_model_dir(VAD_MODEL_ID, VAD_MODEL_PATH)
|
||||
return AutoModel(
|
||||
model=FUN_ASR_NANO_MODEL_PATH,
|
||||
vad_model=VAD_MODEL_PATH,
|
||||
device=device,
|
||||
disable_update=True,
|
||||
check_latest=False,
|
||||
vad_kwargs={"check_latest": False},
|
||||
hub="ms",
|
||||
)
|
||||
last_error = None
|
||||
for model_source in FUN_ASR_NANO_MODEL_SOURCES:
|
||||
try:
|
||||
return AutoModel(**model_source, **common_kwargs)
|
||||
except (KeyError, RuntimeError, ValueError) as exc:
|
||||
if not is_unregistered_model_error(exc):
|
||||
raise
|
||||
last_error = exc
|
||||
|
||||
raise last_error
|
||||
|
||||
def create_damo_model(device):
|
||||
ensure_model_dir(VAD_MODEL_ID, VAD_MODEL_PATH, revision="v2.0.4")
|
||||
ensure_model_dir(PUNC_MODEL_ID, PUNC_MODEL_PATH, revision="v2.0.4")
|
||||
ensure_model_dir(PARAFORMER_MODEL_ID, PARAFORMER_MODEL_PATH, revision="v2.0.4")
|
||||
return AutoModel(
|
||||
model=PARAFORMER_MODEL_PATH,
|
||||
model_revision="v2.0.4",
|
||||
vad_model=VAD_MODEL_PATH,
|
||||
vad_model_revision="v2.0.4",
|
||||
vad_kwargs={"check_latest": False},
|
||||
punc_model=PUNC_MODEL_PATH,
|
||||
punc_model_revision="v2.0.4",
|
||||
punc_kwargs={"check_latest": False},
|
||||
device=device,
|
||||
disable_update=True,
|
||||
check_latest=False,
|
||||
hub="ms",
|
||||
)
|
||||
|
||||
|
||||
def create_uniasr_model(device):
|
||||
ensure_model_dir(UNIASR_MODEL_ID, UNIASR_MODEL_PATH, revision="master")
|
||||
return AutoModel(
|
||||
model=UNIASR_MODEL_PATH,
|
||||
model_revision="master",
|
||||
device=device,
|
||||
disable_update=True,
|
||||
check_latest=False,
|
||||
hub="ms",
|
||||
)
|
||||
|
||||
|
||||
def create_model(language="zh", **kwargs):
|
||||
backend = kwargs.get("backend", "fun-asr-nano")
|
||||
import torch
|
||||
|
||||
# For non-classic backends, route to multilingual models regardless of language
|
||||
if backend in ("fun-asr-nano", "sensevoice") and language != "yue":
|
||||
import torch
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
cache_key = f"{language}_{backend}"
|
||||
if cache_key in funasr_models:
|
||||
return funasr_models[cache_key]
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
cache_key = f"{backend}_{language}"
|
||||
if cache_key in funasr_models:
|
||||
return funasr_models[cache_key]
|
||||
|
||||
if backend == "fun-asr-nano":
|
||||
model = create_fun_asr_nano_model(device)
|
||||
print(f"FunASR Fun-ASR-Nano 模型加载完成: {language.upper()}")
|
||||
else:
|
||||
model = AutoModel(
|
||||
model="iic/SenseVoiceSmall",
|
||||
vad_model="fsmn-vad",
|
||||
device=device,
|
||||
disable_update=True,
|
||||
)
|
||||
print(f"FunASR SenseVoice 模型加载完成: {language.upper()}")
|
||||
|
||||
funasr_models[cache_key] = model
|
||||
return model
|
||||
|
||||
if language == "zh":
|
||||
path_vad = "tools/asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch"
|
||||
path_punc = "tools/asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
|
||||
path_asr = "tools/asr/models/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
|
||||
snapshot_download(
|
||||
"iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
|
||||
local_dir="tools/asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch",
|
||||
)
|
||||
snapshot_download(
|
||||
"iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
|
||||
local_dir="tools/asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
|
||||
)
|
||||
snapshot_download(
|
||||
"iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
|
||||
local_dir="tools/asr/models/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
|
||||
)
|
||||
model_revision = "v2.0.4"
|
||||
vad_model_revision = punc_model_revision = "v2.0.4"
|
||||
elif language == "yue":
|
||||
path_asr = "tools/asr/models/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
|
||||
snapshot_download(
|
||||
"iic/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
|
||||
local_dir="tools/asr/models/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
|
||||
)
|
||||
path_vad = path_punc = None
|
||||
vad_model_revision = punc_model_revision = ""
|
||||
model_revision = "master"
|
||||
if backend == "damo":
|
||||
if language != "zh":
|
||||
raise ValueError("达摩 ASR 仅支持 zh")
|
||||
model = create_damo_model(device)
|
||||
print(f"FunASR 达摩 Paraformer 模型加载完成: {language.upper()}")
|
||||
elif backend == "fun-asr-nano" and language == "yue":
|
||||
model = create_uniasr_model(device)
|
||||
print(f"FunASR UniASR 粤语模型加载完成: {language.upper()}")
|
||||
elif backend == "fun-asr-nano":
|
||||
model = create_fun_asr_nano_model(device)
|
||||
print(f"FunASR Fun-ASR-Nano 模型加载完成: {language.upper()}")
|
||||
else:
|
||||
raise ValueError(f"{language} is not supported. Supported: zh, yue, ja, en, ko, auto")
|
||||
raise ValueError(f"{backend} is not supported")
|
||||
|
||||
if language in funasr_models:
|
||||
return funasr_models[language]
|
||||
else:
|
||||
model = AutoModel(
|
||||
model=path_asr,
|
||||
model_revision=model_revision,
|
||||
vad_model=path_vad,
|
||||
vad_model_revision=vad_model_revision,
|
||||
punc_model=path_punc,
|
||||
punc_model_revision=punc_model_revision,
|
||||
)
|
||||
print(f"FunASR 模型加载完成: {language.upper()}")
|
||||
|
||||
funasr_models[language] = model
|
||||
return model
|
||||
funasr_models[cache_key] = model
|
||||
return model
|
||||
|
||||
|
||||
def execute_asr(input_folder, output_folder, model_size, language, backend="fun-asr-nano"):
|
||||
@ -163,10 +176,14 @@ if __name__ == "__main__":
|
||||
parser.add_argument(
|
||||
"-p", "--precision", type=str, default="float16", choices=["float16", "float32"], help="fp16 or fp32"
|
||||
) # 还没接入
|
||||
parser.add_argument(
|
||||
"--backend", type=str, default="fun-asr-nano", choices=["fun-asr-nano", "damo"], help="ASR backend"
|
||||
)
|
||||
cmd = parser.parse_args()
|
||||
execute_asr(
|
||||
input_folder=cmd.input_folder,
|
||||
output_folder=cmd.output_folder,
|
||||
model_size=cmd.model_size,
|
||||
language=cmd.language,
|
||||
backend=cmd.backend,
|
||||
)
|
||||
|
||||
6
webui.py
6
webui.py
@ -381,6 +381,8 @@ def open_asr(asr_inp_dir, asr_opt_dir, asr_model, asr_model_size, asr_lang, asr_
|
||||
cmd += f" -s {asr_model_size}"
|
||||
cmd += f" -l {asr_lang}"
|
||||
cmd += f" -p {asr_precision}"
|
||||
if "backend" in asr_dict[asr_model]:
|
||||
cmd += f" --backend {asr_dict[asr_model]['backend']}"
|
||||
output_file_name = os.path.basename(asr_inp_dir)
|
||||
output_folder = asr_opt_dir or "output/asr_opt"
|
||||
output_file_path = os.path.abspath(f"{output_folder}/{output_file_name}.list")
|
||||
@ -1412,7 +1414,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
|
||||
label=i18n("ASR 模型"),
|
||||
choices=list(asr_dict.keys()),
|
||||
interactive=True,
|
||||
value="达摩 ASR (中文)",
|
||||
value="达摩 ASR (中文经典)",
|
||||
)
|
||||
asr_size = gr.Dropdown(
|
||||
label=i18n("ASR 模型尺寸"), choices=["large"], interactive=True, value="large"
|
||||
@ -1496,7 +1498,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
|
||||
version_checkbox = gr.Radio(
|
||||
label=i18n("训练模型的版本"),
|
||||
value=version,
|
||||
choices=["v2", "v2Pro", "v2ProPlus", "v4", "v5dev", "v5turbo"],
|
||||
choices=["v2", "v2Pro", "v2ProPlus", "v4"],
|
||||
scale=5,
|
||||
)
|
||||
with gr.Accordion(label=i18n("预训练模型路径"), open=False):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user