Compare commits

...

5 Commits

Author SHA1 Message Date
Sai Asish Y
f4406e4f9c
Merge 4509e68e0430cc5170977c8ea3dffaed17578e8c into 6d51ffceafbf240145619af4c92408a6e54b7924 2026-11-16 00:17:25 -08:00
RVC-Boss
6d51ffceaf Update ASR configuration, Faster Whisper, FunASR and WebUI from update_asr package 2026-11-12 00:26:17 +08:00
Kakaru
59c44686f5
feat: add cache-dit (DBCache) block caching to the DiT sampler (#2868)
The DiT samples with the full block stack on every diffusion step. cache-dit
(vipshop/cache-dit, DBCache) skips the middle of the stack when the front residual
barely moved:

    front : trunk(x) -> blocks[0:Fn] -> x_F,   R = x_F - trunk(x)
            fd = sum|R - R_prev| / (sum|R_prev| + eps)
    hit   : x_mid = x_F + MnResidual_prev, then norm_out/proj_out only
    miss  : x_F -> blocks[Fn:depth] -> x_M, store MnResidual = x_M - x_F, R -> R_prev

The head (norm_out + proj_out) runs unconditionally, hits included. A hit additionally
needs `fd < threshold` outside the warmup steps and must pass the max_cached / max_cont /
max_accum gates.

The conditional and unconditional CFG branches are two separate forward passes, so each
keeps its own R_prev, MnResidual, hit/miss counters and gate counters.

Implementation:

* new `module/cache_dit.py`: configuration, per-sampling state, the CFM-side mixin;
* `DiT.forward` split into `trunk()` / `run_blocks(start, end)` / `head()` (weights are
  shared, nothing is copied or duplicated);
* `CFM.inference` creates one cache state per sampling run, advances it per step and reads
  back a single scalar per step (the only device/host sync);
* `cache_dit=off|auto|f4|f8|f16` plus dbcache_fn/dbcache_threshold/dbcache_warmup/
  dbcache_downsample on TTS_infer_pack.run, api.py, api_v2.py, both WebUIs and the CLI,
  with GSV_DBCACHE* environment fallbacks for scripts that build no request dict.

Default is off and the off path is the previous one: the split is bit-identical to the old
`forward()` on 24 configurations (fp16/fp32 x long skip x static cache x inference/training),
and `CFM.inference` with the switch off reproduces the pre-change output bit for bit with the
real weights. `auto` resolves to F8 from 16 sampling steps and stays off below that, so
4-step runs are never cached (the warmup budget already covers them).

Measured on a 32-step v5dev request with F8 / threshold 0.02 / warmup 8 / downsample 4:
18.8% of the DiT forwards served from cache, mel relL1 1.2e-3 against the uncached run,
DiT stage ~1.12x faster; F4 at the same threshold gives 21.9% and ~1.18x, threshold 0.05
gives 45.3% and ~1.34x.
2026-11-09 21:59:04 +08:00
RVC-Boss
04a44eb977 Sync 20261031 v5 code update and remove unused v5_inference module 2026-10-31 23:56:56 +08:00
Sai Asish Y
4509e68e04 Fix inference_cli.py never loading the requested SoVITS weights
Signed-off-by: Sai Asish Y <say.apm35@gmail.com>
2026-08-25 16:22:51 -07:00
14 changed files with 942 additions and 413 deletions

View File

@ -512,7 +512,6 @@ class TTS:
self.t2s_model: Text2SemanticLightningModule = None
self.t2s_accel = None
self.streaming_accel = None
self.vits_model: Union[SynthesizerTrn, SynthesizerTrnV3] = None
self.bert_tokenizer: AutoTokenizer = None
self.bert_model: AutoModelForMaskedLM = None
@ -679,9 +678,6 @@ class TTS:
def init_t2s_weights(self, weights_path: str):
if self.streaming_accel is not None:
self.streaming_accel.close()
self.streaming_accel = None
if self.t2s_accel is not None:
self.t2s_accel.close()
self.t2s_accel = None
@ -795,9 +791,6 @@ class TTS:
print("Half precision is not supported on CPU.")
return
if self.streaming_accel is not None:
self.streaming_accel.close()
self.streaming_accel = None
if self.t2s_accel is not None:
self.t2s_accel.close()
self.t2s_accel = None
@ -834,9 +827,6 @@ class TTS:
Args:
device: torch.device, the device to use for all models.
"""
if self.streaming_accel is not None:
self.streaming_accel.close()
self.streaming_accel = None
if self.t2s_accel is not None:
self.t2s_accel.close()
self.t2s_accel = None
@ -1104,6 +1094,41 @@ class TTS:
"""
self.stop_flag = True
def apply_cache_dit(self, inputs: dict = None, sample_steps: int = 32):
"""cache-dit / DBCache switch for the DiT sampler. **Default: off.**
Reads the same field names as the HTTP API, so a request, an environment variable and
a direct call give the same result:
cache_dit: off | auto | f4 | f8 | f16 (alias: dbcache)
dbcache_fn, dbcache_threshold, dbcache_warmup, dbcache_downsample
dbcache_warmup_interval, dbcache_max_cached, dbcache_max_cont, dbcache_max_accum
Env fallbacks: ``GSV_DBCACHE``, ``GSV_DBCACHE_THRESHOLD``, ``GSV_DBCACHE_WARMUP``,
``GSV_DBCACHE_DOWNSAMPLE`` (and the other ``GSV_DBCACHE_*`` in module/cache_dit.py).
``auto`` = F8 when ``sample_steps >= 16`` and off below that, so a 4-step run is never
cached by accident (its warmup budget already covers the whole run, so there is nothing
to gain and the quality risk is real).
"""
cfm = getattr(self.vits_model, "cfm", None)
if cfm is None or not hasattr(cfm, "set_dbcache"):
return None
from module.cache_dit import config_from_params
cfg = config_from_params(dict(inputs or {}), int(sample_steps),
getattr(cfm, "cache_depth", 0) or None)
if cfg.enabled and str(getattr(self.configs, "version", "")) == "v5turbo" and not cfg.force:
# 4-step turbo: nothing to cache, keep it off
print(i18n("cache-dit: not supported for v5turbo (4 steps), staying off"))
cfg.fn = 0
cfm.set_dbcache(cfg)
if cfg.enabled:
if int(sample_steps) < 16:
print(i18n("cache-dit: only %d sampling steps, expect a quality drop "
"(4-step v5turbo should keep cache-dit off)" % int(sample_steps)))
print(i18n("cache-dit: %s" % cfg))
return cfg
@torch.no_grad()
def run(self, inputs: dict):
"""
@ -1131,6 +1156,13 @@ class TTS:
"parallel_infer": True, # bool. whether to use parallel inference.
"repetition_penalty": 1.35, # float. repetition penalty for T2S model.
"sample_steps": 32, # int. number of sampling steps for VITS model V3.
"cfg_rate": 1.30, # float. classifier-free-guidance rate for V5dev/V5.
"cache_dit": "off", # str. off|auto|f4|f8|f16. DBCache for the V3/V4/V5 DiT:
# fewer DiT forwards, near-identical audio. off by default.
"dbcache_fn": 8, # int. front blocks (auto = F8 when sample_steps >= 16).
"dbcache_threshold": 0.02, # float. relative residual-diff threshold.
"dbcache_warmup": 8, # int. forced full-compute steps at the start.
"dbcache_downsample": 4, # int. time downsample of the cached Fn residual.
"super_sampling": False, # bool. whether to use super-sampling for audio when using VITS model V3.
"return_fragment": False, # bool. step by step return the audio fragment. (Best Quality, Slowest response speed. old version of streaming mode)
"streaming_mode": False, # bool. return audio chunk by chunk. (Medium quality, Slow response speed)
@ -1170,6 +1202,8 @@ class TTS:
cfg_rate = inputs.get("cfg_rate")
sample_steps = (4 if self.configs.version == "v5turbo" else 32) if sample_steps is None else int(sample_steps)
cfg_rate = (1.30 if self.configs.version == "v5dev" else 0.0) if cfg_rate is None else float(cfg_rate)
# cache-dit (DBCache): off unless the request asks for it, see module/cache_dit.py
self.apply_cache_dit(inputs, sample_steps)
super_sampling = inputs.get("super_sampling", False)
streaming_mode = inputs.get("streaming_mode", False)
overlap_length = inputs.get("overlap_length", 2)
@ -1177,31 +1211,7 @@ class TTS:
fixed_length_chunk = inputs.get("fixed_length_chunk", False)
chunk_split_thershold = 0.0 # 该值代表语义token与mute token的余弦相似度阈值,若大于该阈值,则视为可切分点。
gpt_streaming = inputs.get("gpt_streaming", False)
if gpt_streaming:
from .streaming import (validate_streaming_options, PipelineStop, semantic_updates,
v2_token_chunks, v5_reference, stream_v5_windows,
StreamingWindowSchedule)
validate_streaming_options(inputs, self.configs.version)
stream_stop = PipelineStop(self)
stream_options = dict(inputs, sample_steps=sample_steps, cfg_rate=cfg_rate,
speed_factor=speed_factor,
streaming_chunk_seconds=inputs.get("streaming_chunk_seconds", 2.0),
streaming_lookahead_seconds=inputs.get("streaming_lookahead_seconds", 0.12),
streaming_overlap_seconds=inputs.get("streaming_overlap_seconds", 0.04),
gpt_streaming_lookahead_seconds=inputs.get("gpt_streaming_lookahead_seconds", 0.5))
acoustic_rng = torch.Generator(device=self.configs.device).manual_seed(actual_seed)
# Reference semantic codes depend on the active SoVITS model.
if getattr(self, "_streaming_ref_model", None) != id(self.vits_model):
ref_audio_path = ref_audio_path or self.prompt_cache["ref_audio_path"]
if not ref_audio_path:
raise ValueError("A reloadable reference audio is required after model changes")
self.prompt_cache["ref_audio_path"] = None
self._streaming_ref_model = id(self.vits_model)
if self.prompt_cache["prompt_lang"] != prompt_lang:
self.prompt_cache["prompt_text"] = None
if streaming_mode and self.configs.use_vocoder and not gpt_streaming:
if streaming_mode and self.configs.use_vocoder:
print(i18n("SoVits V3/4模型不支持流式推理模式,已自动回退到分段返回模式"))
streaming_mode = False
return_fragment = True
@ -1253,7 +1263,7 @@ class TTS:
assert prompt_lang in self.configs.languages
if no_prompt_text and self.configs.use_vocoder:
raise NO_PROMPT_ERROR("prompt_text cannot be empty when using SoVITS_V3")
raise NO_PROMPT_ERROR("SoVITS V3/V4/V5 do not support reference-text-free mode; prompt_text cannot be empty")
if ref_audio_path in [None, ""] and (
(self.prompt_cache["prompt_semantic"] is None) or (self.prompt_cache["refer_spec"] in [None, []])
@ -1321,15 +1331,7 @@ class TTS:
)
else:
print(f"############ {i18n('切分文本')} ############")
# Preserve cut0's single-session behavior; other methods use the
# frontend's normal punctuation/newline and long-text handling.
texts = self.text_preprocessor.pre_seg_text(
text.replace("\n", " ") if gpt_streaming and text_split_method == "cut0" else text,
text_lang,
text_split_method,
)
if gpt_streaming and text_split_method == "cut0" and len(texts) > 1:
raise ValueError("Text exceeds the single-session frontend limit; shorten the input")
texts = self.text_preprocessor.pre_seg_text(text, text_lang, text_split_method)
data = []
for i in range(len(texts)):
if i % batch_size == 0:
@ -1365,8 +1367,6 @@ class TTS:
return batch[0]
t2 = time.perf_counter()
streaming_tokens = streaming_audio = semantic_token_generator = None
streaming_reference = None
try:
print("############ 推理 ############")
###### inference ######
@ -1376,8 +1376,6 @@ class TTS:
is_first_package = True
output_sr = self.configs.sampling_rate if not self.configs.use_vocoder else self.vocoder_configs["sr"]
for item in data:
if gpt_streaming and stream_stop.is_set():
break
t3 = time.perf_counter()
if return_fragment or streaming_mode:
item = make_batch(item)
@ -1536,59 +1534,27 @@ class TTS:
)
batch_audio_fragment.append(audio_fragment)
elif gpt_streaming and self.configs.version in V5_VERSIONS:
if streaming_reference is None:
streaming_reference = v5_reference(self)
reference = streaming_reference
# Validate the acoustic context before starting the AR owner thread.
frames = round(stream_options["streaming_chunk_seconds"] * 100)
future = round(stream_options["streaming_lookahead_seconds"] * 100)
if reference[6].shape[-1] + frames + future > 1000:
raise ValueError("Reference plus streaming window exceeds the V5 context")
streaming_tokens = semantic_updates(self, all_phoneme_ids[0], prompt,
all_bert_features[0], stream_options, stream_stop)
streaming_audio = stream_v5_windows(self.vits_model, self.vocoder, denorm_spec,
self.configs.device, streaming_tokens, batch_phones[0], reference,
stream_options, stream_stop)
for pcm in streaming_audio:
yield output_sr, pcm
streaming_audio.close()
streaming_tokens.close()
streaming_audio = streaming_tokens = None
if not stream_stop.is_set() and fragment_interval:
yield output_sr, np.zeros(int(output_sr * fragment_interval), dtype=np.int16)
t4 = time.perf_counter()
else:
# refer_audio_spec: torch.Tensor = [
# item.to(dtype=self.precision, device=self.configs.device)
# for item in self.prompt_cache["refer_spec"]
# ]
window_schedule = None
if gpt_streaming:
if inputs.get("streaming_window_mode", "fixed") == "hybrid":
window_schedule = StreamingWindowSchedule(min_chunk_length, min_chunk_length * 4)
streaming_tokens = semantic_updates(self, all_phoneme_ids[0], prompt,
all_bert_features[0], stream_options, stream_stop)
semantic_token_generator = v2_token_chunks(streaming_tokens, min_chunk_length,
self.configs.mute_emb_sim_matrix if not fixed_length_chunk else None,
chunk_split_thershold, schedule=window_schedule)
else:
semantic_token_generator =self.t2s_model.model.infer_panel(
all_phoneme_ids[0].unsqueeze(0),
all_phoneme_lens,
prompt,
all_bert_features[0].unsqueeze(0),
top_k=top_k,
top_p=top_p,
temperature=temperature,
early_stop_num=self.configs.hz * self.configs.max_sec,
max_len=max_len,
repetition_penalty=repetition_penalty,
streaming_mode=True,
chunk_length=min_chunk_length,
mute_emb_sim_matrix=self.configs.mute_emb_sim_matrix if not fixed_length_chunk else None,
chunk_split_thershold=chunk_split_thershold,
)
semantic_token_generator =self.t2s_model.model.infer_panel(
all_phoneme_ids[0].unsqueeze(0),
all_phoneme_lens,
prompt,
all_bert_features[0].unsqueeze(0),
top_k=top_k,
top_p=top_p,
temperature=temperature,
early_stop_num=self.configs.hz * self.configs.max_sec,
max_len=max_len,
repetition_penalty=repetition_penalty,
streaming_mode=True,
chunk_length=min_chunk_length,
mute_emb_sim_matrix=self.configs.mute_emb_sim_matrix if not fixed_length_chunk else None,
chunk_split_thershold=chunk_split_thershold,
)
t4 = time.perf_counter()
t_34 += t4 - t3
phones = batch_phones[0].unsqueeze(0).to(self.configs.device)
@ -1653,8 +1619,7 @@ class TTS:
result_length=semantic_tokens.shape[-1]+overlap_len if not is_first_chunk else None,
overlap_frames=last_latent[:,:,-overlap_len*(2 if self.vits_model.semantic_frame_rate == "25hz" else 1):] \
if last_latent is not None else None,
padding_length=token_padding_length,
**({"generator": acoustic_rng} if gpt_streaming else {})
padding_length=token_padding_length
)
audio_chunk=audio_chunk.detach()[0, 0, :]
else:
@ -1669,7 +1634,7 @@ class TTS:
audio_chunk_ = audio_chunk_[:-overlap_size]
elif is_first_chunk and is_final:
is_first_chunk = False
elif not is_first_chunk and (not is_final or gpt_streaming):
elif not is_first_chunk and not is_final:
audio_chunk_ = self.sola_algorithm([last_audio_chunk, audio_chunk_], overlap_size)
audio_chunk_ = (
audio_chunk_[last_audio_chunk.shape[0]-overlap_size:-overlap_size] if not is_final \
@ -1678,7 +1643,7 @@ class TTS:
last_latent = latent
last_audio_chunk = audio_chunk
audio_result = self.audio_postprocess(
yield self.audio_postprocess(
[[audio_chunk_]],
output_sr,
None,
@ -1687,21 +1652,13 @@ class TTS:
0.0,
super_sampling if self.configs.use_vocoder and self.configs.version == "v3" else False,
)
if window_schedule is not None:
window_schedule.observe(semantic_tokens.numel(), audio_result[1].size, output_sr)
yield audio_result
if is_first_package:
print(f"first_package_delay: {time.perf_counter()-t0:.3f}")
is_first_package = False
if gpt_streaming:
semantic_token_generator.close()
streaming_tokens.close()
semantic_token_generator = streaming_tokens = None
if not gpt_streaming or not stream_stop.is_set():
yield output_sr, np.zeros(int(output_sr*fragment_interval), dtype=np.int16)
yield output_sr, np.zeros(int(output_sr*fragment_interval), dtype=np.int16)
t5 = time.perf_counter()
t_45 += t5 - t4
@ -1740,8 +1697,6 @@ class TTS:
)
except Exception as e:
if gpt_streaming:
raise
traceback.print_exc()
# 必须返回一个空音频, 否则会导致显存不释放。
yield 16000, np.zeros(int(16000), dtype=np.int16)
@ -1754,11 +1709,6 @@ class TTS:
self.init_vits_weights(self.configs.vits_weights_path)
raise e
finally:
if gpt_streaming:
stream_stop.set()
for iterator in (streaming_audio, semantic_token_generator, streaming_tokens):
if iterator is not None:
iterator.close()
self.empty_cache()
def empty_cache(self):

View File

@ -171,7 +171,58 @@ class DiT(nn.Module):
text_cache=None, # torch tensor as text_embed
dt_cache=None, # torch tensor as dt
static_cache=None,
# cache-dit / DBCache (see module/cache_dit.py). `dbcache=None` => untouched path.
dbcache=None, # module.cache_dit.DBCacheState
dbcache_branch=0, # 0 = CFG positive, 1 = CFG negative
):
x, t, mask, rope, text_embed, dt = self.trunk(
x0, cond0, x_lens, time, dt_base_bootstrap=dt_base_bootstrap, text0=text0,
drop_audio_cond=drop_audio_cond, drop_text=drop_text, infer=infer,
text_cache=text_cache, dt_cache=dt_cache, static_cache=static_cache,
)
residual = x
if self.long_skip_connection is None:
residual = None
if dbcache is None or not dbcache.enabled:
# ---- original path: all blocks, then head ------------------------------
x = self.run_blocks(x, t, mask, rope, 0, self.depth, use_grad_ckpt)
else:
# ---- cache-dit: front -> (hit: head only | miss: mid -> head) -----------
fn = max(0, min(int(dbcache.fn), self.depth - 1))
x_f = self.run_blocks(x, t, mask, rope, 0, fn, use_grad_ckpt)
fd = dbcache.front_update(dbcache_branch, x_f, x)
if dbcache.can_cache(dbcache_branch, fd):
x = x_f + dbcache.mid_residual(dbcache_branch)
dbcache.on_hit(dbcache_branch)
if dbcache.trace:
print("[dbcache] step=%d HIT fd=%.5f (hits=%d misses=%d)"
% (dbcache.step, fd, dbcache.branch(dbcache_branch).hits,
dbcache.branch(dbcache_branch).misses), flush=True)
else:
x = self.run_blocks(x_f, t, mask, rope, fn, self.depth, use_grad_ckpt)
dbcache.on_miss(dbcache_branch, x, x_f)
if dbcache.trace:
print("[dbcache] step=%d MISS fd=%.5f (hits=%d misses=%d)"
% (dbcache.step, fd, dbcache.branch(dbcache_branch).hits,
dbcache.branch(dbcache_branch).misses), flush=True)
output = self.head(x, t, residual)
if infer:
return output, text_embed if static_cache is None else None, dt
else:
return output
# ---------------------------------------------------------------------------------
# Three-segment split used by cache-dit; the weights are shared, nothing is copied.
# ---------------------------------------------------------------------------------
def trunk(self, x0, cond0, x_lens, time, dt_base_bootstrap=None, text0=None,
drop_audio_cond=False, drop_text=False, infer=False, text_cache=None,
dt_cache=None, static_cache=None):
"""Everything before the transformer blocks: t/dt embedding + input embedding."""
text_embed = None
x = x0.transpose(2, 1)
cond = cond0.transpose(2, 1)
text = text0.transpose(2, 1)
@ -207,22 +258,22 @@ class DiT(nn.Module):
rope = (static_cache["rope"] if static_cache is not None
else self.rotary_embed.forward_from_seq_len(seq_len))
if self.long_skip_connection is not None:
residual = x
return x, t, mask, rope, text_embed, dt
for block in self.transformer_blocks:
def run_blocks(self, x, t, mask, rope, start=0, end=None, use_grad_ckpt=False):
"""`front` is blocks[0:fn], `mid` is blocks[fn:depth]."""
end = self.depth if end is None else end
for block in self.transformer_blocks[start:end]:
if use_grad_ckpt:
x = checkpoint(self.ckpt_wrapper(block), x, t, mask, rope, use_reentrant=False)
else:
x = block(x, t, mask=mask, rope=rope)
return x
def head(self, x, t, residual=None):
"""`norm_out -> proj_out`, computed on every step, hits included."""
if self.long_skip_connection is not None:
x = self.long_skip_connection(torch.cat((x, residual), dim=-1))
x = self.norm_out(x, t)
output = self.proj_out(x)
return self.proj_out(x)
if infer:
return output, text_embed if static_cache is None else None, dt
else:
return output

View File

@ -3,6 +3,7 @@ import os
import soundfile as sf
from tools.i18n.i18n import I18nAuto
import GPT_SoVITS.inference_webui as webui_ui
from GPT_SoVITS.inference_webui import change_gpt_weights, change_sovits_weights, get_tts_wav
i18n = I18nAuto()
@ -28,7 +29,12 @@ def synthesize(
# Change model weights
change_gpt_weights(gpt_path=GPT_model_path)
change_sovits_weights(sovits_path=SoVITS_model_path)
for _ in change_sovits_weights(
sovits_path=SoVITS_model_path,
prompt_language=i18n(ref_language),
text_language=i18n(target_language),
):
pass
# Synthesize audio
synthesis_result = get_tts_wav(
@ -67,9 +73,25 @@ def main():
help="Language of the target text",
)
parser.add_argument("--output_path", required=True, help="Path to the output directory")
parser.add_argument(
"--cache_dit",
default="off",
choices=["off", "auto", "f4", "f8", "f16"],
help="cache-dit (DBCache) for the V3/V4/V5dev DiT, off by default; "
"fewer DiT forwards with near-identical audio",
)
parser.add_argument("--dbcache_fn", type=int, default=-1, help="front blocks (default 8)")
parser.add_argument("--dbcache_threshold", type=float, default=0.02, help="residual-diff threshold")
parser.add_argument("--dbcache_warmup", type=int, default=8, help="full-compute steps first")
parser.add_argument("--dbcache_downsample", type=int, default=4, help="residual time downsample")
args = parser.parse_args()
webui_ui.update_cache_dit(
args.cache_dit, args.dbcache_fn, args.dbcache_threshold,
args.dbcache_warmup, args.dbcache_downsample,
)
synthesize(
args.gpt_model,
args.sovits_model,

View File

@ -100,8 +100,8 @@ cnhubert.cnhubert_base_path = cnhubert_base_path
import random
from tools.acceleration import create_acceleration, cuda_graph_available, flash_attention_available
from module.models import Generator, SynthesizerTrn, SynthesizerTrnV3
from module.models_v5 import synthesize_v5_mel, V5_VERSIONS
from module.models import CFM, Generator, SynthesizerTrn, SynthesizerTrnV3
from module.models_v5 import V5_VERSIONS
def set_seed(seed):
@ -283,14 +283,14 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None)
{
"__type__": "update",
"visible": visible_sample_steps,
"value": (4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8),
"value": (32 if model_version == "v3" else 8),
"choices": [4, 8, 16, 32],
},
{"__type__": "update", "visible": visible_inp_refs},
{"__type__": "update", "value": False, "interactive": True if model_version not in v3v4set else False},
{"__type__": "update", "visible": True if model_version == "v3" else False},
{"__type__": "update", "value": i18n("模型加载中,请等待"), "interactive": False},
{"__type__": "update", "visible": model_version in V5_VERSIONS, "value": (1.30 if model_version == "v5dev" else 0.0)},
{"__type__": "update", "visible": False, "value": 0.0, "interactive": False},
)
dict_s2 = load_sovits_new(sovits_path)
@ -355,6 +355,10 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None)
# torch.save(vq_model.state_dict(),"merge_win.pth")
vq_model.eval()
if model_version in V5_VERSIONS:
vq_model.cfm.noise_temperature = None
vq_model.cfm.use_static_cache = False
yield (
{"__type__": "update", "choices": list(dict_language.keys())},
{"__type__": "update", "choices": list(dict_language.keys())},
@ -365,14 +369,14 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None)
{
"__type__": "update",
"visible": visible_sample_steps,
"value": (4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8),
"value": (32 if model_version == "v3" else 8),
"choices": [4, 8, 16, 32],
},
{"__type__": "update", "visible": visible_inp_refs},
{"__type__": "update", "value": False, "interactive": True if model_version not in v3v4set else False},
{"__type__": "update", "visible": True if model_version == "v3" else False},
{"__type__": "update", "value": i18n("合成语音"), "interactive": True},
{"__type__": "update", "visible": model_version in V5_VERSIONS, "value": (1.30 if model_version == "v5dev" else 0.0)},
{"__type__": "update", "visible": False, "value": 0.0, "interactive": False},
)
with open("./weight.json") as f:
data = f.read()
@ -492,13 +496,12 @@ def init_hifigan():
hifigan_model.eval()
hifigan_model.remove_weight_norm()
state_dict_g = torch.load(
"%s/GPT_SoVITS/pretrained_models/gsv-%s-pretrained/vocoder.pth" %
(now_dir, "v5" if model_version in V5_VERSIONS else model_version),
"%s/GPT_SoVITS/pretrained_models/gsv-v4-pretrained/vocoder.pth" % (now_dir,),
map_location="cpu",
weights_only=False,
)
print("loading vocoder", hifigan_model.load_state_dict(state_dict_g))
hifigan_version = model_version
hifigan_version = "v4"
clean_bigvgan_model()
clean_sv_cn_model()
if is_half == True:
@ -841,6 +844,36 @@ def audio_sr(audio, sr):
# cache_tokens={}#暂未实现清理机制
cache = {}
# cache-dit (DBCache) for the DiT sampler: off by default. The UI writes here, get_tts_wav
# reads it; the GSV_DBCACHE* environment variables work as well.
CACHE_DIT = {"cache_dit": "off", "dbcache_fn": None, "dbcache_threshold": 0.02,
"dbcache_warmup": 8, "dbcache_downsample": 4}
def update_cache_dit(mode, fn, thr, warmup, ds):
CACHE_DIT["cache_dit"] = str(mode or "off")
CACHE_DIT["dbcache_fn"] = None if fn in (None, "") or int(fn) < 0 else int(fn)
CACHE_DIT["dbcache_threshold"] = float(thr) if thr not in (None, "") else 0.02
CACHE_DIT["dbcache_warmup"] = int(warmup) if warmup not in (None, "") else 8
CACHE_DIT["dbcache_downsample"] = int(ds) if ds not in (None, "") else 4
def apply_cache_dit(vq_model, sample_steps, params=None):
"""cache-dit (DBCache) switch; off unless configured (UI or GSV_DBCACHE* env)."""
cfm = getattr(vq_model, "cfm", None)
if cfm is None or not hasattr(cfm, "set_dbcache"):
return None
from module.cache_dit import config_from_params
cfg = config_from_params(dict(params if params is not None else CACHE_DIT),
int(sample_steps), getattr(cfm, "cache_depth", 0) or None)
if cfg.enabled and str(getattr(vq_model, "version", "")) == "v5turbo" and not cfg.force:
cfg.fn = 0 # 4-step turbo: no gain and unstable quality, keep it off
cfm.set_dbcache(cfg)
if cfg.enabled:
print("cache-dit:", cfg)
return cfg
def get_tts_wav(
ref_wav_path,
@ -865,10 +898,11 @@ def get_tts_wav(
):
global cache, t2s_model_cudagraph
if sample_steps is None:
sample_steps = 4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8
sample_steps = 32 if model_version == "v3" else 8
if cfg_rate is None:
cfg_rate = 1.30 if model_version == "v5dev" else 0.0
cfg_rate = 0.0
sample_steps, cfg_rate = int(sample_steps), float(cfg_rate)
apply_cache_dit(globals().get("vq_model"), sample_steps)
if ref_wav_path:
pass
else:
@ -1081,30 +1115,27 @@ def get_tts_wav(
fea_todo, ge = vq_model.decode_encp(pred_semantic, phoneme_ids1, refer, ge, speed)
cfm_resss = []
idx = 0
if model_version in V5_VERSIONS:
cfm_res = synthesize_v5_mel(vq_model, fea_ref, fea_todo, mel2, sample_steps, cfg_rate)
cfm_res = cfm_res.to(fea_todo.dtype)
else:
while 1:
fea_todo_chunk = fea_todo[:, :, idx : idx + chunk_len]
if fea_todo_chunk.shape[-1] == 0:
break
idx += chunk_len
fea = torch.cat([fea_ref, fea_todo_chunk], 2).transpose(2, 1)
cfm_res = vq_model.cfm.inference(
fea, torch.LongTensor([fea.size(1)]).to(fea.device), mel2, sample_steps, inference_cfg_rate=0
)
cfm_res = cfm_res[:, :, mel2.shape[2] :]
mel2 = cfm_res[:, :, -T_min:]
fea_ref = fea_todo_chunk[:, :, -T_min:]
cfm_resss.append(cfm_res)
cfm_res = torch.cat(cfm_resss, 2)
while 1:
fea_todo_chunk = fea_todo[:, :, idx : idx + chunk_len]
if fea_todo_chunk.shape[-1] == 0:
break
idx += chunk_len
fea = torch.cat([fea_ref, fea_todo_chunk], 2).transpose(2, 1)
cfm_res = CFM.inference(
vq_model.cfm, fea, torch.LongTensor([fea.size(1)]).to(fea.device), mel2, sample_steps,
temperature=1.0, inference_cfg_rate=0,
)
cfm_res = cfm_res[:, :, mel2.shape[2] :]
mel2 = cfm_res[:, :, -T_min:]
fea_ref = fea_todo_chunk[:, :, -T_min:]
cfm_resss.append(cfm_res)
cfm_res = torch.cat(cfm_resss, 2)
cfm_res = denorm_spec(cfm_res)
if model_version == "v3":
if bigvgan_model == None:
init_bigvgan()
else: # v4
if hifigan_model is None or hifigan_version != model_version:
if hifigan_model is None or hifigan_version != "v4":
init_hifigan()
vocoder_model = bigvgan_model if model_version == "v3" else hifigan_model
with torch.inference_mode():
@ -1299,7 +1330,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
with gr.Column(scale=13):
ref_text_free = gr.Checkbox(
label=i18n("开启无参考文本模式。不填参考文本亦相当于开启。")
+ i18n("v3暂不支持该模式,使用了会报错。"),
+ i18n("v3、v4、v5 均不支持无参考文本模式,请填写参考文本!"),
value=False,
interactive=True if model_version not in v3v4set else False,
show_label=True,
@ -1322,8 +1353,8 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
)
cfg_rate = gr.Slider(
minimum=0, maximum=2, step=0.05,
value=(1.30 if model_version == "v5dev" else 0.0), label="CFG",
visible=model_version in V5_VERSIONS, interactive=True, scale=1,
value=0.0, label="CFG",
visible=False, interactive=False, scale=1,
)
inp_refs = (
gr.File(
@ -1344,7 +1375,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
sample_steps = (
gr.Radio(
label=i18n("采样步数,如果觉得电,提高试试,如果觉得慢,降低试试"),
value=(4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8),
value=(32 if model_version == "v3" else 8),
choices=[4, 8, 16, 32],
visible=True,
)
@ -1353,9 +1384,29 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
label=i18n("采样步数,如果觉得电,提高试试,如果觉得慢,降低试试"),
choices=[4, 8, 16, 32],
visible=False,
value=(4 if model_version == "v5turbo" else 32 if model_version in {"v3", "v5dev"} else 8),
value=(32 if model_version == "v3" else 8),
)
)
with gr.Accordion(i18n("DiT 加速 cache-dit(默认关闭)"), open=False):
with gr.Row():
cache_dit_mode = gr.Dropdown(
label=i18n("cache-dit"),
choices=["off", "auto", "f4", "f8", "f16"],
value="off", scale=2,
)
dbcache_fn = gr.Number(label="fn", value=-1, precision=0, scale=1)
dbcache_threshold = gr.Number(label="thr", value=0.02, scale=1)
dbcache_warmup = gr.Number(label="warmup", value=8, precision=0, scale=1)
dbcache_downsample = gr.Number(label="ds", value=4, precision=0, scale=1)
gr.Markdown(i18n("DBCache:只算前 fn 个 block,残差变化小于 thr 时跳过中间 block。"
"fn=-1 表示跟随模式(auto = 步数>=16 时 F8)。4 步的 v5turbo 不支持。"))
for _cd in (cache_dit_mode, dbcache_fn, dbcache_threshold,
dbcache_warmup, dbcache_downsample):
_cd.change(
update_cache_dit,
[cache_dit_mode, dbcache_fn, dbcache_threshold,
dbcache_warmup, dbcache_downsample],
)
if_sr_Checkbox = gr.Checkbox(
label=i18n("v3输出如果觉得闷可以试试开超分"),
value=False,

View File

@ -73,6 +73,19 @@ language = os.environ.get("language", "Auto")
language = sys.argv[-1] if sys.argv[-1] in scan_language_list() else language
i18n = I18nAuto(language=language)
# cache-dit (DBCache) for the DiT sampler: off by default. The UI writes here, tts_infer
# forwards it to TTS_infer_pack; the GSV_DBCACHE* environment variables work too.
CACHE_DIT = {"cache_dit": "off", "dbcache_fn": None, "dbcache_threshold": 0.02,
"dbcache_warmup": 8, "dbcache_downsample": 4}
def update_cache_dit(mode, fn, thr, warmup, ds):
CACHE_DIT["cache_dit"] = str(mode or "off")
CACHE_DIT["dbcache_fn"] = None if fn in (None, "") or int(fn) < 0 else int(fn)
CACHE_DIT["dbcache_threshold"] = float(thr) if thr not in (None, "") else 0.02
CACHE_DIT["dbcache_warmup"] = int(warmup) if warmup not in (None, "") else 8
CACHE_DIT["dbcache_downsample"] = int(ds) if ds not in (None, "") else 4
# os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1' # 确保直接启动推理UI时也能够设置。
@ -209,11 +222,12 @@ def inference(
"cfg_rate": float(cfg_rate),
"super_sampling": super_sampling,
}
inputs.update(CACHE_DIT) # cache-dit (DBCache), off unless the UI / env enables it
try:
for item in tts_pipeline.run(inputs):
yield item, actual_seed
except NO_PROMPT_ERROR:
gr.Warning(i18n("V3不支持无参考文本模式,请填写参考文本!"))
gr.Warning(i18n("v3、v4、v5 均不支持无参考文本模式,请填写参考文本!"))
def custom_sort_key(s):
@ -373,7 +387,8 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
)
with gr.Column():
ref_text_free = gr.Checkbox(
label=i18n("开启无参考文本模式。不填参考文本亦相当于开启。"),
label=i18n("开启无参考文本模式。不填参考文本亦相当于开启。")
+ i18n("v3、v4、v5 均不支持无参考文本模式,请填写参考文本!"),
value=False,
interactive=True if model_version != "v3" else False,
show_label=True,
@ -420,6 +435,25 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
repetition_penalty = gr.Slider(
minimum=0, maximum=2, step=0.05, label=i18n("重复惩罚"), value=1.35, interactive=True
)
with gr.Accordion(i18n("DiT 加速 cache-dit(默认关闭)"), open=False):
with gr.Row():
cache_dit_mode = gr.Dropdown(
label=i18n("cache-dit"), choices=["off", "auto", "f4", "f8", "f16"],
value="off", interactive=True, scale=2,
)
dbcache_fn = gr.Number(label="fn", value=-1, precision=0, scale=1)
dbcache_threshold = gr.Number(label="thr", value=0.02, scale=1)
dbcache_warmup = gr.Number(label="warmup", value=8, precision=0, scale=1)
dbcache_downsample = gr.Number(label="ds", value=4, precision=0, scale=1)
gr.Markdown(i18n("DBCache:只算前 fn 个 block,残差变化小于 thr 时跳过中间 block。"
"fn=-1 表示跟随模式(auto = 步数>=16 时 F8)。v5turbo 4 步不支持。"))
for _cd in (cache_dit_mode, dbcache_fn, dbcache_threshold,
dbcache_warmup, dbcache_downsample):
_cd.change(
update_cache_dit,
[cache_dit_mode, dbcache_fn, dbcache_threshold,
dbcache_warmup, dbcache_downsample],
)
with gr.Column():
with gr.Row():

View File

@ -0,0 +1,437 @@
"""cache-dit / DBCache (three-segment block caching) for the DiT/CFM sampler.
Follows vipshop/cache-dit (`CachedBlocks_Pattern_Base.forward`), adapted to the way this repo
samples: the block stack is split into three segments, and the middle one is skipped when the
front residual barely moved.
front : trunk(x_in) -> blocks[0:Fn] -> x_F (every step)
R = x_F - trunk(x_in) Fn residual, the decision signal
fd = sum|R - R_prev| / (sum|R_prev| + eps) (relative -> scale free)
hit : x_mid = x_F + MnResidual_prev, then head only (norm_out -> proj_out)
miss : x_F -> blocks[Fn:DEPTH] -> x_M; MnResidual = x_M - x_F; R -> R_prev
head : x_mid -> norm_out -> proj_out -> vel (every step, hit included)
A hit additionally requires `can_cache`: not in warmup, `fd < threshold`, and the three gates
`max_cached` (cached steps per sampling run), `max_cont` (consecutive hits) and `max_accum`
(accumulated residual drift).
CFG. This repo runs the conditional and the unconditional branch as **two separate forward
passes** (`CFM.inference`), so each branch owns its own `R_prev`, `MnResidual`, hit/miss
counters and gate counters -- the equivalent of cache-dit's `enable_separate_cfg` plus
`cfg_diff_compute_separate`. Nothing is shared between the branches.
Everything here is inert unless a `DBCacheConfig` with `fn > 0` is handed to `DiT.forward`;
`cache_dit="off"` keeps the sampler on the identical code path it had before this change.
"""
from __future__ import annotations
import os
from dataclasses import dataclass, replace
import torch
__all__ = [
"DBCacheConfig",
"DBCacheState",
"resolve_fn",
"config_from_params",
"MODES",
]
MODES = ("off", "auto", "f4", "f8", "f16")
# Defaults of the cache-dit reference recipe, and the `auto` policy used by the HTTP API
# (F8 from 16 sampling steps on, off below that).
DEFAULT_FN = 8
DEFAULT_THRESHOLD = 0.02
DEFAULT_WARMUP = 8
DEFAULT_DOWNSAMPLE = 4
AUTO_FN = 8
AUTO_MIN_STEPS = 16
_EPS = 1e-6
def resolve_fn(mode, steps=32, depth=None):
"""`off|auto|f4|f8|f16` (or a plain integer) -> number of front blocks."""
if mode is None:
return 0
if isinstance(mode, bool):
mode = int(mode)
if isinstance(mode, (int, float)):
fn = int(mode)
else:
text = str(mode).strip().lower()
if text in ("", "off", "none", "0", "false", "no"):
fn = 0
elif text in ("on", "true", "yes"):
fn = DEFAULT_FN
elif text == "auto":
fn = AUTO_FN if int(steps) >= AUTO_MIN_STEPS else 0
elif text.startswith("f") and text[1:].isdigit():
fn = int(text[1:])
elif text.isdigit():
fn = int(text)
else:
raise ValueError("unknown cache_dit mode %r (expected off|auto|f4|f8|f16)" % (mode,))
if depth is not None:
fn = max(0, min(fn, int(depth) - 1))
return fn
@dataclass
class DBCacheConfig:
"""cache-dit knobs. `fn == 0` means off (the default)."""
mode: str = "off"
fn: int = 0
threshold: float = DEFAULT_THRESHOLD
warmup: int = DEFAULT_WARMUP
downsample: int = DEFAULT_DOWNSAMPLE
warmup_interval: int = 1
max_cached: int = -1
max_cont: int = -1
max_accum: float = 0.0
# fd is reduced over the whole sequence by default: the prompt/reference frames carry a
# nearly constant residual, so they dilute the ratio and threshold 0.02 stays usable. Set
# to True to reduce over the target frames only -- a ~3x stricter signal on this model, so
# thresholds are not transferable between the two conventions.
fd_ignore_prompt: bool = False
trace: bool = False
# expert override: allow cache-dit below 16 sampling steps (see docs in
# `DBCacheMixin.dbcache_config`: too few steps gain nothing and the quality risk is real)
force: bool = False
@property
def enabled(self) -> bool:
return self.fn > 0
def clamped(self, depth: int | None = None) -> "DBCacheConfig":
if depth is None:
return self
return replace(self, fn=max(0, min(int(self.fn), int(depth) - 1)))
def __str__(self) -> str: # for logs
if not self.enabled:
return "off"
return ("%s(fn=%d thr=%.3g warmup=%d ds=%d mc=%d mcont=%d macc=%.3g)"
% (self.mode, self.fn, self.threshold, self.warmup, self.downsample,
self.max_cached, self.max_cont, self.max_accum))
def _get(params, key, env, default, cast):
"""request dict first (API field names), then env, then default."""
if params is not None:
for k in (key,):
if k in params and params[k] is not None and params[k] != "":
try:
return cast(params[k])
except (TypeError, ValueError):
pass
val = os.environ.get(env)
if val is not None and val != "":
try:
return cast(val)
except (TypeError, ValueError):
pass
return default
def config_from_params(params=None, steps=32, depth=None) -> DBCacheConfig:
"""Build the config from a request dict / env vars.
Field names match the HTTP API of our C++ server:
cache_dit (alias dbcache) = off|auto|f4|f8|f16
dbcache_fn, dbcache_threshold, dbcache_warmup, dbcache_downsample
dbcache_warmup_interval, dbcache_max_cached, dbcache_max_cont, dbcache_max_accum
Env fallbacks: GSV_DBCACHE / GSV_DBCACHE_FN / _THRESHOLD / _WARMUP / _DOWNSAMPLE / ...
"""
p = dict(params or {})
mode = p.get("cache_dit", p.get("dbcache", os.environ.get("GSV_DBCACHE", "off")))
mode = "off" if mode is None else str(mode).strip().lower()
cfg = DBCacheConfig(mode=mode)
cfg.threshold = _get(p, "dbcache_threshold", "GSV_DBCACHE_THRESHOLD", cfg.threshold, float)
cfg.warmup = _get(p, "dbcache_warmup", "GSV_DBCACHE_WARMUP", cfg.warmup, int)
cfg.downsample = max(1, _get(p, "dbcache_downsample", "GSV_DBCACHE_DOWNSAMPLE", cfg.downsample, int))
cfg.warmup_interval = max(1, _get(p, "dbcache_warmup_interval", "GSV_DBCACHE_WARMUP_INTERVAL",
cfg.warmup_interval, int))
cfg.max_cached = _get(p, "dbcache_max_cached", "GSV_DBCACHE_MAX_CACHED", cfg.max_cached, int)
cfg.max_cont = _get(p, "dbcache_max_cont", "GSV_DBCACHE_MAX_CONT", cfg.max_cont, int)
cfg.max_accum = _get(p, "dbcache_max_accum", "GSV_DBCACHE_MAX_ACCUM", cfg.max_accum, float)
cfg.fd_ignore_prompt = bool(_get(p, "dbcache_fd_ignore_prompt", "GSV_DBCACHE_FD_IGNORE_PROMPT",
int(cfg.fd_ignore_prompt), int))
cfg.trace = bool(_get(p, "dbcache_trace", "GSV_DBCACHE_TRACE", 0, int))
cfg.force = bool(_get(p, "dbcache_force", "GSV_DBCACHE_FORCE", 0, int))
# an explicit fn wins over the mode
if p.get("dbcache_fn") not in (None, ""):
cfg.fn = int(p["dbcache_fn"])
cfg.mode = "custom"
elif os.environ.get("GSV_DBCACHE_FN"):
cfg.fn = int(os.environ["GSV_DBCACHE_FN"])
cfg.mode = "custom"
else:
cfg.fn = resolve_fn(mode, steps)
if depth is not None:
cfg.fn = max(0, min(cfg.fn, int(depth) - 1))
return cfg
class DBCacheMixin:
"""The CFM-side plumbing (`set_dbcache` / per-run state / hit counters).
Mixed into `module.models.CFM` and `module.v5_inference.CFMV5`; both expose the DiT as
`self.estimator`. Defaults are class attributes, so `cache off` needs no `__init__` change
and the object layout of an existing checkpoint-compatible model is untouched.
"""
dbcache_spec = None # DBCacheConfig or request dict of dbcache_* fields
last_dbcache_stats = None # stats of the most recent inference() call
_dbcache_cum = None # accumulated over all calls (reset by dbcache_stats)
# ------------------------------------------------------------------ config
@property
def cache_depth(self):
return int(getattr(self.estimator, "depth", 0) or 0)
def set_dbcache(self, spec=None, **kw):
"""Enable/disable cache-dit for the next sampling runs.
`spec` is either a `module.cache_dit.DBCacheConfig` or a dict using the C++ server's
field names (`cache_dit`/`dbcache` + `dbcache_fn|threshold|warmup|downsample|...`).
Nothing is applied retroactively: `inference()` resolves it per call.
"""
if isinstance(spec, DBCacheConfig):
self.dbcache_spec = spec
else:
merged = dict(spec or {})
merged.update(kw)
cfg = merged.pop("config", None)
self.dbcache_spec = cfg if isinstance(cfg, DBCacheConfig) else merged
return self.dbcache_spec
def dbcache_config(self, steps):
"""Resolve the spec against the step count of the run that is about to start.
With no explicit spec the env switch (`GSV_DBCACHE=off|auto|f4|f8|f16`) is honoured,
so every entry point (WebUI, api.py, api_v2.py, CLI, Colab) can enable cache-dit
without extra plumbing. Below 16 sampling steps (the 4-step v5turbo) it stays off:
the measured gain is negative and the quality is unstable there.
"""
depth = self.cache_depth or None
spec = self.dbcache_spec
if isinstance(spec, DBCacheConfig):
fn = spec.fn if spec.mode in ("custom", None) else resolve_fn(spec.mode, steps, depth)
cfg = DBCacheConfig(**{**spec.__dict__,
"fn": fn if depth is None else min(fn, depth - 1)})
else:
cfg = config_from_params(spec, steps, depth)
if cfg.enabled and int(steps) < AUTO_MIN_STEPS and not cfg.force:
cfg.fn = 0
return cfg
# ------------------------------------------------------------------ per run
def _dbcache_begin(self, steps, seq_len, prompt_len, x_lens, device):
"""Fresh state per sampling run (new request / new chunk)."""
cfg = self.dbcache_config(steps)
if not cfg.enabled:
self.last_dbcache_stats = None
return None
return DBCacheState(cfg, self.cache_depth).begin(seq_len, prompt_len, x_lens, device)
def _dbcache_end(self, state):
if state is None:
return
st = state.stats()
self.last_dbcache_stats = st
cum = self._dbcache_cum
if cum is None:
cum = self._dbcache_cum = {"calls": 0, "steps": 0, "hits_pos": 0, "hits_neg": 0,
"misses_pos": 0, "misses_neg": 0}
cum["calls"] += 1
for k in ("steps", "hits_pos", "hits_neg", "misses_pos", "misses_neg"):
cum[k] = cum.get(k, 0) + st[k]
cum["hits"] = cum["hits_pos"] + cum["hits_neg"]
cum["misses"] = cum["misses_pos"] + cum["misses_neg"]
cum["total"] = cum["hits"] + cum["misses"]
cum["hit_rate"] = cum["hits"] / cum["total"] if cum["total"] else 0.0
cum["fn"] = st.get("fn")
cum["threshold"] = st.get("threshold")
cum["last"] = st
def dbcache_stats(self, reset=True):
"""Hit/miss counters accumulated since the last reset (None when cache-dit is off)."""
st = self._dbcache_cum
if reset:
self._dbcache_cum = None
return st
class _Branch:
"""Per-CFG-branch bookkeeping."""
__slots__ = ("valid", "r_prev", "r_cur", "m_res", "hits", "misses",
"cached_steps", "cont", "acc", "last_fd")
def __init__(self):
self.valid = False
self.r_prev = None
self.r_cur = None
self.m_res = None
self.hits = 0
self.misses = 0
self.cached_steps = 0
self.cont = 0
self.acc = 0.0
self.last_fd = 0.0
def reset(self):
self.__init__()
class DBCacheState:
"""State of one sampling run (`CFM.inference` = one run = one new request / chunk)."""
def __init__(self, config: DBCacheConfig, depth: int, trace: bool | None = None):
self.cfg = config
self.depth = int(depth)
self.trace = config.trace if trace is None else trace
self.step = 0
self.n_steps = 0
self.branches = [_Branch(), _Branch()]
self.ds = 1
self.note = "" # free-form tag for logs
# ------------------------------------------------------------------ lifecycle
def begin(self, seq_len: int, prompt_len: int, x_lens=None, device=None):
"""Reset for a new sampling run. Called once per `CFM.inference`."""
for b in self.branches:
b.reset()
self.step = 0
self.n_steps = 0
ds = int(self.cfg.downsample) if self.cfg.downsample else 1
self.ds = ds if ds >= 1 and seq_len % ds == 0 else 1
self._build_frame_mask(seq_len, prompt_len, x_lens, device)
return self
def _build_frame_mask(self, seq_len, prompt_len, x_lens, device):
"""Frames used for fd: skip the prompt/reference rows and the padding."""
self.mask = None
if not self.cfg.fd_ignore_prompt:
return
idx = torch.arange(seq_len, device=device)
mask = idx.unsqueeze(0) >= int(max(0, prompt_len))
if x_lens is not None:
lens = torch.as_tensor(x_lens, device=device).reshape(-1, 1)
if lens.shape[0] == 1 and seq_len >= int(lens.max()):
mask = mask & (idx.unsqueeze(0) < lens)
mask = mask.to(torch.float32)
if self.ds > 1:
mask = mask[:, :: self.ds]
mask = mask.unsqueeze(-1) # (b, n/ds, 1)
if float(mask.sum()) <= 0.0: # degenerate (all-prompt) -> fall back to everything
mask = None
self.mask = mask
def step_begin(self, step: int):
"""`db_set_step(j)`: current diffusion step, drives warmup and the trace."""
self.step = int(step)
self.n_steps += 1
def in_warmup(self) -> bool:
if self.step >= self.cfg.warmup:
return False
return (self.step % max(1, self.cfg.warmup_interval)) == 0
# ------------------------------------------------------------------ per-step ops
@property
def enabled(self) -> bool:
return self.cfg.enabled
@property
def fn(self) -> int:
return self.cfg.fn
def branch(self, index: int) -> _Branch:
return self.branches[0 if index == 0 else 1]
def front_update(self, index: int, x_f, trunk) -> float:
"""Store x_F / R, update R_prev-based similarity, return `fd` (one host sync)."""
br = self.branch(index)
r = x_f.float() - trunk.float() # Fn residual; fp32 keeps fd stable
if self.ds > 1:
r = r[:, :: self.ds, :]
r_prev = br.r_prev
if r_prev is None or r_prev.shape != r.shape:
r_prev = torch.zeros_like(r)
num = (r - r_prev).abs()
den = r_prev.abs()
if self.mask is not None and self.mask.shape[0] == num.shape[0] \
and self.mask.shape[1] == num.shape[1]:
num = num * self.mask
den = den * self.mask
num = num.sum(dim=(1, 2))
den = den.sum(dim=(1, 2)) + _EPS
fd = float((num / den).max().item()) # the only device->host scalar per step
br.r_cur = r
br.last_fd = fd
return fd
def can_cache(self, index: int, fd: float) -> bool:
br = self.branch(index)
cfg = self.cfg
hit = False
if br.valid and cfg.threshold > 0.0 and not self.in_warmup():
ok = fd < cfg.threshold
if ok and cfg.max_cached >= 0 and br.cached_steps >= cfg.max_cached:
ok = False
if ok and cfg.max_cont >= 0 and br.cont >= cfg.max_cont:
ok = False
br.cont = 0
if ok and cfg.max_accum > 0.0 and br.acc >= cfg.max_accum:
ok = False
hit = ok
if fd > 0.0:
br.acc += fd
return hit
def mid_residual(self, index: int):
return self.branch(index).m_res
def on_hit(self, index: int):
br = self.branch(index)
br.hits += 1
br.cached_steps += 1
br.cont += 1
def on_miss(self, index: int, x_m, x_f):
br = self.branch(index)
br.misses += 1
br.valid = True
br.cont = 0
br.m_res = (x_m - x_f)
if br.r_cur is not None:
if br.r_prev is None or br.r_prev.shape != br.r_cur.shape:
br.r_prev = br.r_cur.clone()
else:
br.r_prev.copy_(br.r_cur)
# ------------------------------------------------------------------ reporting
def stats(self) -> dict:
out = {"steps": self.n_steps, "last_step": self.step, "fn": self.cfg.fn,
"threshold": self.cfg.threshold, "warmup": self.cfg.warmup,
"downsample": self.ds, "mode": self.cfg.mode}
for i, name in enumerate(("pos", "neg")):
br = self.branches[i]
out["hits_" + name] = br.hits
out["misses_" + name] = br.misses
out["fd_" + name] = br.last_fd
out["acc_" + name] = br.acc
out["hits"] = out["hits_pos"] + out["hits_neg"]
out["misses"] = out["misses_pos"] + out["misses_neg"]
total = out["hits"] + out["misses"]
out["total"] = total
out["hit_rate"] = (out["hits"] / total) if total else 0.0
return out

View File

@ -10,6 +10,7 @@ from torch.nn import functional as F
from module import commons
from module import modules
from module import attentions
from module.cache_dit import DBCacheMixin
from f5_tts.model import DiT
from torch.nn import Conv1d, ConvTranspose1d, Conv2d
from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
@ -1099,7 +1100,7 @@ class SynthesizerTrn(nn.Module):
return codes.transpose(0, 1)
class CFM(torch.nn.Module):
class CFM(DBCacheMixin, torch.nn.Module):
def __init__(self, in_channels, dit):
super().__init__()
self.sigma_min = 1e-6
@ -1138,7 +1139,11 @@ class CFM(torch.nn.Module):
if self.use_conditioner_cache and self.use_static_cache else None)
d_tensor = (torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * d
if self.use_step_embedding else None)
# cache-dit: one fresh state per sampling run; None when the switch is off
dbcache = self._dbcache_begin(n_timesteps, T, prompt_len, x_lens, x.device)
for j in range(n_timesteps):
if dbcache is not None:
dbcache.step_begin(j)
t_tensor = torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * (t if self.use_step_embedding else j * d)
step_condition = {"dt_base_bootstrap": d_tensor, "dt_cache": dt_cache} if self.use_step_embedding else {}
v_pred, text_emb, dt = self.estimator(
@ -1153,6 +1158,8 @@ class CFM(torch.nn.Module):
infer=True,
text_cache=text_cache,
static_cache=static_cache,
dbcache=dbcache,
dbcache_branch=0,
**step_condition,
)
v_pred = v_pred.transpose(2, 1)
@ -1174,6 +1181,8 @@ class CFM(torch.nn.Module):
infer=True,
text_cache=text_cfg_cache if self.cfg_drop_text else text_cache,
static_cache=static_cache,
dbcache=dbcache,
dbcache_branch=1,
**step_condition,
)
neg = neg.transpose(2, 1)
@ -1183,6 +1192,7 @@ class CFM(torch.nn.Module):
x = x + d * v_pred
t = t + d
x[:, :, :prompt_len] = 0
self._dbcache_end(dbcache)
return x
def forward(self, x1, x_lens, prompt_lens, mu, use_grad_ckpt):

View File

@ -1,133 +0,0 @@
import math
import os
import torch
from torch import nn
V5_TEMPERATURE = 0.875
V5_REFERENCE_ANCHOR = 1.0
V5_ROLLING_TAIL_FRAMES = 32
V5_MAX_TARGET_CHUNK_FRAMES = 640
V5_REFERENCE_FRAMES = 500
V5_TOTAL_CHUNK_FRAMES = 1000
V5_DEFAULT_CFG = 1.30
V5_REQUEST_DEFAULT_CFG = 0.0
V5_VERSIONS = {"v5", "v5dev", "v5turbo"}
def sampling_defaults(version, legacy_steps=32):
if version == "v5turbo":
return 4, 0.0
if version in V5_VERSIONS:
return 32, 1.30
return legacy_steps, 0.0
def resolve_sampling(version, sample_steps=None, cfg_rate=None, legacy_steps=32):
steps, cfg = sampling_defaults(version, legacy_steps)
return (steps if sample_steps is None else int(sample_steps),
cfg if cfg_rate is None else float(cfg_rate))
def validate_v5_sampling(sample_steps, cfg_rate):
steps = int(sample_steps)
cfg = float(cfg_rate)
if steps not in (4, 8, 16, 32):
raise ValueError("Euler steps must be one of 4, 8, 16, 32")
if not math.isfinite(cfg) or cfg < 0 or cfg > 2:
raise ValueError("CFG must be a finite value between 0 and 2")
return steps, cfg
class CFMV5(nn.Module):
def __init__(self, in_channels, dit):
super().__init__()
self.in_channels = in_channels
self.estimator = dit
self.use_conditioner_cache = True
self.use_static_cache = os.environ.get(
"GSVV5_CFM_STATIC_CACHE", os.environ.get("GSVV4_CFM_STATIC_CACHE", "1")
) != "0"
@torch.inference_mode()
def inference(self, mu, x_lens, prompt, n_timesteps, temperature=V5_TEMPERATURE,
inference_cfg_rate=V5_DEFAULT_CFG):
steps, cfg = validate_v5_sampling(n_timesteps, inference_cfg_rate)
batch, frames = mu.shape[:2]
prompt_len = prompt.shape[-1]
# Match v4: noise, time and model inputs follow mu/model dtype.
state_dtype = mu.dtype
model_dtype = mu.dtype
x = torch.randn(batch, self.in_channels, frames, device=mu.device, dtype=state_dtype) * V5_TEMPERATURE
x[..., :prompt_len] = 0
prompt_x = torch.zeros_like(x, dtype=model_dtype)
prompt_x[..., :prompt_len] = prompt.to(model_dtype)
condition = mu.transpose(2, 1)
cache_enabled = bool(self.use_conditioner_cache and self.use_static_cache)
cache = (self.estimator.prepare_static_cache(prompt_x, x_lens, condition)
if cache_enabled else None)
text_cache = None
step = 1.0 / steps
for index in range(steps):
time = torch.full((batch,), index * step, device=mu.device, dtype=state_dtype)
velocity, text_embedding, _ = self.estimator(
x, prompt_x, x_lens, time, None, condition,
drop_audio_cond=False, drop_text=False, static_cache=cache,
infer=True, text_cache=text_cache,
)
if self.use_conditioner_cache and cache is None:
text_cache = text_embedding
velocity = velocity.transpose(2, 1)
if cfg > 1e-5:
negative, _, _ = self.estimator(
x, prompt_x, x_lens, time, None, condition,
drop_audio_cond=True, drop_text=False, static_cache=cache,
infer=True, text_cache=text_cache,
)
negative = negative.transpose(2, 1)
velocity = velocity + cfg * (velocity - negative)
x = x + step * velocity
x[..., :prompt_len] = 0
return x
@torch.inference_mode()
def synthesize_v5_mel(model, reference_features, target_features, reference_mel,
sample_steps=None, cfg_rate=None):
sample_steps, cfg_rate = resolve_sampling(model.version, sample_steps, cfg_rate)
steps, cfg = validate_v5_sampling(sample_steps, cfg_rate)
reference_frames = min(reference_mel.shape[-1], reference_features.shape[-1])
if reference_frames < 1:
raise ValueError("V5 requires nonempty reference mel and semantic features")
reference_mel = reference_mel[..., :reference_frames][..., -V5_REFERENCE_FRAMES:]
reference_features = reference_features[..., :reference_frames][..., -V5_REFERENCE_FRAMES:]
reference_frames = reference_mel.shape[-1]
chunk_frames = min(V5_TOTAL_CHUNK_FRAMES - reference_frames, V5_MAX_TARGET_CHUNK_FRAMES)
original_mel = reference_mel
original_features = reference_features
rolling_mel = reference_mel
rolling_features = reference_features
results = []
for index, start in enumerate(range(0, target_features.shape[-1], chunk_frames)):
target = target_features[..., start:start + chunk_frames]
if index == 0:
prompt = original_mel
features = original_features
else:
tail = min(V5_ROLLING_TAIL_FRAMES, reference_frames,
rolling_mel.shape[-1], rolling_features.shape[-1])
prefix = reference_frames - tail
prompt = torch.cat((original_mel[..., :prefix], rolling_mel[..., -tail:]), dim=-1)
features = torch.cat((original_features[..., :prefix], rolling_features[..., -tail:]), dim=-1)
mu = torch.cat((features, target), dim=-1).transpose(2, 1)
lengths = torch.full((mu.shape[0],), mu.shape[1], dtype=torch.long, device=mu.device)
generated = model.cfm.inference(
mu, lengths, prompt, steps, inference_cfg_rate=cfg,
)[..., prompt.shape[-1]:]
results.append(generated)
rolling_mel = generated[..., -reference_frames:].to(reference_mel.dtype)
rolling_features = target[..., -reference_frames:]
if not results:
raise ValueError("V5 requires nonempty target semantic features")
return torch.cat(results, dim=-1)

58
api.py
View File

@ -829,6 +829,28 @@ splits = {
}
def apply_cache_dit(vq_model, params: dict, sample_steps: int):
"""cache-dit (DBCache) switch for the DiT sampler; off unless configured.
Field names match api_v2.py:
``cache_dit`` = off|auto|f4|f8|f16, ``dbcache_fn``, ``dbcache_threshold``,
``dbcache_warmup``, ``dbcache_downsample``. The ``GSV_DBCACHE*`` env vars work too,
also on the paths that never go through these request fields.
"""
cfm = getattr(vq_model, "cfm", None)
if cfm is None or not hasattr(cfm, "set_dbcache"):
return None
from module.cache_dit import config_from_params
cfg = config_from_params(params, int(sample_steps), getattr(cfm, "cache_depth", 0) or None)
if cfg.enabled and str(getattr(vq_model, "version", "")) == "v5turbo" and not cfg.force:
cfg.fn = 0 # 4-step turbo: no gain and unstable quality, keep it off
cfm.set_dbcache(cfg)
if cfg.enabled:
print("cache-dit:", cfg)
return cfg
def get_tts_wav(
ref_wav_path,
prompt_text,
@ -844,6 +866,11 @@ def get_tts_wav(
if_sr=False,
spk="default",
cfg_rate=None,
cache_dit=None,
dbcache_fn=None,
dbcache_threshold=None,
dbcache_warmup=None,
dbcache_downsample=None,
):
infer_sovits = speaker_list[spk].sovits
vq_model = infer_sovits.vq_model
@ -867,6 +894,12 @@ def get_tts_wav(
if sample_steps not in [4, 8, 16, 32]:
sample_steps = 8
# cache-dit (DBCache) for the DiT sampler: off unless asked for
apply_cache_dit(vq_model, dict(
cache_dit=cache_dit, dbcache_fn=dbcache_fn, dbcache_threshold=dbcache_threshold,
dbcache_warmup=dbcache_warmup, dbcache_downsample=dbcache_downsample,
), sample_steps)
if if_sr and version != "v3":
if_sr = False
@ -1126,6 +1159,11 @@ def handle(
sample_steps,
if_sr,
cfg_rate=None,
cache_dit=None,
dbcache_fn=None,
dbcache_threshold=None,
dbcache_warmup=None,
dbcache_downsample=None,
):
if (
refer_wav_path == ""
@ -1163,6 +1201,11 @@ def handle(
sample_steps,
if_sr,
cfg_rate=cfg_rate,
cache_dit=cache_dit,
dbcache_fn=dbcache_fn,
dbcache_threshold=dbcache_threshold,
dbcache_warmup=dbcache_warmup,
dbcache_downsample=dbcache_downsample,
),
media_type="audio/" + media_type,
)
@ -1372,6 +1415,11 @@ async def tts_endpoint(request: Request):
json_post_raw.get("sample_steps"),
json_post_raw.get("if_sr", False),
cfg_rate=json_post_raw.get("cfg_rate"),
cache_dit=json_post_raw.get("cache_dit"),
dbcache_fn=json_post_raw.get("dbcache_fn"),
dbcache_threshold=json_post_raw.get("dbcache_threshold"),
dbcache_warmup=json_post_raw.get("dbcache_warmup"),
dbcache_downsample=json_post_raw.get("dbcache_downsample"),
)
@ -1391,6 +1439,11 @@ async def tts_endpoint(
sample_steps: int = None,
if_sr: bool = False,
cfg_rate: float = None,
cache_dit: str = None,
dbcache_fn: int = None,
dbcache_threshold: float = None,
dbcache_warmup: int = None,
dbcache_downsample: int = None,
):
return handle(
refer_wav_path,
@ -1407,6 +1460,11 @@ async def tts_endpoint(
sample_steps,
if_sr,
cfg_rate=cfg_rate,
cache_dit=cache_dit,
dbcache_fn=dbcache_fn,
dbcache_threshold=dbcache_threshold,
dbcache_warmup=dbcache_warmup,
dbcache_downsample=dbcache_downsample,
)

View File

@ -44,6 +44,11 @@ POST:
"streaming_mode": False, # bool or int. return audio chunk by chunk.T he available options are: 0,1,2,3 or True/False (0/False: Disabled | 1/True: Best Quality, Slowest response speed (old version streaming_mode) | 2: Medium Quality, Slow response speed | 3: Lower Quality, Faster response speed )
"overlap_length": 2, # int. overlap length of semantic tokens for streaming mode.
"min_chunk_length": 16, # int. The minimum chunk length of semantic tokens for streaming mode. (affects audio chunk size)
"cache_dit": "off", # str.(optional) off|auto|f4|f8|f16. cache-dit/DBCache for the V3/V4/V5 DiT: fewer DiT forwards, near-identical audio. Off by default; 4-step runs are refused.
"dbcache_fn": 8, # int.(optional) front blocks (default 8; auto = F8 when sample_steps >= 16).
"dbcache_threshold": 0.02, # float.(optional) relative residual-diff threshold (default 0.02).
"dbcache_warmup": 8, # int.(optional) forced full-compute steps at the start (default 8).
"dbcache_downsample": 4, # int.(optional) time downsample of the cached Fn residual (default 4).
"gpt_streaming": false, # bool. opt in to semantic-token streaming.
"streaming_chunk_seconds": 2.0,
"streaming_window_mode": "fixed",
@ -201,6 +206,13 @@ class TTS_Request(BaseModel):
repetition_penalty: float = 1.35
sample_steps: Optional[int] = None
cfg_rate: Optional[float] = None
# cache-dit (DBCache) for the V3/V4/V5 DiT sampler; off by default. Field names match
# the ones the TTS pipeline and the CLI use.
cache_dit: str = "off" # off | auto | f4 | f8 | f16
dbcache_fn: Optional[int] = None # front blocks (overrides cache_dit)
dbcache_threshold: Optional[float] = None
dbcache_warmup: Optional[int] = None
dbcache_downsample: Optional[int] = None
super_sampling: bool = False
overlap_length: int = 2
min_chunk_length: int = 16
@ -613,6 +625,11 @@ async def tts_get_endpoint(
repetition_penalty: float = 1.35,
sample_steps: Optional[int] = None,
cfg_rate: Optional[float] = None,
cache_dit: str = "off",
dbcache_fn: Optional[int] = None,
dbcache_threshold: Optional[float] = None,
dbcache_warmup: Optional[int] = None,
dbcache_downsample: Optional[int] = None,
super_sampling: bool = False,
streaming_mode: Union[bool, int] = False,
overlap_length: int = 2,
@ -649,6 +666,11 @@ async def tts_get_endpoint(
"repetition_penalty": float(repetition_penalty),
"sample_steps": None if sample_steps is None else int(sample_steps),
"cfg_rate": None if cfg_rate is None else float(cfg_rate),
"cache_dit": cache_dit,
"dbcache_fn": dbcache_fn,
"dbcache_threshold": dbcache_threshold,
"dbcache_warmup": dbcache_warmup,
"dbcache_downsample": dbcache_downsample,
"super_sampling": super_sampling,
"overlap_length": int(overlap_length),
"min_chunk_length": int(min_chunk_length),

View File

@ -10,9 +10,20 @@ def get_models():
asr_dict = {
"Fun-ASR-Nano (31语种+方言, 推荐)": {"lang": ["zh", "en", "ja", "ko", "yue", "auto"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
"SenseVoice (极速, 5语种)": {"lang": ["zh", "en", "ja", "ko", "yue", "auto"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
"达摩 ASR (中文经典)": {"lang": ["zh", "yue"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
"达摩 ASR (中文经典)": {
"lang": ["zh"],
"size": ["large"],
"path": "funasr_asr.py",
"backend": "damo",
"precision": ["float32"],
},
"Fun-ASR-Nano(速度更慢,更准)": {
"lang": ["zh", "en", "ja", "ko", "yue", "auto"],
"size": ["large"],
"path": "funasr_asr.py",
"backend": "fun-asr-nano",
"precision": ["float32"],
},
"Faster Whisper (多语种)": {
"lang": ["auto", "en", "ja", "ko"],
"size": get_models(),

View File

@ -40,6 +40,26 @@ language_code_list = [
def download_model(model_size: str):
model_path = ""
if "distil" in model_size:
if "3.5" in model_size:
repo_id = "distil-whisper/distil-large-v3.5-ct2"
model_path = "tools/asr/models/faster-distil-whisper-large-v3.5"
else:
repo_id = "Systran/faster-{}-whisper-{}".format(*model_size.split("-", maxsplit=1))
elif model_size == "large-v3-turbo":
repo_id = "mobiuslabsgmbh/faster-whisper-large-v3-turbo"
model_path = "tools/asr/models/faster-whisper-large-v3-turbo"
else:
repo_id = f"Systran/faster-whisper-{model_size}"
model_path = model_path or f"tools/asr/models/{repo_id.replace('Systran/', '').replace('distil-whisper/', '', 1)}"
if os.path.isfile(os.path.join(model_path, "config.json")) and os.path.isfile(
os.path.join(model_path, "model.bin")
):
print(f"Using local Faster Whisper model: {model_path}")
return model_path
url = "https://huggingface.co/api/models/gpt2"
try:
requests.get(url, timeout=3)
@ -47,26 +67,6 @@ def download_model(model_size: str):
except Exception:
source = "ModelScope"
model_path = ""
if source == "HF":
if "distil" in model_size:
if "3.5" in model_size:
repo_id = "distil-whisper/distil-large-v3.5-ct2"
model_path = "tools/asr/models/faster-distil-whisper-large-v3.5"
else:
repo_id = "Systran/faster-{}-whisper-{}".format(*model_size.split("-", maxsplit=1))
elif model_size == "large-v3-turbo":
repo_id = "mobiuslabsgmbh/faster-whisper-large-v3-turbo"
model_path = "tools/asr/models/faster-whisper-large-v3-turbo"
else:
repo_id = f"Systran/faster-whisper-{model_size}"
model_path = (
model_path or f"tools/asr/models/{repo_id.replace('Systran/', '').replace('distil-whisper/', '', 1)}"
)
else:
repo_id = "XXXXRT/faster-whisper"
model_path = "tools/asr/models"
files: list[str] = [
"config.json",
"model.bin",
@ -76,12 +76,8 @@ def download_model(model_size: str):
if "large-v3" in model_size or "distil" in model_size:
files.append("preprocessor_config.json")
files.append("vocabulary.json")
files.remove("vocabulary.txt")
if source == "ModelScope":
files = [f"faster-whisper-{model_size}/{file}".replace("whisper-distil", "distil-whisper") for file in files]
if source == "HF":
print(f"Downloading model from HuggingFace: {repo_id} to {model_path}")
snapshot_download_hf(
@ -91,13 +87,14 @@ def download_model(model_size: str):
allow_patterns=files,
)
else:
print(f"Downloading model from ModelScope: {repo_id} to {model_path}")
repo_id = "XXXXRT/faster-whisper"
files = [f"faster-whisper-{model_size}/{file}".replace("whisper-distil", "distil-whisper") for file in files]
print(f"Downloading model from ModelScope: {repo_id} to tools/asr/models")
snapshot_download_ms(
repo_id,
local_dir=model_path,
local_dir="tools/asr/models",
allow_patterns=files,
)
return model_path + f"/faster-whisper-{model_size}".replace("whisper-distil", "distil-whisper")
return model_path

View File

@ -2,18 +2,45 @@
import argparse
import os
import sys
import traceback
import setuptools._distutils as _distutils
import setuptools._distutils.util as _distutils_util
import setuptools._distutils.version as _distutils_version
sys.modules.setdefault("distutils", _distutils)
sys.modules.setdefault("distutils.util", _distutils_util)
sys.modules.setdefault("distutils.version", _distutils_version)
from funasr import AutoModel
from modelscope import snapshot_download
from tqdm import tqdm
funasr_models = {} # 存储模型避免重复加载
ASR_MODEL_ROOT = "tools/asr/models"
FUN_ASR_NANO_MODEL_ID = "FunAudioLLM/Fun-ASR-Nano-2512"
FUN_ASR_NANO_MODEL_SOURCES = (
{"model": FUN_ASR_NANO_MODEL_ID, "hub": "hf", "trust_remote_code": True},
{"model": FUN_ASR_NANO_MODEL_ID, "hub": "ms", "trust_remote_code": False},
)
VAD_MODEL_ID = "iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"
PUNC_MODEL_ID = "iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
PARAFORMER_MODEL_ID = "iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
UNIASR_MODEL_ID = "iic/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
FUN_ASR_NANO_MODEL_PATH = f"{ASR_MODEL_ROOT}/Fun-ASR-Nano-2512"
VAD_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_fsmn_vad_zh-cn-16k-common-pytorch"
PUNC_MODEL_PATH = f"{ASR_MODEL_ROOT}/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
PARAFORMER_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
UNIASR_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
def ensure_model_dir(model_id, model_path, revision=None):
if os.path.isfile(os.path.join(model_path, "model.pt")):
return model_path
download_kwargs = {"local_dir": model_path}
if revision is not None:
download_kwargs["revision"] = revision
print(f"Downloading FunASR model: {model_id} -> {model_path}")
snapshot_download(model_id, **download_kwargs)
return model_path
def only_asr(input_file, language, backend="fun-asr-nano"):
@ -26,100 +53,86 @@ def only_asr(input_file, language, backend="fun-asr-nano"):
return text
def is_unregistered_model_error(exc):
message = str(exc).lower()
return "model" in message and "not registered" in message
def disable_flash_attention_probe():
import transformers.utils
import transformers.utils.import_utils
transformers.utils.import_utils.is_flash_attn_2_available = lambda: False
transformers.utils.is_flash_attn_2_available = lambda: False
def create_fun_asr_nano_model(device):
common_kwargs = dict(
vad_model="fsmn-vad",
disable_flash_attention_probe()
ensure_model_dir(FUN_ASR_NANO_MODEL_ID, FUN_ASR_NANO_MODEL_PATH)
ensure_model_dir(VAD_MODEL_ID, VAD_MODEL_PATH)
return AutoModel(
model=FUN_ASR_NANO_MODEL_PATH,
vad_model=VAD_MODEL_PATH,
device=device,
disable_update=True,
check_latest=False,
vad_kwargs={"check_latest": False},
hub="ms",
)
last_error = None
for model_source in FUN_ASR_NANO_MODEL_SOURCES:
try:
return AutoModel(**model_source, **common_kwargs)
except (KeyError, RuntimeError, ValueError) as exc:
if not is_unregistered_model_error(exc):
raise
last_error = exc
raise last_error
def create_damo_model(device):
ensure_model_dir(VAD_MODEL_ID, VAD_MODEL_PATH, revision="v2.0.4")
ensure_model_dir(PUNC_MODEL_ID, PUNC_MODEL_PATH, revision="v2.0.4")
ensure_model_dir(PARAFORMER_MODEL_ID, PARAFORMER_MODEL_PATH, revision="v2.0.4")
return AutoModel(
model=PARAFORMER_MODEL_PATH,
model_revision="v2.0.4",
vad_model=VAD_MODEL_PATH,
vad_model_revision="v2.0.4",
vad_kwargs={"check_latest": False},
punc_model=PUNC_MODEL_PATH,
punc_model_revision="v2.0.4",
punc_kwargs={"check_latest": False},
device=device,
disable_update=True,
check_latest=False,
hub="ms",
)
def create_uniasr_model(device):
ensure_model_dir(UNIASR_MODEL_ID, UNIASR_MODEL_PATH, revision="master")
return AutoModel(
model=UNIASR_MODEL_PATH,
model_revision="master",
device=device,
disable_update=True,
check_latest=False,
hub="ms",
)
def create_model(language="zh", **kwargs):
backend = kwargs.get("backend", "fun-asr-nano")
import torch
# For non-classic backends, route to multilingual models regardless of language
if backend in ("fun-asr-nano", "sensevoice") and language != "yue":
import torch
device = "cuda" if torch.cuda.is_available() else "cpu"
cache_key = f"{language}_{backend}"
if cache_key in funasr_models:
return funasr_models[cache_key]
device = "cuda" if torch.cuda.is_available() else "cpu"
cache_key = f"{backend}_{language}"
if cache_key in funasr_models:
return funasr_models[cache_key]
if backend == "fun-asr-nano":
model = create_fun_asr_nano_model(device)
print(f"FunASR Fun-ASR-Nano 模型加载完成: {language.upper()}")
else:
model = AutoModel(
model="iic/SenseVoiceSmall",
vad_model="fsmn-vad",
device=device,
disable_update=True,
)
print(f"FunASR SenseVoice 模型加载完成: {language.upper()}")
funasr_models[cache_key] = model
return model
if language == "zh":
path_vad = "tools/asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch"
path_punc = "tools/asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
path_asr = "tools/asr/models/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
snapshot_download(
"iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
local_dir="tools/asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch",
)
snapshot_download(
"iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
local_dir="tools/asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
)
snapshot_download(
"iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
local_dir="tools/asr/models/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
)
model_revision = "v2.0.4"
vad_model_revision = punc_model_revision = "v2.0.4"
elif language == "yue":
path_asr = "tools/asr/models/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
snapshot_download(
"iic/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
local_dir="tools/asr/models/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
)
path_vad = path_punc = None
vad_model_revision = punc_model_revision = ""
model_revision = "master"
if backend == "damo":
if language != "zh":
raise ValueError("达摩 ASR 仅支持 zh")
model = create_damo_model(device)
print(f"FunASR 达摩 Paraformer 模型加载完成: {language.upper()}")
elif backend == "fun-asr-nano" and language == "yue":
model = create_uniasr_model(device)
print(f"FunASR UniASR 粤语模型加载完成: {language.upper()}")
elif backend == "fun-asr-nano":
model = create_fun_asr_nano_model(device)
print(f"FunASR Fun-ASR-Nano 模型加载完成: {language.upper()}")
else:
raise ValueError(f"{language} is not supported. Supported: zh, yue, ja, en, ko, auto")
raise ValueError(f"{backend} is not supported")
if language in funasr_models:
return funasr_models[language]
else:
model = AutoModel(
model=path_asr,
model_revision=model_revision,
vad_model=path_vad,
vad_model_revision=vad_model_revision,
punc_model=path_punc,
punc_model_revision=punc_model_revision,
)
print(f"FunASR 模型加载完成: {language.upper()}")
funasr_models[language] = model
return model
funasr_models[cache_key] = model
return model
def execute_asr(input_folder, output_folder, model_size, language, backend="fun-asr-nano"):
@ -163,10 +176,14 @@ if __name__ == "__main__":
parser.add_argument(
"-p", "--precision", type=str, default="float16", choices=["float16", "float32"], help="fp16 or fp32"
) # 还没接入
parser.add_argument(
"--backend", type=str, default="fun-asr-nano", choices=["fun-asr-nano", "damo"], help="ASR backend"
)
cmd = parser.parse_args()
execute_asr(
input_folder=cmd.input_folder,
output_folder=cmd.output_folder,
model_size=cmd.model_size,
language=cmd.language,
backend=cmd.backend,
)

View File

@ -381,6 +381,8 @@ def open_asr(asr_inp_dir, asr_opt_dir, asr_model, asr_model_size, asr_lang, asr_
cmd += f" -s {asr_model_size}"
cmd += f" -l {asr_lang}"
cmd += f" -p {asr_precision}"
if "backend" in asr_dict[asr_model]:
cmd += f" --backend {asr_dict[asr_model]['backend']}"
output_file_name = os.path.basename(asr_inp_dir)
output_folder = asr_opt_dir or "output/asr_opt"
output_file_path = os.path.abspath(f"{output_folder}/{output_file_name}.list")
@ -1412,7 +1414,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
label=i18n("ASR 模型"),
choices=list(asr_dict.keys()),
interactive=True,
value="达摩 ASR (中文)",
value="达摩 ASR (中文经典)",
)
asr_size = gr.Dropdown(
label=i18n("ASR 模型尺寸"), choices=["large"], interactive=True, value="large"
@ -1496,7 +1498,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
version_checkbox = gr.Radio(
label=i18n("训练模型的版本"),
value=version,
choices=["v2", "v2Pro", "v2ProPlus", "v4", "v5dev", "v5turbo"],
choices=["v2", "v2Pro", "v2ProPlus", "v4"],
scale=5,
)
with gr.Accordion(label=i18n("预训练模型路径"), open=False):