Compare commits

...

4 Commits

Author SHA1 Message Date
Ding Junyao
b86e3687ea
Merge 7e66149757785f355ef3a9194bd68e162ad051ad into 6d51ffceafbf240145619af4c92408a6e54b7924 2026-11-14 18:45:44 +08:00
RVC-Boss
6d51ffceaf Update ASR configuration, Faster Whisper, FunASR and WebUI from update_asr package 2026-11-12 00:26:17 +08:00
Kakaru
59c44686f5
feat: add cache-dit (DBCache) block caching to the DiT sampler (#2868)
The DiT samples with the full block stack on every diffusion step. cache-dit
(vipshop/cache-dit, DBCache) skips the middle of the stack when the front residual
barely moved:

    front : trunk(x) -> blocks[0:Fn] -> x_F,   R = x_F - trunk(x)
            fd = sum|R - R_prev| / (sum|R_prev| + eps)
    hit   : x_mid = x_F + MnResidual_prev, then norm_out/proj_out only
    miss  : x_F -> blocks[Fn:depth] -> x_M, store MnResidual = x_M - x_F, R -> R_prev

The head (norm_out + proj_out) runs unconditionally, hits included. A hit additionally
needs `fd < threshold` outside the warmup steps and must pass the max_cached / max_cont /
max_accum gates.

The conditional and unconditional CFG branches are two separate forward passes, so each
keeps its own R_prev, MnResidual, hit/miss counters and gate counters.

Implementation:

* new `module/cache_dit.py`: configuration, per-sampling state, the CFM-side mixin;
* `DiT.forward` split into `trunk()` / `run_blocks(start, end)` / `head()` (weights are
  shared, nothing is copied or duplicated);
* `CFM.inference` creates one cache state per sampling run, advances it per step and reads
  back a single scalar per step (the only device/host sync);
* `cache_dit=off|auto|f4|f8|f16` plus dbcache_fn/dbcache_threshold/dbcache_warmup/
  dbcache_downsample on TTS_infer_pack.run, api.py, api_v2.py, both WebUIs and the CLI,
  with GSV_DBCACHE* environment fallbacks for scripts that build no request dict.

Default is off and the off path is the previous one: the split is bit-identical to the old
`forward()` on 24 configurations (fp16/fp32 x long skip x static cache x inference/training),
and `CFM.inference` with the switch off reproduces the pre-change output bit for bit with the
real weights. `auto` resolves to F8 from 16 sampling steps and stays off below that, so
4-step runs are never cached (the warmup budget already covers them).

Measured on a 32-step v5dev request with F8 / threshold 0.02 / warmup 8 / downsample 4:
18.8% of the DiT forwards served from cache, mel relL1 1.2e-3 against the uncached run,
DiT stage ~1.12x faster; F4 at the same threshold gives 21.9% and ~1.18x, threshold 0.05
gives 45.3% and ~1.34x.
2026-11-09 21:59:04 +08:00
Ding Junyao
7e66149757
fix: encoding err when extracting text BERT features because of no encoding setting in open() 2026-10-27 18:37:53 +08:00
16 changed files with 880 additions and 130 deletions

View File

@ -1094,6 +1094,41 @@ class TTS:
"""
self.stop_flag = True
def apply_cache_dit(self, inputs: dict = None, sample_steps: int = 32):
"""cache-dit / DBCache switch for the DiT sampler. **Default: off.**
Reads the same field names as the HTTP API, so a request, an environment variable and
a direct call give the same result:
cache_dit: off | auto | f4 | f8 | f16 (alias: dbcache)
dbcache_fn, dbcache_threshold, dbcache_warmup, dbcache_downsample
dbcache_warmup_interval, dbcache_max_cached, dbcache_max_cont, dbcache_max_accum
Env fallbacks: ``GSV_DBCACHE``, ``GSV_DBCACHE_THRESHOLD``, ``GSV_DBCACHE_WARMUP``,
``GSV_DBCACHE_DOWNSAMPLE`` (and the other ``GSV_DBCACHE_*`` in module/cache_dit.py).
``auto`` = F8 when ``sample_steps >= 16`` and off below that, so a 4-step run is never
cached by accident (its warmup budget already covers the whole run, so there is nothing
to gain and the quality risk is real).
"""
cfm = getattr(self.vits_model, "cfm", None)
if cfm is None or not hasattr(cfm, "set_dbcache"):
return None
from module.cache_dit import config_from_params
cfg = config_from_params(dict(inputs or {}), int(sample_steps),
getattr(cfm, "cache_depth", 0) or None)
if cfg.enabled and str(getattr(self.configs, "version", "")) == "v5turbo" and not cfg.force:
# 4-step turbo: nothing to cache, keep it off
print(i18n("cache-dit: not supported for v5turbo (4 steps), staying off"))
cfg.fn = 0
cfm.set_dbcache(cfg)
if cfg.enabled:
if int(sample_steps) < 16:
print(i18n("cache-dit: only %d sampling steps, expect a quality drop "
"(4-step v5turbo should keep cache-dit off)" % int(sample_steps)))
print(i18n("cache-dit: %s" % cfg))
return cfg
@torch.no_grad()
def run(self, inputs: dict):
"""
@ -1121,6 +1156,13 @@ class TTS:
"parallel_infer": True, # bool. whether to use parallel inference.
"repetition_penalty": 1.35, # float. repetition penalty for T2S model.
"sample_steps": 32, # int. number of sampling steps for VITS model V3.
"cfg_rate": 1.30, # float. classifier-free-guidance rate for V5dev/V5.
"cache_dit": "off", # str. off|auto|f4|f8|f16. DBCache for the V3/V4/V5 DiT:
# fewer DiT forwards, near-identical audio. off by default.
"dbcache_fn": 8, # int. front blocks (auto = F8 when sample_steps >= 16).
"dbcache_threshold": 0.02, # float. relative residual-diff threshold.
"dbcache_warmup": 8, # int. forced full-compute steps at the start.
"dbcache_downsample": 4, # int. time downsample of the cached Fn residual.
"super_sampling": False, # bool. whether to use super-sampling for audio when using VITS model V3.
"return_fragment": False, # bool. step by step return the audio fragment. (Best Quality, Slowest response speed. old version of streaming mode)
"streaming_mode": False, # bool. return audio chunk by chunk. (Medium quality, Slow response speed)
@ -1160,6 +1202,8 @@ class TTS:
cfg_rate = inputs.get("cfg_rate")
sample_steps = (4 if self.configs.version == "v5turbo" else 32) if sample_steps is None else int(sample_steps)
cfg_rate = (1.30 if self.configs.version == "v5dev" else 0.0) if cfg_rate is None else float(cfg_rate)
# cache-dit (DBCache): off unless the request asks for it, see module/cache_dit.py
self.apply_cache_dit(inputs, sample_steps)
super_sampling = inputs.get("super_sampling", False)
streaming_mode = inputs.get("streaming_mode", False)
overlap_length = inputs.get("overlap_length", 2)

View File

@ -171,7 +171,58 @@ class DiT(nn.Module):
text_cache=None, # torch tensor as text_embed
dt_cache=None, # torch tensor as dt
static_cache=None,
# cache-dit / DBCache (see module/cache_dit.py). `dbcache=None` => untouched path.
dbcache=None, # module.cache_dit.DBCacheState
dbcache_branch=0, # 0 = CFG positive, 1 = CFG negative
):
x, t, mask, rope, text_embed, dt = self.trunk(
x0, cond0, x_lens, time, dt_base_bootstrap=dt_base_bootstrap, text0=text0,
drop_audio_cond=drop_audio_cond, drop_text=drop_text, infer=infer,
text_cache=text_cache, dt_cache=dt_cache, static_cache=static_cache,
)
residual = x
if self.long_skip_connection is None:
residual = None
if dbcache is None or not dbcache.enabled:
# ---- original path: all blocks, then head ------------------------------
x = self.run_blocks(x, t, mask, rope, 0, self.depth, use_grad_ckpt)
else:
# ---- cache-dit: front -> (hit: head only | miss: mid -> head) -----------
fn = max(0, min(int(dbcache.fn), self.depth - 1))
x_f = self.run_blocks(x, t, mask, rope, 0, fn, use_grad_ckpt)
fd = dbcache.front_update(dbcache_branch, x_f, x)
if dbcache.can_cache(dbcache_branch, fd):
x = x_f + dbcache.mid_residual(dbcache_branch)
dbcache.on_hit(dbcache_branch)
if dbcache.trace:
print("[dbcache] step=%d HIT fd=%.5f (hits=%d misses=%d)"
% (dbcache.step, fd, dbcache.branch(dbcache_branch).hits,
dbcache.branch(dbcache_branch).misses), flush=True)
else:
x = self.run_blocks(x_f, t, mask, rope, fn, self.depth, use_grad_ckpt)
dbcache.on_miss(dbcache_branch, x, x_f)
if dbcache.trace:
print("[dbcache] step=%d MISS fd=%.5f (hits=%d misses=%d)"
% (dbcache.step, fd, dbcache.branch(dbcache_branch).hits,
dbcache.branch(dbcache_branch).misses), flush=True)
output = self.head(x, t, residual)
if infer:
return output, text_embed if static_cache is None else None, dt
else:
return output
# ---------------------------------------------------------------------------------
# Three-segment split used by cache-dit; the weights are shared, nothing is copied.
# ---------------------------------------------------------------------------------
def trunk(self, x0, cond0, x_lens, time, dt_base_bootstrap=None, text0=None,
drop_audio_cond=False, drop_text=False, infer=False, text_cache=None,
dt_cache=None, static_cache=None):
"""Everything before the transformer blocks: t/dt embedding + input embedding."""
text_embed = None
x = x0.transpose(2, 1)
cond = cond0.transpose(2, 1)
text = text0.transpose(2, 1)
@ -207,22 +258,22 @@ class DiT(nn.Module):
rope = (static_cache["rope"] if static_cache is not None
else self.rotary_embed.forward_from_seq_len(seq_len))
if self.long_skip_connection is not None:
residual = x
return x, t, mask, rope, text_embed, dt
for block in self.transformer_blocks:
def run_blocks(self, x, t, mask, rope, start=0, end=None, use_grad_ckpt=False):
"""`front` is blocks[0:fn], `mid` is blocks[fn:depth]."""
end = self.depth if end is None else end
for block in self.transformer_blocks[start:end]:
if use_grad_ckpt:
x = checkpoint(self.ckpt_wrapper(block), x, t, mask, rope, use_reentrant=False)
else:
x = block(x, t, mask=mask, rope=rope)
return x
def head(self, x, t, residual=None):
"""`norm_out -> proj_out`, computed on every step, hits included."""
if self.long_skip_connection is not None:
x = self.long_skip_connection(torch.cat((x, residual), dim=-1))
x = self.norm_out(x, t)
output = self.proj_out(x)
return self.proj_out(x)
if infer:
return output, text_embed if static_cache is None else None, dt
else:
return output

View File

@ -3,6 +3,7 @@ import os
import soundfile as sf
from tools.i18n.i18n import I18nAuto
import GPT_SoVITS.inference_webui as webui_ui
from GPT_SoVITS.inference_webui import change_gpt_weights, change_sovits_weights, get_tts_wav
i18n = I18nAuto()
@ -67,9 +68,25 @@ def main():
help="Language of the target text",
)
parser.add_argument("--output_path", required=True, help="Path to the output directory")
parser.add_argument(
"--cache_dit",
default="off",
choices=["off", "auto", "f4", "f8", "f16"],
help="cache-dit (DBCache) for the V3/V4/V5dev DiT, off by default; "
"fewer DiT forwards with near-identical audio",
)
parser.add_argument("--dbcache_fn", type=int, default=-1, help="front blocks (default 8)")
parser.add_argument("--dbcache_threshold", type=float, default=0.02, help="residual-diff threshold")
parser.add_argument("--dbcache_warmup", type=int, default=8, help="full-compute steps first")
parser.add_argument("--dbcache_downsample", type=int, default=4, help="residual time downsample")
args = parser.parse_args()
webui_ui.update_cache_dit(
args.cache_dit, args.dbcache_fn, args.dbcache_threshold,
args.dbcache_warmup, args.dbcache_downsample,
)
synthesize(
args.gpt_model,
args.sovits_model,

View File

@ -844,6 +844,36 @@ def audio_sr(audio, sr):
# cache_tokens={}#暂未实现清理机制
cache = {}
# cache-dit (DBCache) for the DiT sampler: off by default. The UI writes here, get_tts_wav
# reads it; the GSV_DBCACHE* environment variables work as well.
CACHE_DIT = {"cache_dit": "off", "dbcache_fn": None, "dbcache_threshold": 0.02,
"dbcache_warmup": 8, "dbcache_downsample": 4}
def update_cache_dit(mode, fn, thr, warmup, ds):
CACHE_DIT["cache_dit"] = str(mode or "off")
CACHE_DIT["dbcache_fn"] = None if fn in (None, "") or int(fn) < 0 else int(fn)
CACHE_DIT["dbcache_threshold"] = float(thr) if thr not in (None, "") else 0.02
CACHE_DIT["dbcache_warmup"] = int(warmup) if warmup not in (None, "") else 8
CACHE_DIT["dbcache_downsample"] = int(ds) if ds not in (None, "") else 4
def apply_cache_dit(vq_model, sample_steps, params=None):
"""cache-dit (DBCache) switch; off unless configured (UI or GSV_DBCACHE* env)."""
cfm = getattr(vq_model, "cfm", None)
if cfm is None or not hasattr(cfm, "set_dbcache"):
return None
from module.cache_dit import config_from_params
cfg = config_from_params(dict(params if params is not None else CACHE_DIT),
int(sample_steps), getattr(cfm, "cache_depth", 0) or None)
if cfg.enabled and str(getattr(vq_model, "version", "")) == "v5turbo" and not cfg.force:
cfg.fn = 0 # 4-step turbo: no gain and unstable quality, keep it off
cfm.set_dbcache(cfg)
if cfg.enabled:
print("cache-dit:", cfg)
return cfg
def get_tts_wav(
ref_wav_path,
@ -872,6 +902,7 @@ def get_tts_wav(
if cfg_rate is None:
cfg_rate = 0.0
sample_steps, cfg_rate = int(sample_steps), float(cfg_rate)
apply_cache_dit(globals().get("vq_model"), sample_steps)
if ref_wav_path:
pass
else:
@ -1356,6 +1387,26 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
value=(32 if model_version == "v3" else 8),
)
)
with gr.Accordion(i18n("DiT 加速 cache-dit(默认关闭)"), open=False):
with gr.Row():
cache_dit_mode = gr.Dropdown(
label=i18n("cache-dit"),
choices=["off", "auto", "f4", "f8", "f16"],
value="off", scale=2,
)
dbcache_fn = gr.Number(label="fn", value=-1, precision=0, scale=1)
dbcache_threshold = gr.Number(label="thr", value=0.02, scale=1)
dbcache_warmup = gr.Number(label="warmup", value=8, precision=0, scale=1)
dbcache_downsample = gr.Number(label="ds", value=4, precision=0, scale=1)
gr.Markdown(i18n("DBCache:只算前 fn 个 block,残差变化小于 thr 时跳过中间 block。"
"fn=-1 表示跟随模式(auto = 步数>=16 时 F8)。4 步的 v5turbo 不支持。"))
for _cd in (cache_dit_mode, dbcache_fn, dbcache_threshold,
dbcache_warmup, dbcache_downsample):
_cd.change(
update_cache_dit,
[cache_dit_mode, dbcache_fn, dbcache_threshold,
dbcache_warmup, dbcache_downsample],
)
if_sr_Checkbox = gr.Checkbox(
label=i18n("v3输出如果觉得闷可以试试开超分"),
value=False,

View File

@ -73,6 +73,19 @@ language = os.environ.get("language", "Auto")
language = sys.argv[-1] if sys.argv[-1] in scan_language_list() else language
i18n = I18nAuto(language=language)
# cache-dit (DBCache) for the DiT sampler: off by default. The UI writes here, tts_infer
# forwards it to TTS_infer_pack; the GSV_DBCACHE* environment variables work too.
CACHE_DIT = {"cache_dit": "off", "dbcache_fn": None, "dbcache_threshold": 0.02,
"dbcache_warmup": 8, "dbcache_downsample": 4}
def update_cache_dit(mode, fn, thr, warmup, ds):
CACHE_DIT["cache_dit"] = str(mode or "off")
CACHE_DIT["dbcache_fn"] = None if fn in (None, "") or int(fn) < 0 else int(fn)
CACHE_DIT["dbcache_threshold"] = float(thr) if thr not in (None, "") else 0.02
CACHE_DIT["dbcache_warmup"] = int(warmup) if warmup not in (None, "") else 8
CACHE_DIT["dbcache_downsample"] = int(ds) if ds not in (None, "") else 4
# os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1' # 确保直接启动推理UI时也能够设置。
@ -209,6 +222,7 @@ def inference(
"cfg_rate": float(cfg_rate),
"super_sampling": super_sampling,
}
inputs.update(CACHE_DIT) # cache-dit (DBCache), off unless the UI / env enables it
try:
for item in tts_pipeline.run(inputs):
yield item, actual_seed
@ -421,6 +435,25 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
repetition_penalty = gr.Slider(
minimum=0, maximum=2, step=0.05, label=i18n("重复惩罚"), value=1.35, interactive=True
)
with gr.Accordion(i18n("DiT 加速 cache-dit(默认关闭)"), open=False):
with gr.Row():
cache_dit_mode = gr.Dropdown(
label=i18n("cache-dit"), choices=["off", "auto", "f4", "f8", "f16"],
value="off", interactive=True, scale=2,
)
dbcache_fn = gr.Number(label="fn", value=-1, precision=0, scale=1)
dbcache_threshold = gr.Number(label="thr", value=0.02, scale=1)
dbcache_warmup = gr.Number(label="warmup", value=8, precision=0, scale=1)
dbcache_downsample = gr.Number(label="ds", value=4, precision=0, scale=1)
gr.Markdown(i18n("DBCache:只算前 fn 个 block,残差变化小于 thr 时跳过中间 block。"
"fn=-1 表示跟随模式(auto = 步数>=16 时 F8)。v5turbo 4 步不支持。"))
for _cd in (cache_dit_mode, dbcache_fn, dbcache_threshold,
dbcache_warmup, dbcache_downsample):
_cd.change(
update_cache_dit,
[cache_dit_mode, dbcache_fn, dbcache_threshold,
dbcache_warmup, dbcache_downsample],
)
with gr.Column():
with gr.Row():

View File

@ -0,0 +1,437 @@
"""cache-dit / DBCache (three-segment block caching) for the DiT/CFM sampler.
Follows vipshop/cache-dit (`CachedBlocks_Pattern_Base.forward`), adapted to the way this repo
samples: the block stack is split into three segments, and the middle one is skipped when the
front residual barely moved.
front : trunk(x_in) -> blocks[0:Fn] -> x_F (every step)
R = x_F - trunk(x_in) Fn residual, the decision signal
fd = sum|R - R_prev| / (sum|R_prev| + eps) (relative -> scale free)
hit : x_mid = x_F + MnResidual_prev, then head only (norm_out -> proj_out)
miss : x_F -> blocks[Fn:DEPTH] -> x_M; MnResidual = x_M - x_F; R -> R_prev
head : x_mid -> norm_out -> proj_out -> vel (every step, hit included)
A hit additionally requires `can_cache`: not in warmup, `fd < threshold`, and the three gates
`max_cached` (cached steps per sampling run), `max_cont` (consecutive hits) and `max_accum`
(accumulated residual drift).
CFG. This repo runs the conditional and the unconditional branch as **two separate forward
passes** (`CFM.inference`), so each branch owns its own `R_prev`, `MnResidual`, hit/miss
counters and gate counters -- the equivalent of cache-dit's `enable_separate_cfg` plus
`cfg_diff_compute_separate`. Nothing is shared between the branches.
Everything here is inert unless a `DBCacheConfig` with `fn > 0` is handed to `DiT.forward`;
`cache_dit="off"` keeps the sampler on the identical code path it had before this change.
"""
from __future__ import annotations
import os
from dataclasses import dataclass, replace
import torch
__all__ = [
"DBCacheConfig",
"DBCacheState",
"resolve_fn",
"config_from_params",
"MODES",
]
MODES = ("off", "auto", "f4", "f8", "f16")
# Defaults of the cache-dit reference recipe, and the `auto` policy used by the HTTP API
# (F8 from 16 sampling steps on, off below that).
DEFAULT_FN = 8
DEFAULT_THRESHOLD = 0.02
DEFAULT_WARMUP = 8
DEFAULT_DOWNSAMPLE = 4
AUTO_FN = 8
AUTO_MIN_STEPS = 16
_EPS = 1e-6
def resolve_fn(mode, steps=32, depth=None):
"""`off|auto|f4|f8|f16` (or a plain integer) -> number of front blocks."""
if mode is None:
return 0
if isinstance(mode, bool):
mode = int(mode)
if isinstance(mode, (int, float)):
fn = int(mode)
else:
text = str(mode).strip().lower()
if text in ("", "off", "none", "0", "false", "no"):
fn = 0
elif text in ("on", "true", "yes"):
fn = DEFAULT_FN
elif text == "auto":
fn = AUTO_FN if int(steps) >= AUTO_MIN_STEPS else 0
elif text.startswith("f") and text[1:].isdigit():
fn = int(text[1:])
elif text.isdigit():
fn = int(text)
else:
raise ValueError("unknown cache_dit mode %r (expected off|auto|f4|f8|f16)" % (mode,))
if depth is not None:
fn = max(0, min(fn, int(depth) - 1))
return fn
@dataclass
class DBCacheConfig:
"""cache-dit knobs. `fn == 0` means off (the default)."""
mode: str = "off"
fn: int = 0
threshold: float = DEFAULT_THRESHOLD
warmup: int = DEFAULT_WARMUP
downsample: int = DEFAULT_DOWNSAMPLE
warmup_interval: int = 1
max_cached: int = -1
max_cont: int = -1
max_accum: float = 0.0
# fd is reduced over the whole sequence by default: the prompt/reference frames carry a
# nearly constant residual, so they dilute the ratio and threshold 0.02 stays usable. Set
# to True to reduce over the target frames only -- a ~3x stricter signal on this model, so
# thresholds are not transferable between the two conventions.
fd_ignore_prompt: bool = False
trace: bool = False
# expert override: allow cache-dit below 16 sampling steps (see docs in
# `DBCacheMixin.dbcache_config`: too few steps gain nothing and the quality risk is real)
force: bool = False
@property
def enabled(self) -> bool:
return self.fn > 0
def clamped(self, depth: int | None = None) -> "DBCacheConfig":
if depth is None:
return self
return replace(self, fn=max(0, min(int(self.fn), int(depth) - 1)))
def __str__(self) -> str: # for logs
if not self.enabled:
return "off"
return ("%s(fn=%d thr=%.3g warmup=%d ds=%d mc=%d mcont=%d macc=%.3g)"
% (self.mode, self.fn, self.threshold, self.warmup, self.downsample,
self.max_cached, self.max_cont, self.max_accum))
def _get(params, key, env, default, cast):
"""request dict first (API field names), then env, then default."""
if params is not None:
for k in (key,):
if k in params and params[k] is not None and params[k] != "":
try:
return cast(params[k])
except (TypeError, ValueError):
pass
val = os.environ.get(env)
if val is not None and val != "":
try:
return cast(val)
except (TypeError, ValueError):
pass
return default
def config_from_params(params=None, steps=32, depth=None) -> DBCacheConfig:
"""Build the config from a request dict / env vars.
Field names match the HTTP API of our C++ server:
cache_dit (alias dbcache) = off|auto|f4|f8|f16
dbcache_fn, dbcache_threshold, dbcache_warmup, dbcache_downsample
dbcache_warmup_interval, dbcache_max_cached, dbcache_max_cont, dbcache_max_accum
Env fallbacks: GSV_DBCACHE / GSV_DBCACHE_FN / _THRESHOLD / _WARMUP / _DOWNSAMPLE / ...
"""
p = dict(params or {})
mode = p.get("cache_dit", p.get("dbcache", os.environ.get("GSV_DBCACHE", "off")))
mode = "off" if mode is None else str(mode).strip().lower()
cfg = DBCacheConfig(mode=mode)
cfg.threshold = _get(p, "dbcache_threshold", "GSV_DBCACHE_THRESHOLD", cfg.threshold, float)
cfg.warmup = _get(p, "dbcache_warmup", "GSV_DBCACHE_WARMUP", cfg.warmup, int)
cfg.downsample = max(1, _get(p, "dbcache_downsample", "GSV_DBCACHE_DOWNSAMPLE", cfg.downsample, int))
cfg.warmup_interval = max(1, _get(p, "dbcache_warmup_interval", "GSV_DBCACHE_WARMUP_INTERVAL",
cfg.warmup_interval, int))
cfg.max_cached = _get(p, "dbcache_max_cached", "GSV_DBCACHE_MAX_CACHED", cfg.max_cached, int)
cfg.max_cont = _get(p, "dbcache_max_cont", "GSV_DBCACHE_MAX_CONT", cfg.max_cont, int)
cfg.max_accum = _get(p, "dbcache_max_accum", "GSV_DBCACHE_MAX_ACCUM", cfg.max_accum, float)
cfg.fd_ignore_prompt = bool(_get(p, "dbcache_fd_ignore_prompt", "GSV_DBCACHE_FD_IGNORE_PROMPT",
int(cfg.fd_ignore_prompt), int))
cfg.trace = bool(_get(p, "dbcache_trace", "GSV_DBCACHE_TRACE", 0, int))
cfg.force = bool(_get(p, "dbcache_force", "GSV_DBCACHE_FORCE", 0, int))
# an explicit fn wins over the mode
if p.get("dbcache_fn") not in (None, ""):
cfg.fn = int(p["dbcache_fn"])
cfg.mode = "custom"
elif os.environ.get("GSV_DBCACHE_FN"):
cfg.fn = int(os.environ["GSV_DBCACHE_FN"])
cfg.mode = "custom"
else:
cfg.fn = resolve_fn(mode, steps)
if depth is not None:
cfg.fn = max(0, min(cfg.fn, int(depth) - 1))
return cfg
class DBCacheMixin:
"""The CFM-side plumbing (`set_dbcache` / per-run state / hit counters).
Mixed into `module.models.CFM` and `module.v5_inference.CFMV5`; both expose the DiT as
`self.estimator`. Defaults are class attributes, so `cache off` needs no `__init__` change
and the object layout of an existing checkpoint-compatible model is untouched.
"""
dbcache_spec = None # DBCacheConfig or request dict of dbcache_* fields
last_dbcache_stats = None # stats of the most recent inference() call
_dbcache_cum = None # accumulated over all calls (reset by dbcache_stats)
# ------------------------------------------------------------------ config
@property
def cache_depth(self):
return int(getattr(self.estimator, "depth", 0) or 0)
def set_dbcache(self, spec=None, **kw):
"""Enable/disable cache-dit for the next sampling runs.
`spec` is either a `module.cache_dit.DBCacheConfig` or a dict using the C++ server's
field names (`cache_dit`/`dbcache` + `dbcache_fn|threshold|warmup|downsample|...`).
Nothing is applied retroactively: `inference()` resolves it per call.
"""
if isinstance(spec, DBCacheConfig):
self.dbcache_spec = spec
else:
merged = dict(spec or {})
merged.update(kw)
cfg = merged.pop("config", None)
self.dbcache_spec = cfg if isinstance(cfg, DBCacheConfig) else merged
return self.dbcache_spec
def dbcache_config(self, steps):
"""Resolve the spec against the step count of the run that is about to start.
With no explicit spec the env switch (`GSV_DBCACHE=off|auto|f4|f8|f16`) is honoured,
so every entry point (WebUI, api.py, api_v2.py, CLI, Colab) can enable cache-dit
without extra plumbing. Below 16 sampling steps (the 4-step v5turbo) it stays off:
the measured gain is negative and the quality is unstable there.
"""
depth = self.cache_depth or None
spec = self.dbcache_spec
if isinstance(spec, DBCacheConfig):
fn = spec.fn if spec.mode in ("custom", None) else resolve_fn(spec.mode, steps, depth)
cfg = DBCacheConfig(**{**spec.__dict__,
"fn": fn if depth is None else min(fn, depth - 1)})
else:
cfg = config_from_params(spec, steps, depth)
if cfg.enabled and int(steps) < AUTO_MIN_STEPS and not cfg.force:
cfg.fn = 0
return cfg
# ------------------------------------------------------------------ per run
def _dbcache_begin(self, steps, seq_len, prompt_len, x_lens, device):
"""Fresh state per sampling run (new request / new chunk)."""
cfg = self.dbcache_config(steps)
if not cfg.enabled:
self.last_dbcache_stats = None
return None
return DBCacheState(cfg, self.cache_depth).begin(seq_len, prompt_len, x_lens, device)
def _dbcache_end(self, state):
if state is None:
return
st = state.stats()
self.last_dbcache_stats = st
cum = self._dbcache_cum
if cum is None:
cum = self._dbcache_cum = {"calls": 0, "steps": 0, "hits_pos": 0, "hits_neg": 0,
"misses_pos": 0, "misses_neg": 0}
cum["calls"] += 1
for k in ("steps", "hits_pos", "hits_neg", "misses_pos", "misses_neg"):
cum[k] = cum.get(k, 0) + st[k]
cum["hits"] = cum["hits_pos"] + cum["hits_neg"]
cum["misses"] = cum["misses_pos"] + cum["misses_neg"]
cum["total"] = cum["hits"] + cum["misses"]
cum["hit_rate"] = cum["hits"] / cum["total"] if cum["total"] else 0.0
cum["fn"] = st.get("fn")
cum["threshold"] = st.get("threshold")
cum["last"] = st
def dbcache_stats(self, reset=True):
"""Hit/miss counters accumulated since the last reset (None when cache-dit is off)."""
st = self._dbcache_cum
if reset:
self._dbcache_cum = None
return st
class _Branch:
"""Per-CFG-branch bookkeeping."""
__slots__ = ("valid", "r_prev", "r_cur", "m_res", "hits", "misses",
"cached_steps", "cont", "acc", "last_fd")
def __init__(self):
self.valid = False
self.r_prev = None
self.r_cur = None
self.m_res = None
self.hits = 0
self.misses = 0
self.cached_steps = 0
self.cont = 0
self.acc = 0.0
self.last_fd = 0.0
def reset(self):
self.__init__()
class DBCacheState:
"""State of one sampling run (`CFM.inference` = one run = one new request / chunk)."""
def __init__(self, config: DBCacheConfig, depth: int, trace: bool | None = None):
self.cfg = config
self.depth = int(depth)
self.trace = config.trace if trace is None else trace
self.step = 0
self.n_steps = 0
self.branches = [_Branch(), _Branch()]
self.ds = 1
self.note = "" # free-form tag for logs
# ------------------------------------------------------------------ lifecycle
def begin(self, seq_len: int, prompt_len: int, x_lens=None, device=None):
"""Reset for a new sampling run. Called once per `CFM.inference`."""
for b in self.branches:
b.reset()
self.step = 0
self.n_steps = 0
ds = int(self.cfg.downsample) if self.cfg.downsample else 1
self.ds = ds if ds >= 1 and seq_len % ds == 0 else 1
self._build_frame_mask(seq_len, prompt_len, x_lens, device)
return self
def _build_frame_mask(self, seq_len, prompt_len, x_lens, device):
"""Frames used for fd: skip the prompt/reference rows and the padding."""
self.mask = None
if not self.cfg.fd_ignore_prompt:
return
idx = torch.arange(seq_len, device=device)
mask = idx.unsqueeze(0) >= int(max(0, prompt_len))
if x_lens is not None:
lens = torch.as_tensor(x_lens, device=device).reshape(-1, 1)
if lens.shape[0] == 1 and seq_len >= int(lens.max()):
mask = mask & (idx.unsqueeze(0) < lens)
mask = mask.to(torch.float32)
if self.ds > 1:
mask = mask[:, :: self.ds]
mask = mask.unsqueeze(-1) # (b, n/ds, 1)
if float(mask.sum()) <= 0.0: # degenerate (all-prompt) -> fall back to everything
mask = None
self.mask = mask
def step_begin(self, step: int):
"""`db_set_step(j)`: current diffusion step, drives warmup and the trace."""
self.step = int(step)
self.n_steps += 1
def in_warmup(self) -> bool:
if self.step >= self.cfg.warmup:
return False
return (self.step % max(1, self.cfg.warmup_interval)) == 0
# ------------------------------------------------------------------ per-step ops
@property
def enabled(self) -> bool:
return self.cfg.enabled
@property
def fn(self) -> int:
return self.cfg.fn
def branch(self, index: int) -> _Branch:
return self.branches[0 if index == 0 else 1]
def front_update(self, index: int, x_f, trunk) -> float:
"""Store x_F / R, update R_prev-based similarity, return `fd` (one host sync)."""
br = self.branch(index)
r = x_f.float() - trunk.float() # Fn residual; fp32 keeps fd stable
if self.ds > 1:
r = r[:, :: self.ds, :]
r_prev = br.r_prev
if r_prev is None or r_prev.shape != r.shape:
r_prev = torch.zeros_like(r)
num = (r - r_prev).abs()
den = r_prev.abs()
if self.mask is not None and self.mask.shape[0] == num.shape[0] \
and self.mask.shape[1] == num.shape[1]:
num = num * self.mask
den = den * self.mask
num = num.sum(dim=(1, 2))
den = den.sum(dim=(1, 2)) + _EPS
fd = float((num / den).max().item()) # the only device->host scalar per step
br.r_cur = r
br.last_fd = fd
return fd
def can_cache(self, index: int, fd: float) -> bool:
br = self.branch(index)
cfg = self.cfg
hit = False
if br.valid and cfg.threshold > 0.0 and not self.in_warmup():
ok = fd < cfg.threshold
if ok and cfg.max_cached >= 0 and br.cached_steps >= cfg.max_cached:
ok = False
if ok and cfg.max_cont >= 0 and br.cont >= cfg.max_cont:
ok = False
br.cont = 0
if ok and cfg.max_accum > 0.0 and br.acc >= cfg.max_accum:
ok = False
hit = ok
if fd > 0.0:
br.acc += fd
return hit
def mid_residual(self, index: int):
return self.branch(index).m_res
def on_hit(self, index: int):
br = self.branch(index)
br.hits += 1
br.cached_steps += 1
br.cont += 1
def on_miss(self, index: int, x_m, x_f):
br = self.branch(index)
br.misses += 1
br.valid = True
br.cont = 0
br.m_res = (x_m - x_f)
if br.r_cur is not None:
if br.r_prev is None or br.r_prev.shape != br.r_cur.shape:
br.r_prev = br.r_cur.clone()
else:
br.r_prev.copy_(br.r_cur)
# ------------------------------------------------------------------ reporting
def stats(self) -> dict:
out = {"steps": self.n_steps, "last_step": self.step, "fn": self.cfg.fn,
"threshold": self.cfg.threshold, "warmup": self.cfg.warmup,
"downsample": self.ds, "mode": self.cfg.mode}
for i, name in enumerate(("pos", "neg")):
br = self.branches[i]
out["hits_" + name] = br.hits
out["misses_" + name] = br.misses
out["fd_" + name] = br.last_fd
out["acc_" + name] = br.acc
out["hits"] = out["hits_pos"] + out["hits_neg"]
out["misses"] = out["misses_pos"] + out["misses_neg"]
total = out["hits"] + out["misses"]
out["total"] = total
out["hit_rate"] = (out["hits"] / total) if total else 0.0
return out

View File

@ -10,6 +10,7 @@ from torch.nn import functional as F
from module import commons
from module import modules
from module import attentions
from module.cache_dit import DBCacheMixin
from f5_tts.model import DiT
from torch.nn import Conv1d, ConvTranspose1d, Conv2d
from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
@ -1099,7 +1100,7 @@ class SynthesizerTrn(nn.Module):
return codes.transpose(0, 1)
class CFM(torch.nn.Module):
class CFM(DBCacheMixin, torch.nn.Module):
def __init__(self, in_channels, dit):
super().__init__()
self.sigma_min = 1e-6
@ -1138,7 +1139,11 @@ class CFM(torch.nn.Module):
if self.use_conditioner_cache and self.use_static_cache else None)
d_tensor = (torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * d
if self.use_step_embedding else None)
# cache-dit: one fresh state per sampling run; None when the switch is off
dbcache = self._dbcache_begin(n_timesteps, T, prompt_len, x_lens, x.device)
for j in range(n_timesteps):
if dbcache is not None:
dbcache.step_begin(j)
t_tensor = torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * (t if self.use_step_embedding else j * d)
step_condition = {"dt_base_bootstrap": d_tensor, "dt_cache": dt_cache} if self.use_step_embedding else {}
v_pred, text_emb, dt = self.estimator(
@ -1153,6 +1158,8 @@ class CFM(torch.nn.Module):
infer=True,
text_cache=text_cache,
static_cache=static_cache,
dbcache=dbcache,
dbcache_branch=0,
**step_condition,
)
v_pred = v_pred.transpose(2, 1)
@ -1174,6 +1181,8 @@ class CFM(torch.nn.Module):
infer=True,
text_cache=text_cfg_cache if self.cfg_drop_text else text_cache,
static_cache=static_cache,
dbcache=dbcache,
dbcache_branch=1,
**step_condition,
)
neg = neg.transpose(2, 1)
@ -1183,6 +1192,7 @@ class CFM(torch.nn.Module):
x = x + d * v_pred
t = t + d
x[:, :, :prompt_len] = 0
self._dbcache_end(dbcache)
return x
def forward(self, x1, x_lens, prompt_lens, mu, use_grad_ckpt):

View File

@ -13,7 +13,7 @@ normalizer = lambda x: cn2an.transform(x, "an2cn")
current_file_path = os.path.dirname(__file__)
pinyin_to_symbol_map = {
line.split("\t")[0]: line.strip().split("\t")[1]
for line in open(os.path.join(current_file_path, "opencpop-strict.txt")).readlines()
for line in open(os.path.join(current_file_path, "opencpop-strict.txt"), encoding="utf-8").readlines()
}
import jieba_fast

View File

@ -14,7 +14,7 @@ normalizer = lambda x: cn2an.transform(x, "an2cn")
current_file_path = os.path.dirname(__file__)
pinyin_to_symbol_map = {
line.split("\t")[0]: line.strip().split("\t")[1]
for line in open(os.path.join(current_file_path, "opencpop-strict.txt")).readlines()
for line in open(os.path.join(current_file_path, "opencpop-strict.txt"), encoding="utf-8").readlines()
}
import jieba_fast

View File

@ -131,7 +131,7 @@ def replace_consecutive_punctuation(text):
def read_dict():
g2p_dict = {}
start_line = 49
with open(CMU_DICT_PATH) as f:
with open(CMU_DICT_PATH, encoding="utf-8") as f:
line = f.readline()
line_index = 1
while line:
@ -154,7 +154,7 @@ def read_dict():
def read_dict_new():
g2p_dict = {}
with open(CMU_DICT_PATH) as f:
with open(CMU_DICT_PATH, encoding="utf-8") as f:
line = f.readline()
line_index = 1
while line:
@ -167,7 +167,7 @@ def read_dict_new():
line_index = line_index + 1
line = f.readline()
with open(CMU_DICT_FAST_PATH) as f:
with open(CMU_DICT_FAST_PATH, encoding="utf-8") as f:
line = f.readline()
line_index = 1
while line:
@ -185,7 +185,7 @@ def read_dict_new():
def hot_reload_hot(g2p_dict):
with open(CMU_DICT_HOT_PATH) as f:
with open(CMU_DICT_HOT_PATH, encoding="utf-8") as f:
line = f.readline()
line_index = 1
while line:

58
api.py
View File

@ -829,6 +829,28 @@ splits = {
}
def apply_cache_dit(vq_model, params: dict, sample_steps: int):
"""cache-dit (DBCache) switch for the DiT sampler; off unless configured.
Field names match api_v2.py:
``cache_dit`` = off|auto|f4|f8|f16, ``dbcache_fn``, ``dbcache_threshold``,
``dbcache_warmup``, ``dbcache_downsample``. The ``GSV_DBCACHE*`` env vars work too,
also on the paths that never go through these request fields.
"""
cfm = getattr(vq_model, "cfm", None)
if cfm is None or not hasattr(cfm, "set_dbcache"):
return None
from module.cache_dit import config_from_params
cfg = config_from_params(params, int(sample_steps), getattr(cfm, "cache_depth", 0) or None)
if cfg.enabled and str(getattr(vq_model, "version", "")) == "v5turbo" and not cfg.force:
cfg.fn = 0 # 4-step turbo: no gain and unstable quality, keep it off
cfm.set_dbcache(cfg)
if cfg.enabled:
print("cache-dit:", cfg)
return cfg
def get_tts_wav(
ref_wav_path,
prompt_text,
@ -844,6 +866,11 @@ def get_tts_wav(
if_sr=False,
spk="default",
cfg_rate=None,
cache_dit=None,
dbcache_fn=None,
dbcache_threshold=None,
dbcache_warmup=None,
dbcache_downsample=None,
):
infer_sovits = speaker_list[spk].sovits
vq_model = infer_sovits.vq_model
@ -867,6 +894,12 @@ def get_tts_wav(
if sample_steps not in [4, 8, 16, 32]:
sample_steps = 8
# cache-dit (DBCache) for the DiT sampler: off unless asked for
apply_cache_dit(vq_model, dict(
cache_dit=cache_dit, dbcache_fn=dbcache_fn, dbcache_threshold=dbcache_threshold,
dbcache_warmup=dbcache_warmup, dbcache_downsample=dbcache_downsample,
), sample_steps)
if if_sr and version != "v3":
if_sr = False
@ -1126,6 +1159,11 @@ def handle(
sample_steps,
if_sr,
cfg_rate=None,
cache_dit=None,
dbcache_fn=None,
dbcache_threshold=None,
dbcache_warmup=None,
dbcache_downsample=None,
):
if (
refer_wav_path == ""
@ -1163,6 +1201,11 @@ def handle(
sample_steps,
if_sr,
cfg_rate=cfg_rate,
cache_dit=cache_dit,
dbcache_fn=dbcache_fn,
dbcache_threshold=dbcache_threshold,
dbcache_warmup=dbcache_warmup,
dbcache_downsample=dbcache_downsample,
),
media_type="audio/" + media_type,
)
@ -1372,6 +1415,11 @@ async def tts_endpoint(request: Request):
json_post_raw.get("sample_steps"),
json_post_raw.get("if_sr", False),
cfg_rate=json_post_raw.get("cfg_rate"),
cache_dit=json_post_raw.get("cache_dit"),
dbcache_fn=json_post_raw.get("dbcache_fn"),
dbcache_threshold=json_post_raw.get("dbcache_threshold"),
dbcache_warmup=json_post_raw.get("dbcache_warmup"),
dbcache_downsample=json_post_raw.get("dbcache_downsample"),
)
@ -1391,6 +1439,11 @@ async def tts_endpoint(
sample_steps: int = None,
if_sr: bool = False,
cfg_rate: float = None,
cache_dit: str = None,
dbcache_fn: int = None,
dbcache_threshold: float = None,
dbcache_warmup: int = None,
dbcache_downsample: int = None,
):
return handle(
refer_wav_path,
@ -1407,6 +1460,11 @@ async def tts_endpoint(
sample_steps,
if_sr,
cfg_rate=cfg_rate,
cache_dit=cache_dit,
dbcache_fn=dbcache_fn,
dbcache_threshold=dbcache_threshold,
dbcache_warmup=dbcache_warmup,
dbcache_downsample=dbcache_downsample,
)

View File

@ -44,6 +44,11 @@ POST:
"streaming_mode": False, # bool or int. return audio chunk by chunk.T he available options are: 0,1,2,3 or True/False (0/False: Disabled | 1/True: Best Quality, Slowest response speed (old version streaming_mode) | 2: Medium Quality, Slow response speed | 3: Lower Quality, Faster response speed )
"overlap_length": 2, # int. overlap length of semantic tokens for streaming mode.
"min_chunk_length": 16, # int. The minimum chunk length of semantic tokens for streaming mode. (affects audio chunk size)
"cache_dit": "off", # str.(optional) off|auto|f4|f8|f16. cache-dit/DBCache for the V3/V4/V5 DiT: fewer DiT forwards, near-identical audio. Off by default; 4-step runs are refused.
"dbcache_fn": 8, # int.(optional) front blocks (default 8; auto = F8 when sample_steps >= 16).
"dbcache_threshold": 0.02, # float.(optional) relative residual-diff threshold (default 0.02).
"dbcache_warmup": 8, # int.(optional) forced full-compute steps at the start (default 8).
"dbcache_downsample": 4, # int.(optional) time downsample of the cached Fn residual (default 4).
"gpt_streaming": false, # bool. opt in to semantic-token streaming.
"streaming_chunk_seconds": 2.0,
"streaming_window_mode": "fixed",
@ -201,6 +206,13 @@ class TTS_Request(BaseModel):
repetition_penalty: float = 1.35
sample_steps: Optional[int] = None
cfg_rate: Optional[float] = None
# cache-dit (DBCache) for the V3/V4/V5 DiT sampler; off by default. Field names match
# the ones the TTS pipeline and the CLI use.
cache_dit: str = "off" # off | auto | f4 | f8 | f16
dbcache_fn: Optional[int] = None # front blocks (overrides cache_dit)
dbcache_threshold: Optional[float] = None
dbcache_warmup: Optional[int] = None
dbcache_downsample: Optional[int] = None
super_sampling: bool = False
overlap_length: int = 2
min_chunk_length: int = 16
@ -613,6 +625,11 @@ async def tts_get_endpoint(
repetition_penalty: float = 1.35,
sample_steps: Optional[int] = None,
cfg_rate: Optional[float] = None,
cache_dit: str = "off",
dbcache_fn: Optional[int] = None,
dbcache_threshold: Optional[float] = None,
dbcache_warmup: Optional[int] = None,
dbcache_downsample: Optional[int] = None,
super_sampling: bool = False,
streaming_mode: Union[bool, int] = False,
overlap_length: int = 2,
@ -649,6 +666,11 @@ async def tts_get_endpoint(
"repetition_penalty": float(repetition_penalty),
"sample_steps": None if sample_steps is None else int(sample_steps),
"cfg_rate": None if cfg_rate is None else float(cfg_rate),
"cache_dit": cache_dit,
"dbcache_fn": dbcache_fn,
"dbcache_threshold": dbcache_threshold,
"dbcache_warmup": dbcache_warmup,
"dbcache_downsample": dbcache_downsample,
"super_sampling": super_sampling,
"overlap_length": int(overlap_length),
"min_chunk_length": int(min_chunk_length),

View File

@ -10,9 +10,20 @@ def get_models():
asr_dict = {
"Fun-ASR-Nano (31语种+方言, 推荐)": {"lang": ["zh", "en", "ja", "ko", "yue", "auto"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
"SenseVoice (极速, 5语种)": {"lang": ["zh", "en", "ja", "ko", "yue", "auto"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
"达摩 ASR (中文经典)": {"lang": ["zh", "yue"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
"达摩 ASR (中文经典)": {
"lang": ["zh"],
"size": ["large"],
"path": "funasr_asr.py",
"backend": "damo",
"precision": ["float32"],
},
"Fun-ASR-Nano(速度更慢,更准)": {
"lang": ["zh", "en", "ja", "ko", "yue", "auto"],
"size": ["large"],
"path": "funasr_asr.py",
"backend": "fun-asr-nano",
"precision": ["float32"],
},
"Faster Whisper (多语种)": {
"lang": ["auto", "en", "ja", "ko"],
"size": get_models(),

View File

@ -40,6 +40,26 @@ language_code_list = [
def download_model(model_size: str):
model_path = ""
if "distil" in model_size:
if "3.5" in model_size:
repo_id = "distil-whisper/distil-large-v3.5-ct2"
model_path = "tools/asr/models/faster-distil-whisper-large-v3.5"
else:
repo_id = "Systran/faster-{}-whisper-{}".format(*model_size.split("-", maxsplit=1))
elif model_size == "large-v3-turbo":
repo_id = "mobiuslabsgmbh/faster-whisper-large-v3-turbo"
model_path = "tools/asr/models/faster-whisper-large-v3-turbo"
else:
repo_id = f"Systran/faster-whisper-{model_size}"
model_path = model_path or f"tools/asr/models/{repo_id.replace('Systran/', '').replace('distil-whisper/', '', 1)}"
if os.path.isfile(os.path.join(model_path, "config.json")) and os.path.isfile(
os.path.join(model_path, "model.bin")
):
print(f"Using local Faster Whisper model: {model_path}")
return model_path
url = "https://huggingface.co/api/models/gpt2"
try:
requests.get(url, timeout=3)
@ -47,26 +67,6 @@ def download_model(model_size: str):
except Exception:
source = "ModelScope"
model_path = ""
if source == "HF":
if "distil" in model_size:
if "3.5" in model_size:
repo_id = "distil-whisper/distil-large-v3.5-ct2"
model_path = "tools/asr/models/faster-distil-whisper-large-v3.5"
else:
repo_id = "Systran/faster-{}-whisper-{}".format(*model_size.split("-", maxsplit=1))
elif model_size == "large-v3-turbo":
repo_id = "mobiuslabsgmbh/faster-whisper-large-v3-turbo"
model_path = "tools/asr/models/faster-whisper-large-v3-turbo"
else:
repo_id = f"Systran/faster-whisper-{model_size}"
model_path = (
model_path or f"tools/asr/models/{repo_id.replace('Systran/', '').replace('distil-whisper/', '', 1)}"
)
else:
repo_id = "XXXXRT/faster-whisper"
model_path = "tools/asr/models"
files: list[str] = [
"config.json",
"model.bin",
@ -76,12 +76,8 @@ def download_model(model_size: str):
if "large-v3" in model_size or "distil" in model_size:
files.append("preprocessor_config.json")
files.append("vocabulary.json")
files.remove("vocabulary.txt")
if source == "ModelScope":
files = [f"faster-whisper-{model_size}/{file}".replace("whisper-distil", "distil-whisper") for file in files]
if source == "HF":
print(f"Downloading model from HuggingFace: {repo_id} to {model_path}")
snapshot_download_hf(
@ -91,13 +87,14 @@ def download_model(model_size: str):
allow_patterns=files,
)
else:
print(f"Downloading model from ModelScope: {repo_id} to {model_path}")
repo_id = "XXXXRT/faster-whisper"
files = [f"faster-whisper-{model_size}/{file}".replace("whisper-distil", "distil-whisper") for file in files]
print(f"Downloading model from ModelScope: {repo_id} to tools/asr/models")
snapshot_download_ms(
repo_id,
local_dir=model_path,
local_dir="tools/asr/models",
allow_patterns=files,
)
return model_path + f"/faster-whisper-{model_size}".replace("whisper-distil", "distil-whisper")
return model_path

View File

@ -2,18 +2,45 @@
import argparse
import os
import sys
import traceback
import setuptools._distutils as _distutils
import setuptools._distutils.util as _distutils_util
import setuptools._distutils.version as _distutils_version
sys.modules.setdefault("distutils", _distutils)
sys.modules.setdefault("distutils.util", _distutils_util)
sys.modules.setdefault("distutils.version", _distutils_version)
from funasr import AutoModel
from modelscope import snapshot_download
from tqdm import tqdm
funasr_models = {} # 存储模型避免重复加载
ASR_MODEL_ROOT = "tools/asr/models"
FUN_ASR_NANO_MODEL_ID = "FunAudioLLM/Fun-ASR-Nano-2512"
FUN_ASR_NANO_MODEL_SOURCES = (
{"model": FUN_ASR_NANO_MODEL_ID, "hub": "hf", "trust_remote_code": True},
{"model": FUN_ASR_NANO_MODEL_ID, "hub": "ms", "trust_remote_code": False},
)
VAD_MODEL_ID = "iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"
PUNC_MODEL_ID = "iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
PARAFORMER_MODEL_ID = "iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
UNIASR_MODEL_ID = "iic/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
FUN_ASR_NANO_MODEL_PATH = f"{ASR_MODEL_ROOT}/Fun-ASR-Nano-2512"
VAD_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_fsmn_vad_zh-cn-16k-common-pytorch"
PUNC_MODEL_PATH = f"{ASR_MODEL_ROOT}/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
PARAFORMER_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
UNIASR_MODEL_PATH = f"{ASR_MODEL_ROOT}/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
def ensure_model_dir(model_id, model_path, revision=None):
if os.path.isfile(os.path.join(model_path, "model.pt")):
return model_path
download_kwargs = {"local_dir": model_path}
if revision is not None:
download_kwargs["revision"] = revision
print(f"Downloading FunASR model: {model_id} -> {model_path}")
snapshot_download(model_id, **download_kwargs)
return model_path
def only_asr(input_file, language, backend="fun-asr-nano"):
@ -26,100 +53,86 @@ def only_asr(input_file, language, backend="fun-asr-nano"):
return text
def is_unregistered_model_error(exc):
message = str(exc).lower()
return "model" in message and "not registered" in message
def disable_flash_attention_probe():
import transformers.utils
import transformers.utils.import_utils
transformers.utils.import_utils.is_flash_attn_2_available = lambda: False
transformers.utils.is_flash_attn_2_available = lambda: False
def create_fun_asr_nano_model(device):
common_kwargs = dict(
vad_model="fsmn-vad",
disable_flash_attention_probe()
ensure_model_dir(FUN_ASR_NANO_MODEL_ID, FUN_ASR_NANO_MODEL_PATH)
ensure_model_dir(VAD_MODEL_ID, VAD_MODEL_PATH)
return AutoModel(
model=FUN_ASR_NANO_MODEL_PATH,
vad_model=VAD_MODEL_PATH,
device=device,
disable_update=True,
check_latest=False,
vad_kwargs={"check_latest": False},
hub="ms",
)
last_error = None
for model_source in FUN_ASR_NANO_MODEL_SOURCES:
try:
return AutoModel(**model_source, **common_kwargs)
except (KeyError, RuntimeError, ValueError) as exc:
if not is_unregistered_model_error(exc):
raise
last_error = exc
raise last_error
def create_damo_model(device):
ensure_model_dir(VAD_MODEL_ID, VAD_MODEL_PATH, revision="v2.0.4")
ensure_model_dir(PUNC_MODEL_ID, PUNC_MODEL_PATH, revision="v2.0.4")
ensure_model_dir(PARAFORMER_MODEL_ID, PARAFORMER_MODEL_PATH, revision="v2.0.4")
return AutoModel(
model=PARAFORMER_MODEL_PATH,
model_revision="v2.0.4",
vad_model=VAD_MODEL_PATH,
vad_model_revision="v2.0.4",
vad_kwargs={"check_latest": False},
punc_model=PUNC_MODEL_PATH,
punc_model_revision="v2.0.4",
punc_kwargs={"check_latest": False},
device=device,
disable_update=True,
check_latest=False,
hub="ms",
)
def create_uniasr_model(device):
ensure_model_dir(UNIASR_MODEL_ID, UNIASR_MODEL_PATH, revision="master")
return AutoModel(
model=UNIASR_MODEL_PATH,
model_revision="master",
device=device,
disable_update=True,
check_latest=False,
hub="ms",
)
def create_model(language="zh", **kwargs):
backend = kwargs.get("backend", "fun-asr-nano")
import torch
# For non-classic backends, route to multilingual models regardless of language
if backend in ("fun-asr-nano", "sensevoice") and language != "yue":
import torch
device = "cuda" if torch.cuda.is_available() else "cpu"
cache_key = f"{language}_{backend}"
if cache_key in funasr_models:
return funasr_models[cache_key]
device = "cuda" if torch.cuda.is_available() else "cpu"
cache_key = f"{backend}_{language}"
if cache_key in funasr_models:
return funasr_models[cache_key]
if backend == "fun-asr-nano":
model = create_fun_asr_nano_model(device)
print(f"FunASR Fun-ASR-Nano 模型加载完成: {language.upper()}")
else:
model = AutoModel(
model="iic/SenseVoiceSmall",
vad_model="fsmn-vad",
device=device,
disable_update=True,
)
print(f"FunASR SenseVoice 模型加载完成: {language.upper()}")
funasr_models[cache_key] = model
return model
if language == "zh":
path_vad = "tools/asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch"
path_punc = "tools/asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
path_asr = "tools/asr/models/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
snapshot_download(
"iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
local_dir="tools/asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch",
)
snapshot_download(
"iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
local_dir="tools/asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
)
snapshot_download(
"iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
local_dir="tools/asr/models/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
)
model_revision = "v2.0.4"
vad_model_revision = punc_model_revision = "v2.0.4"
elif language == "yue":
path_asr = "tools/asr/models/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
snapshot_download(
"iic/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
local_dir="tools/asr/models/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
)
path_vad = path_punc = None
vad_model_revision = punc_model_revision = ""
model_revision = "master"
if backend == "damo":
if language != "zh":
raise ValueError("达摩 ASR 仅支持 zh")
model = create_damo_model(device)
print(f"FunASR 达摩 Paraformer 模型加载完成: {language.upper()}")
elif backend == "fun-asr-nano" and language == "yue":
model = create_uniasr_model(device)
print(f"FunASR UniASR 粤语模型加载完成: {language.upper()}")
elif backend == "fun-asr-nano":
model = create_fun_asr_nano_model(device)
print(f"FunASR Fun-ASR-Nano 模型加载完成: {language.upper()}")
else:
raise ValueError(f"{language} is not supported. Supported: zh, yue, ja, en, ko, auto")
raise ValueError(f"{backend} is not supported")
if language in funasr_models:
return funasr_models[language]
else:
model = AutoModel(
model=path_asr,
model_revision=model_revision,
vad_model=path_vad,
vad_model_revision=vad_model_revision,
punc_model=path_punc,
punc_model_revision=punc_model_revision,
)
print(f"FunASR 模型加载完成: {language.upper()}")
funasr_models[language] = model
return model
funasr_models[cache_key] = model
return model
def execute_asr(input_folder, output_folder, model_size, language, backend="fun-asr-nano"):
@ -163,10 +176,14 @@ if __name__ == "__main__":
parser.add_argument(
"-p", "--precision", type=str, default="float16", choices=["float16", "float32"], help="fp16 or fp32"
) # 还没接入
parser.add_argument(
"--backend", type=str, default="fun-asr-nano", choices=["fun-asr-nano", "damo"], help="ASR backend"
)
cmd = parser.parse_args()
execute_asr(
input_folder=cmd.input_folder,
output_folder=cmd.output_folder,
model_size=cmd.model_size,
language=cmd.language,
backend=cmd.backend,
)

View File

@ -381,6 +381,8 @@ def open_asr(asr_inp_dir, asr_opt_dir, asr_model, asr_model_size, asr_lang, asr_
cmd += f" -s {asr_model_size}"
cmd += f" -l {asr_lang}"
cmd += f" -p {asr_precision}"
if "backend" in asr_dict[asr_model]:
cmd += f" --backend {asr_dict[asr_model]['backend']}"
output_file_name = os.path.basename(asr_inp_dir)
output_folder = asr_opt_dir or "output/asr_opt"
output_file_path = os.path.abspath(f"{output_folder}/{output_file_name}.list")
@ -1412,7 +1414,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
label=i18n("ASR 模型"),
choices=list(asr_dict.keys()),
interactive=True,
value="达摩 ASR (中文)",
value="达摩 ASR (中文经典)",
)
asr_size = gr.Dropdown(
label=i18n("ASR 模型尺寸"), choices=["large"], interactive=True, value="large"