diff --git a/GPT_SoVITS/TTS_infer_pack/TTS.py b/GPT_SoVITS/TTS_infer_pack/TTS.py index df9713b9..7a7f200e 100644 --- a/GPT_SoVITS/TTS_infer_pack/TTS.py +++ b/GPT_SoVITS/TTS_infer_pack/TTS.py @@ -1094,6 +1094,41 @@ class TTS: """ self.stop_flag = True + def apply_cache_dit(self, inputs: dict = None, sample_steps: int = 32): + """cache-dit / DBCache switch for the DiT sampler. **Default: off.** + + Reads the same field names as the HTTP API, so a request, an environment variable and + a direct call give the same result: + + cache_dit: off | auto | f4 | f8 | f16 (alias: dbcache) + dbcache_fn, dbcache_threshold, dbcache_warmup, dbcache_downsample + dbcache_warmup_interval, dbcache_max_cached, dbcache_max_cont, dbcache_max_accum + + Env fallbacks: ``GSV_DBCACHE``, ``GSV_DBCACHE_THRESHOLD``, ``GSV_DBCACHE_WARMUP``, + ``GSV_DBCACHE_DOWNSAMPLE`` (and the other ``GSV_DBCACHE_*`` in module/cache_dit.py). + ``auto`` = F8 when ``sample_steps >= 16`` and off below that, so a 4-step run is never + cached by accident (its warmup budget already covers the whole run, so there is nothing + to gain and the quality risk is real). + """ + cfm = getattr(self.vits_model, "cfm", None) + if cfm is None or not hasattr(cfm, "set_dbcache"): + return None + from module.cache_dit import config_from_params + + cfg = config_from_params(dict(inputs or {}), int(sample_steps), + getattr(cfm, "cache_depth", 0) or None) + if cfg.enabled and str(getattr(self.configs, "version", "")) == "v5turbo" and not cfg.force: + # 4-step turbo: nothing to cache, keep it off + print(i18n("cache-dit: not supported for v5turbo (4 steps), staying off")) + cfg.fn = 0 + cfm.set_dbcache(cfg) + if cfg.enabled: + if int(sample_steps) < 16: + print(i18n("cache-dit: only %d sampling steps, expect a quality drop " + "(4-step v5turbo should keep cache-dit off)" % int(sample_steps))) + print(i18n("cache-dit: %s" % cfg)) + return cfg + @torch.no_grad() def run(self, inputs: dict): """ @@ -1121,6 +1156,13 @@ class TTS: "parallel_infer": True, # bool. whether to use parallel inference. "repetition_penalty": 1.35, # float. repetition penalty for T2S model. "sample_steps": 32, # int. number of sampling steps for VITS model V3. + "cfg_rate": 1.30, # float. classifier-free-guidance rate for V5dev/V5. + "cache_dit": "off", # str. off|auto|f4|f8|f16. DBCache for the V3/V4/V5 DiT: + # fewer DiT forwards, near-identical audio. off by default. + "dbcache_fn": 8, # int. front blocks (auto = F8 when sample_steps >= 16). + "dbcache_threshold": 0.02, # float. relative residual-diff threshold. + "dbcache_warmup": 8, # int. forced full-compute steps at the start. + "dbcache_downsample": 4, # int. time downsample of the cached Fn residual. "super_sampling": False, # bool. whether to use super-sampling for audio when using VITS model V3. "return_fragment": False, # bool. step by step return the audio fragment. (Best Quality, Slowest response speed. old version of streaming mode) "streaming_mode": False, # bool. return audio chunk by chunk. (Medium quality, Slow response speed) @@ -1160,6 +1202,8 @@ class TTS: cfg_rate = inputs.get("cfg_rate") sample_steps = (4 if self.configs.version == "v5turbo" else 32) if sample_steps is None else int(sample_steps) cfg_rate = (1.30 if self.configs.version == "v5dev" else 0.0) if cfg_rate is None else float(cfg_rate) + # cache-dit (DBCache): off unless the request asks for it, see module/cache_dit.py + self.apply_cache_dit(inputs, sample_steps) super_sampling = inputs.get("super_sampling", False) streaming_mode = inputs.get("streaming_mode", False) overlap_length = inputs.get("overlap_length", 2) diff --git a/GPT_SoVITS/f5_tts/model/backbones/dit.py b/GPT_SoVITS/f5_tts/model/backbones/dit.py index 46570fcf..2c3ebf23 100644 --- a/GPT_SoVITS/f5_tts/model/backbones/dit.py +++ b/GPT_SoVITS/f5_tts/model/backbones/dit.py @@ -171,7 +171,58 @@ class DiT(nn.Module): text_cache=None, # torch tensor as text_embed dt_cache=None, # torch tensor as dt static_cache=None, + # cache-dit / DBCache (see module/cache_dit.py). `dbcache=None` => untouched path. + dbcache=None, # module.cache_dit.DBCacheState + dbcache_branch=0, # 0 = CFG positive, 1 = CFG negative ): + x, t, mask, rope, text_embed, dt = self.trunk( + x0, cond0, x_lens, time, dt_base_bootstrap=dt_base_bootstrap, text0=text0, + drop_audio_cond=drop_audio_cond, drop_text=drop_text, infer=infer, + text_cache=text_cache, dt_cache=dt_cache, static_cache=static_cache, + ) + + residual = x + if self.long_skip_connection is None: + residual = None + + if dbcache is None or not dbcache.enabled: + # ---- original path: all blocks, then head ------------------------------ + x = self.run_blocks(x, t, mask, rope, 0, self.depth, use_grad_ckpt) + else: + # ---- cache-dit: front -> (hit: head only | miss: mid -> head) ----------- + fn = max(0, min(int(dbcache.fn), self.depth - 1)) + x_f = self.run_blocks(x, t, mask, rope, 0, fn, use_grad_ckpt) + fd = dbcache.front_update(dbcache_branch, x_f, x) + if dbcache.can_cache(dbcache_branch, fd): + x = x_f + dbcache.mid_residual(dbcache_branch) + dbcache.on_hit(dbcache_branch) + if dbcache.trace: + print("[dbcache] step=%d HIT fd=%.5f (hits=%d misses=%d)" + % (dbcache.step, fd, dbcache.branch(dbcache_branch).hits, + dbcache.branch(dbcache_branch).misses), flush=True) + else: + x = self.run_blocks(x_f, t, mask, rope, fn, self.depth, use_grad_ckpt) + dbcache.on_miss(dbcache_branch, x, x_f) + if dbcache.trace: + print("[dbcache] step=%d MISS fd=%.5f (hits=%d misses=%d)" + % (dbcache.step, fd, dbcache.branch(dbcache_branch).hits, + dbcache.branch(dbcache_branch).misses), flush=True) + + output = self.head(x, t, residual) + + if infer: + return output, text_embed if static_cache is None else None, dt + else: + return output + + # --------------------------------------------------------------------------------- + # Three-segment split used by cache-dit; the weights are shared, nothing is copied. + # --------------------------------------------------------------------------------- + def trunk(self, x0, cond0, x_lens, time, dt_base_bootstrap=None, text0=None, + drop_audio_cond=False, drop_text=False, infer=False, text_cache=None, + dt_cache=None, static_cache=None): + """Everything before the transformer blocks: t/dt embedding + input embedding.""" + text_embed = None x = x0.transpose(2, 1) cond = cond0.transpose(2, 1) text = text0.transpose(2, 1) @@ -207,22 +258,22 @@ class DiT(nn.Module): rope = (static_cache["rope"] if static_cache is not None else self.rotary_embed.forward_from_seq_len(seq_len)) - if self.long_skip_connection is not None: - residual = x + return x, t, mask, rope, text_embed, dt - for block in self.transformer_blocks: + def run_blocks(self, x, t, mask, rope, start=0, end=None, use_grad_ckpt=False): + """`front` is blocks[0:fn], `mid` is blocks[fn:depth].""" + end = self.depth if end is None else end + for block in self.transformer_blocks[start:end]: if use_grad_ckpt: x = checkpoint(self.ckpt_wrapper(block), x, t, mask, rope, use_reentrant=False) else: x = block(x, t, mask=mask, rope=rope) + return x + def head(self, x, t, residual=None): + """`norm_out -> proj_out`, computed on every step, hits included.""" if self.long_skip_connection is not None: x = self.long_skip_connection(torch.cat((x, residual), dim=-1)) - x = self.norm_out(x, t) - output = self.proj_out(x) + return self.proj_out(x) - if infer: - return output, text_embed if static_cache is None else None, dt - else: - return output diff --git a/GPT_SoVITS/inference_cli.py b/GPT_SoVITS/inference_cli.py index 459a3d36..45e6d28a 100644 --- a/GPT_SoVITS/inference_cli.py +++ b/GPT_SoVITS/inference_cli.py @@ -3,6 +3,7 @@ import os import soundfile as sf from tools.i18n.i18n import I18nAuto +import GPT_SoVITS.inference_webui as webui_ui from GPT_SoVITS.inference_webui import change_gpt_weights, change_sovits_weights, get_tts_wav i18n = I18nAuto() @@ -67,9 +68,25 @@ def main(): help="Language of the target text", ) parser.add_argument("--output_path", required=True, help="Path to the output directory") + parser.add_argument( + "--cache_dit", + default="off", + choices=["off", "auto", "f4", "f8", "f16"], + help="cache-dit (DBCache) for the V3/V4/V5dev DiT, off by default; " + "fewer DiT forwards with near-identical audio", + ) + parser.add_argument("--dbcache_fn", type=int, default=-1, help="front blocks (default 8)") + parser.add_argument("--dbcache_threshold", type=float, default=0.02, help="residual-diff threshold") + parser.add_argument("--dbcache_warmup", type=int, default=8, help="full-compute steps first") + parser.add_argument("--dbcache_downsample", type=int, default=4, help="residual time downsample") args = parser.parse_args() + webui_ui.update_cache_dit( + args.cache_dit, args.dbcache_fn, args.dbcache_threshold, + args.dbcache_warmup, args.dbcache_downsample, + ) + synthesize( args.gpt_model, args.sovits_model, diff --git a/GPT_SoVITS/inference_webui.py b/GPT_SoVITS/inference_webui.py index d488f5d5..5c72a036 100644 --- a/GPT_SoVITS/inference_webui.py +++ b/GPT_SoVITS/inference_webui.py @@ -844,6 +844,36 @@ def audio_sr(audio, sr): # cache_tokens={}#暂未实现清理机制 cache = {} +# cache-dit (DBCache) for the DiT sampler: off by default. The UI writes here, get_tts_wav +# reads it; the GSV_DBCACHE* environment variables work as well. +CACHE_DIT = {"cache_dit": "off", "dbcache_fn": None, "dbcache_threshold": 0.02, + "dbcache_warmup": 8, "dbcache_downsample": 4} + + +def update_cache_dit(mode, fn, thr, warmup, ds): + CACHE_DIT["cache_dit"] = str(mode or "off") + CACHE_DIT["dbcache_fn"] = None if fn in (None, "") or int(fn) < 0 else int(fn) + CACHE_DIT["dbcache_threshold"] = float(thr) if thr not in (None, "") else 0.02 + CACHE_DIT["dbcache_warmup"] = int(warmup) if warmup not in (None, "") else 8 + CACHE_DIT["dbcache_downsample"] = int(ds) if ds not in (None, "") else 4 + + +def apply_cache_dit(vq_model, sample_steps, params=None): + """cache-dit (DBCache) switch; off unless configured (UI or GSV_DBCACHE* env).""" + cfm = getattr(vq_model, "cfm", None) + if cfm is None or not hasattr(cfm, "set_dbcache"): + return None + from module.cache_dit import config_from_params + + cfg = config_from_params(dict(params if params is not None else CACHE_DIT), + int(sample_steps), getattr(cfm, "cache_depth", 0) or None) + if cfg.enabled and str(getattr(vq_model, "version", "")) == "v5turbo" and not cfg.force: + cfg.fn = 0 # 4-step turbo: no gain and unstable quality, keep it off + cfm.set_dbcache(cfg) + if cfg.enabled: + print("cache-dit:", cfg) + return cfg + def get_tts_wav( ref_wav_path, @@ -872,6 +902,7 @@ def get_tts_wav( if cfg_rate is None: cfg_rate = 0.0 sample_steps, cfg_rate = int(sample_steps), float(cfg_rate) + apply_cache_dit(globals().get("vq_model"), sample_steps) if ref_wav_path: pass else: @@ -1356,6 +1387,26 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css value=(32 if model_version == "v3" else 8), ) ) + with gr.Accordion(i18n("DiT 加速 cache-dit(默认关闭)"), open=False): + with gr.Row(): + cache_dit_mode = gr.Dropdown( + label=i18n("cache-dit"), + choices=["off", "auto", "f4", "f8", "f16"], + value="off", scale=2, + ) + dbcache_fn = gr.Number(label="fn", value=-1, precision=0, scale=1) + dbcache_threshold = gr.Number(label="thr", value=0.02, scale=1) + dbcache_warmup = gr.Number(label="warmup", value=8, precision=0, scale=1) + dbcache_downsample = gr.Number(label="ds", value=4, precision=0, scale=1) + gr.Markdown(i18n("DBCache:只算前 fn 个 block,残差变化小于 thr 时跳过中间 block。" + "fn=-1 表示跟随模式(auto = 步数>=16 时 F8)。4 步的 v5turbo 不支持。")) + for _cd in (cache_dit_mode, dbcache_fn, dbcache_threshold, + dbcache_warmup, dbcache_downsample): + _cd.change( + update_cache_dit, + [cache_dit_mode, dbcache_fn, dbcache_threshold, + dbcache_warmup, dbcache_downsample], + ) if_sr_Checkbox = gr.Checkbox( label=i18n("v3输出如果觉得闷可以试试开超分"), value=False, diff --git a/GPT_SoVITS/inference_webui_fast.py b/GPT_SoVITS/inference_webui_fast.py index 2c0df38e..d1e4158f 100644 --- a/GPT_SoVITS/inference_webui_fast.py +++ b/GPT_SoVITS/inference_webui_fast.py @@ -73,6 +73,19 @@ language = os.environ.get("language", "Auto") language = sys.argv[-1] if sys.argv[-1] in scan_language_list() else language i18n = I18nAuto(language=language) +# cache-dit (DBCache) for the DiT sampler: off by default. The UI writes here, tts_infer +# forwards it to TTS_infer_pack; the GSV_DBCACHE* environment variables work too. +CACHE_DIT = {"cache_dit": "off", "dbcache_fn": None, "dbcache_threshold": 0.02, + "dbcache_warmup": 8, "dbcache_downsample": 4} + + +def update_cache_dit(mode, fn, thr, warmup, ds): + CACHE_DIT["cache_dit"] = str(mode or "off") + CACHE_DIT["dbcache_fn"] = None if fn in (None, "") or int(fn) < 0 else int(fn) + CACHE_DIT["dbcache_threshold"] = float(thr) if thr not in (None, "") else 0.02 + CACHE_DIT["dbcache_warmup"] = int(warmup) if warmup not in (None, "") else 8 + CACHE_DIT["dbcache_downsample"] = int(ds) if ds not in (None, "") else 4 + # os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1' # 确保直接启动推理UI时也能够设置。 @@ -209,6 +222,7 @@ def inference( "cfg_rate": float(cfg_rate), "super_sampling": super_sampling, } + inputs.update(CACHE_DIT) # cache-dit (DBCache), off unless the UI / env enables it try: for item in tts_pipeline.run(inputs): yield item, actual_seed @@ -421,6 +435,25 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css repetition_penalty = gr.Slider( minimum=0, maximum=2, step=0.05, label=i18n("重复惩罚"), value=1.35, interactive=True ) + with gr.Accordion(i18n("DiT 加速 cache-dit(默认关闭)"), open=False): + with gr.Row(): + cache_dit_mode = gr.Dropdown( + label=i18n("cache-dit"), choices=["off", "auto", "f4", "f8", "f16"], + value="off", interactive=True, scale=2, + ) + dbcache_fn = gr.Number(label="fn", value=-1, precision=0, scale=1) + dbcache_threshold = gr.Number(label="thr", value=0.02, scale=1) + dbcache_warmup = gr.Number(label="warmup", value=8, precision=0, scale=1) + dbcache_downsample = gr.Number(label="ds", value=4, precision=0, scale=1) + gr.Markdown(i18n("DBCache:只算前 fn 个 block,残差变化小于 thr 时跳过中间 block。" + "fn=-1 表示跟随模式(auto = 步数>=16 时 F8)。v5turbo 4 步不支持。")) + for _cd in (cache_dit_mode, dbcache_fn, dbcache_threshold, + dbcache_warmup, dbcache_downsample): + _cd.change( + update_cache_dit, + [cache_dit_mode, dbcache_fn, dbcache_threshold, + dbcache_warmup, dbcache_downsample], + ) with gr.Column(): with gr.Row(): diff --git a/GPT_SoVITS/module/cache_dit.py b/GPT_SoVITS/module/cache_dit.py new file mode 100644 index 00000000..3449a861 --- /dev/null +++ b/GPT_SoVITS/module/cache_dit.py @@ -0,0 +1,437 @@ +"""cache-dit / DBCache (three-segment block caching) for the DiT/CFM sampler. + +Follows vipshop/cache-dit (`CachedBlocks_Pattern_Base.forward`), adapted to the way this repo +samples: the block stack is split into three segments, and the middle one is skipped when the +front residual barely moved. + + front : trunk(x_in) -> blocks[0:Fn] -> x_F (every step) + R = x_F - trunk(x_in) Fn residual, the decision signal + fd = sum|R - R_prev| / (sum|R_prev| + eps) (relative -> scale free) + hit : x_mid = x_F + MnResidual_prev, then head only (norm_out -> proj_out) + miss : x_F -> blocks[Fn:DEPTH] -> x_M; MnResidual = x_M - x_F; R -> R_prev + head : x_mid -> norm_out -> proj_out -> vel (every step, hit included) + +A hit additionally requires `can_cache`: not in warmup, `fd < threshold`, and the three gates +`max_cached` (cached steps per sampling run), `max_cont` (consecutive hits) and `max_accum` +(accumulated residual drift). + +CFG. This repo runs the conditional and the unconditional branch as **two separate forward +passes** (`CFM.inference`), so each branch owns its own `R_prev`, `MnResidual`, hit/miss +counters and gate counters -- the equivalent of cache-dit's `enable_separate_cfg` plus +`cfg_diff_compute_separate`. Nothing is shared between the branches. + +Everything here is inert unless a `DBCacheConfig` with `fn > 0` is handed to `DiT.forward`; +`cache_dit="off"` keeps the sampler on the identical code path it had before this change. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, replace + +import torch + +__all__ = [ + "DBCacheConfig", + "DBCacheState", + "resolve_fn", + "config_from_params", + "MODES", +] + +MODES = ("off", "auto", "f4", "f8", "f16") + +# Defaults of the cache-dit reference recipe, and the `auto` policy used by the HTTP API +# (F8 from 16 sampling steps on, off below that). +DEFAULT_FN = 8 +DEFAULT_THRESHOLD = 0.02 +DEFAULT_WARMUP = 8 +DEFAULT_DOWNSAMPLE = 4 +AUTO_FN = 8 +AUTO_MIN_STEPS = 16 + +_EPS = 1e-6 + + +def resolve_fn(mode, steps=32, depth=None): + """`off|auto|f4|f8|f16` (or a plain integer) -> number of front blocks.""" + if mode is None: + return 0 + if isinstance(mode, bool): + mode = int(mode) + if isinstance(mode, (int, float)): + fn = int(mode) + else: + text = str(mode).strip().lower() + if text in ("", "off", "none", "0", "false", "no"): + fn = 0 + elif text in ("on", "true", "yes"): + fn = DEFAULT_FN + elif text == "auto": + fn = AUTO_FN if int(steps) >= AUTO_MIN_STEPS else 0 + elif text.startswith("f") and text[1:].isdigit(): + fn = int(text[1:]) + elif text.isdigit(): + fn = int(text) + else: + raise ValueError("unknown cache_dit mode %r (expected off|auto|f4|f8|f16)" % (mode,)) + if depth is not None: + fn = max(0, min(fn, int(depth) - 1)) + return fn + + +@dataclass +class DBCacheConfig: + """cache-dit knobs. `fn == 0` means off (the default).""" + + mode: str = "off" + fn: int = 0 + threshold: float = DEFAULT_THRESHOLD + warmup: int = DEFAULT_WARMUP + downsample: int = DEFAULT_DOWNSAMPLE + warmup_interval: int = 1 + max_cached: int = -1 + max_cont: int = -1 + max_accum: float = 0.0 + # fd is reduced over the whole sequence by default: the prompt/reference frames carry a + # nearly constant residual, so they dilute the ratio and threshold 0.02 stays usable. Set + # to True to reduce over the target frames only -- a ~3x stricter signal on this model, so + # thresholds are not transferable between the two conventions. + fd_ignore_prompt: bool = False + trace: bool = False + # expert override: allow cache-dit below 16 sampling steps (see docs in + # `DBCacheMixin.dbcache_config`: too few steps gain nothing and the quality risk is real) + force: bool = False + + @property + def enabled(self) -> bool: + return self.fn > 0 + + def clamped(self, depth: int | None = None) -> "DBCacheConfig": + if depth is None: + return self + return replace(self, fn=max(0, min(int(self.fn), int(depth) - 1))) + + def __str__(self) -> str: # for logs + if not self.enabled: + return "off" + return ("%s(fn=%d thr=%.3g warmup=%d ds=%d mc=%d mcont=%d macc=%.3g)" + % (self.mode, self.fn, self.threshold, self.warmup, self.downsample, + self.max_cached, self.max_cont, self.max_accum)) + + +def _get(params, key, env, default, cast): + """request dict first (API field names), then env, then default.""" + if params is not None: + for k in (key,): + if k in params and params[k] is not None and params[k] != "": + try: + return cast(params[k]) + except (TypeError, ValueError): + pass + val = os.environ.get(env) + if val is not None and val != "": + try: + return cast(val) + except (TypeError, ValueError): + pass + return default + + +def config_from_params(params=None, steps=32, depth=None) -> DBCacheConfig: + """Build the config from a request dict / env vars. + + Field names match the HTTP API of our C++ server: + cache_dit (alias dbcache) = off|auto|f4|f8|f16 + dbcache_fn, dbcache_threshold, dbcache_warmup, dbcache_downsample + dbcache_warmup_interval, dbcache_max_cached, dbcache_max_cont, dbcache_max_accum + Env fallbacks: GSV_DBCACHE / GSV_DBCACHE_FN / _THRESHOLD / _WARMUP / _DOWNSAMPLE / ... + """ + p = dict(params or {}) + mode = p.get("cache_dit", p.get("dbcache", os.environ.get("GSV_DBCACHE", "off"))) + mode = "off" if mode is None else str(mode).strip().lower() + + cfg = DBCacheConfig(mode=mode) + cfg.threshold = _get(p, "dbcache_threshold", "GSV_DBCACHE_THRESHOLD", cfg.threshold, float) + cfg.warmup = _get(p, "dbcache_warmup", "GSV_DBCACHE_WARMUP", cfg.warmup, int) + cfg.downsample = max(1, _get(p, "dbcache_downsample", "GSV_DBCACHE_DOWNSAMPLE", cfg.downsample, int)) + cfg.warmup_interval = max(1, _get(p, "dbcache_warmup_interval", "GSV_DBCACHE_WARMUP_INTERVAL", + cfg.warmup_interval, int)) + cfg.max_cached = _get(p, "dbcache_max_cached", "GSV_DBCACHE_MAX_CACHED", cfg.max_cached, int) + cfg.max_cont = _get(p, "dbcache_max_cont", "GSV_DBCACHE_MAX_CONT", cfg.max_cont, int) + cfg.max_accum = _get(p, "dbcache_max_accum", "GSV_DBCACHE_MAX_ACCUM", cfg.max_accum, float) + cfg.fd_ignore_prompt = bool(_get(p, "dbcache_fd_ignore_prompt", "GSV_DBCACHE_FD_IGNORE_PROMPT", + int(cfg.fd_ignore_prompt), int)) + cfg.trace = bool(_get(p, "dbcache_trace", "GSV_DBCACHE_TRACE", 0, int)) + cfg.force = bool(_get(p, "dbcache_force", "GSV_DBCACHE_FORCE", 0, int)) + + # an explicit fn wins over the mode + if p.get("dbcache_fn") not in (None, ""): + cfg.fn = int(p["dbcache_fn"]) + cfg.mode = "custom" + elif os.environ.get("GSV_DBCACHE_FN"): + cfg.fn = int(os.environ["GSV_DBCACHE_FN"]) + cfg.mode = "custom" + else: + cfg.fn = resolve_fn(mode, steps) + if depth is not None: + cfg.fn = max(0, min(cfg.fn, int(depth) - 1)) + return cfg + + +class DBCacheMixin: + """The CFM-side plumbing (`set_dbcache` / per-run state / hit counters). + + Mixed into `module.models.CFM` and `module.v5_inference.CFMV5`; both expose the DiT as + `self.estimator`. Defaults are class attributes, so `cache off` needs no `__init__` change + and the object layout of an existing checkpoint-compatible model is untouched. + """ + + dbcache_spec = None # DBCacheConfig or request dict of dbcache_* fields + last_dbcache_stats = None # stats of the most recent inference() call + _dbcache_cum = None # accumulated over all calls (reset by dbcache_stats) + + # ------------------------------------------------------------------ config + @property + def cache_depth(self): + return int(getattr(self.estimator, "depth", 0) or 0) + + def set_dbcache(self, spec=None, **kw): + """Enable/disable cache-dit for the next sampling runs. + + `spec` is either a `module.cache_dit.DBCacheConfig` or a dict using the C++ server's + field names (`cache_dit`/`dbcache` + `dbcache_fn|threshold|warmup|downsample|...`). + Nothing is applied retroactively: `inference()` resolves it per call. + """ + if isinstance(spec, DBCacheConfig): + self.dbcache_spec = spec + else: + merged = dict(spec or {}) + merged.update(kw) + cfg = merged.pop("config", None) + self.dbcache_spec = cfg if isinstance(cfg, DBCacheConfig) else merged + return self.dbcache_spec + + def dbcache_config(self, steps): + """Resolve the spec against the step count of the run that is about to start. + + With no explicit spec the env switch (`GSV_DBCACHE=off|auto|f4|f8|f16`) is honoured, + so every entry point (WebUI, api.py, api_v2.py, CLI, Colab) can enable cache-dit + without extra plumbing. Below 16 sampling steps (the 4-step v5turbo) it stays off: + the measured gain is negative and the quality is unstable there. + """ + depth = self.cache_depth or None + spec = self.dbcache_spec + if isinstance(spec, DBCacheConfig): + fn = spec.fn if spec.mode in ("custom", None) else resolve_fn(spec.mode, steps, depth) + cfg = DBCacheConfig(**{**spec.__dict__, + "fn": fn if depth is None else min(fn, depth - 1)}) + else: + cfg = config_from_params(spec, steps, depth) + if cfg.enabled and int(steps) < AUTO_MIN_STEPS and not cfg.force: + cfg.fn = 0 + return cfg + + # ------------------------------------------------------------------ per run + def _dbcache_begin(self, steps, seq_len, prompt_len, x_lens, device): + """Fresh state per sampling run (new request / new chunk).""" + cfg = self.dbcache_config(steps) + if not cfg.enabled: + self.last_dbcache_stats = None + return None + return DBCacheState(cfg, self.cache_depth).begin(seq_len, prompt_len, x_lens, device) + + def _dbcache_end(self, state): + if state is None: + return + st = state.stats() + self.last_dbcache_stats = st + cum = self._dbcache_cum + if cum is None: + cum = self._dbcache_cum = {"calls": 0, "steps": 0, "hits_pos": 0, "hits_neg": 0, + "misses_pos": 0, "misses_neg": 0} + cum["calls"] += 1 + for k in ("steps", "hits_pos", "hits_neg", "misses_pos", "misses_neg"): + cum[k] = cum.get(k, 0) + st[k] + cum["hits"] = cum["hits_pos"] + cum["hits_neg"] + cum["misses"] = cum["misses_pos"] + cum["misses_neg"] + cum["total"] = cum["hits"] + cum["misses"] + cum["hit_rate"] = cum["hits"] / cum["total"] if cum["total"] else 0.0 + cum["fn"] = st.get("fn") + cum["threshold"] = st.get("threshold") + cum["last"] = st + + def dbcache_stats(self, reset=True): + """Hit/miss counters accumulated since the last reset (None when cache-dit is off).""" + st = self._dbcache_cum + if reset: + self._dbcache_cum = None + return st + + +class _Branch: + """Per-CFG-branch bookkeeping.""" + + __slots__ = ("valid", "r_prev", "r_cur", "m_res", "hits", "misses", + "cached_steps", "cont", "acc", "last_fd") + + def __init__(self): + self.valid = False + self.r_prev = None + self.r_cur = None + self.m_res = None + self.hits = 0 + self.misses = 0 + self.cached_steps = 0 + self.cont = 0 + self.acc = 0.0 + self.last_fd = 0.0 + + def reset(self): + self.__init__() + + +class DBCacheState: + """State of one sampling run (`CFM.inference` = one run = one new request / chunk).""" + + def __init__(self, config: DBCacheConfig, depth: int, trace: bool | None = None): + self.cfg = config + self.depth = int(depth) + self.trace = config.trace if trace is None else trace + self.step = 0 + self.n_steps = 0 + self.branches = [_Branch(), _Branch()] + self.ds = 1 + self.note = "" # free-form tag for logs + + # ------------------------------------------------------------------ lifecycle + def begin(self, seq_len: int, prompt_len: int, x_lens=None, device=None): + """Reset for a new sampling run. Called once per `CFM.inference`.""" + for b in self.branches: + b.reset() + self.step = 0 + self.n_steps = 0 + ds = int(self.cfg.downsample) if self.cfg.downsample else 1 + self.ds = ds if ds >= 1 and seq_len % ds == 0 else 1 + self._build_frame_mask(seq_len, prompt_len, x_lens, device) + return self + + def _build_frame_mask(self, seq_len, prompt_len, x_lens, device): + """Frames used for fd: skip the prompt/reference rows and the padding.""" + self.mask = None + if not self.cfg.fd_ignore_prompt: + return + idx = torch.arange(seq_len, device=device) + mask = idx.unsqueeze(0) >= int(max(0, prompt_len)) + if x_lens is not None: + lens = torch.as_tensor(x_lens, device=device).reshape(-1, 1) + if lens.shape[0] == 1 and seq_len >= int(lens.max()): + mask = mask & (idx.unsqueeze(0) < lens) + mask = mask.to(torch.float32) + if self.ds > 1: + mask = mask[:, :: self.ds] + mask = mask.unsqueeze(-1) # (b, n/ds, 1) + if float(mask.sum()) <= 0.0: # degenerate (all-prompt) -> fall back to everything + mask = None + self.mask = mask + + def step_begin(self, step: int): + """`db_set_step(j)`: current diffusion step, drives warmup and the trace.""" + self.step = int(step) + self.n_steps += 1 + + def in_warmup(self) -> bool: + if self.step >= self.cfg.warmup: + return False + return (self.step % max(1, self.cfg.warmup_interval)) == 0 + + # ------------------------------------------------------------------ per-step ops + @property + def enabled(self) -> bool: + return self.cfg.enabled + + @property + def fn(self) -> int: + return self.cfg.fn + + def branch(self, index: int) -> _Branch: + return self.branches[0 if index == 0 else 1] + + def front_update(self, index: int, x_f, trunk) -> float: + """Store x_F / R, update R_prev-based similarity, return `fd` (one host sync).""" + br = self.branch(index) + r = x_f.float() - trunk.float() # Fn residual; fp32 keeps fd stable + if self.ds > 1: + r = r[:, :: self.ds, :] + r_prev = br.r_prev + if r_prev is None or r_prev.shape != r.shape: + r_prev = torch.zeros_like(r) + num = (r - r_prev).abs() + den = r_prev.abs() + if self.mask is not None and self.mask.shape[0] == num.shape[0] \ + and self.mask.shape[1] == num.shape[1]: + num = num * self.mask + den = den * self.mask + num = num.sum(dim=(1, 2)) + den = den.sum(dim=(1, 2)) + _EPS + fd = float((num / den).max().item()) # the only device->host scalar per step + br.r_cur = r + br.last_fd = fd + return fd + + def can_cache(self, index: int, fd: float) -> bool: + br = self.branch(index) + cfg = self.cfg + hit = False + if br.valid and cfg.threshold > 0.0 and not self.in_warmup(): + ok = fd < cfg.threshold + if ok and cfg.max_cached >= 0 and br.cached_steps >= cfg.max_cached: + ok = False + if ok and cfg.max_cont >= 0 and br.cont >= cfg.max_cont: + ok = False + br.cont = 0 + if ok and cfg.max_accum > 0.0 and br.acc >= cfg.max_accum: + ok = False + hit = ok + if fd > 0.0: + br.acc += fd + return hit + + def mid_residual(self, index: int): + return self.branch(index).m_res + + def on_hit(self, index: int): + br = self.branch(index) + br.hits += 1 + br.cached_steps += 1 + br.cont += 1 + + def on_miss(self, index: int, x_m, x_f): + br = self.branch(index) + br.misses += 1 + br.valid = True + br.cont = 0 + br.m_res = (x_m - x_f) + if br.r_cur is not None: + if br.r_prev is None or br.r_prev.shape != br.r_cur.shape: + br.r_prev = br.r_cur.clone() + else: + br.r_prev.copy_(br.r_cur) + + # ------------------------------------------------------------------ reporting + def stats(self) -> dict: + out = {"steps": self.n_steps, "last_step": self.step, "fn": self.cfg.fn, + "threshold": self.cfg.threshold, "warmup": self.cfg.warmup, + "downsample": self.ds, "mode": self.cfg.mode} + for i, name in enumerate(("pos", "neg")): + br = self.branches[i] + out["hits_" + name] = br.hits + out["misses_" + name] = br.misses + out["fd_" + name] = br.last_fd + out["acc_" + name] = br.acc + out["hits"] = out["hits_pos"] + out["hits_neg"] + out["misses"] = out["misses_pos"] + out["misses_neg"] + total = out["hits"] + out["misses"] + out["total"] = total + out["hit_rate"] = (out["hits"] / total) if total else 0.0 + return out diff --git a/GPT_SoVITS/module/models.py b/GPT_SoVITS/module/models.py index 0e1f5acd..1b64be01 100644 --- a/GPT_SoVITS/module/models.py +++ b/GPT_SoVITS/module/models.py @@ -10,6 +10,7 @@ from torch.nn import functional as F from module import commons from module import modules from module import attentions +from module.cache_dit import DBCacheMixin from f5_tts.model import DiT from torch.nn import Conv1d, ConvTranspose1d, Conv2d from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm @@ -1099,7 +1100,7 @@ class SynthesizerTrn(nn.Module): return codes.transpose(0, 1) -class CFM(torch.nn.Module): +class CFM(DBCacheMixin, torch.nn.Module): def __init__(self, in_channels, dit): super().__init__() self.sigma_min = 1e-6 @@ -1138,7 +1139,11 @@ class CFM(torch.nn.Module): if self.use_conditioner_cache and self.use_static_cache else None) d_tensor = (torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * d if self.use_step_embedding else None) + # cache-dit: one fresh state per sampling run; None when the switch is off + dbcache = self._dbcache_begin(n_timesteps, T, prompt_len, x_lens, x.device) for j in range(n_timesteps): + if dbcache is not None: + dbcache.step_begin(j) t_tensor = torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * (t if self.use_step_embedding else j * d) step_condition = {"dt_base_bootstrap": d_tensor, "dt_cache": dt_cache} if self.use_step_embedding else {} v_pred, text_emb, dt = self.estimator( @@ -1153,6 +1158,8 @@ class CFM(torch.nn.Module): infer=True, text_cache=text_cache, static_cache=static_cache, + dbcache=dbcache, + dbcache_branch=0, **step_condition, ) v_pred = v_pred.transpose(2, 1) @@ -1174,6 +1181,8 @@ class CFM(torch.nn.Module): infer=True, text_cache=text_cfg_cache if self.cfg_drop_text else text_cache, static_cache=static_cache, + dbcache=dbcache, + dbcache_branch=1, **step_condition, ) neg = neg.transpose(2, 1) @@ -1183,6 +1192,7 @@ class CFM(torch.nn.Module): x = x + d * v_pred t = t + d x[:, :, :prompt_len] = 0 + self._dbcache_end(dbcache) return x def forward(self, x1, x_lens, prompt_lens, mu, use_grad_ckpt): diff --git a/api.py b/api.py index b5dbf2a2..946e2e01 100644 --- a/api.py +++ b/api.py @@ -829,6 +829,28 @@ splits = { } +def apply_cache_dit(vq_model, params: dict, sample_steps: int): + """cache-dit (DBCache) switch for the DiT sampler; off unless configured. + + Field names match api_v2.py: + ``cache_dit`` = off|auto|f4|f8|f16, ``dbcache_fn``, ``dbcache_threshold``, + ``dbcache_warmup``, ``dbcache_downsample``. The ``GSV_DBCACHE*`` env vars work too, + also on the paths that never go through these request fields. + """ + cfm = getattr(vq_model, "cfm", None) + if cfm is None or not hasattr(cfm, "set_dbcache"): + return None + from module.cache_dit import config_from_params + + cfg = config_from_params(params, int(sample_steps), getattr(cfm, "cache_depth", 0) or None) + if cfg.enabled and str(getattr(vq_model, "version", "")) == "v5turbo" and not cfg.force: + cfg.fn = 0 # 4-step turbo: no gain and unstable quality, keep it off + cfm.set_dbcache(cfg) + if cfg.enabled: + print("cache-dit:", cfg) + return cfg + + def get_tts_wav( ref_wav_path, prompt_text, @@ -844,6 +866,11 @@ def get_tts_wav( if_sr=False, spk="default", cfg_rate=None, + cache_dit=None, + dbcache_fn=None, + dbcache_threshold=None, + dbcache_warmup=None, + dbcache_downsample=None, ): infer_sovits = speaker_list[spk].sovits vq_model = infer_sovits.vq_model @@ -867,6 +894,12 @@ def get_tts_wav( if sample_steps not in [4, 8, 16, 32]: sample_steps = 8 + # cache-dit (DBCache) for the DiT sampler: off unless asked for + apply_cache_dit(vq_model, dict( + cache_dit=cache_dit, dbcache_fn=dbcache_fn, dbcache_threshold=dbcache_threshold, + dbcache_warmup=dbcache_warmup, dbcache_downsample=dbcache_downsample, + ), sample_steps) + if if_sr and version != "v3": if_sr = False @@ -1126,6 +1159,11 @@ def handle( sample_steps, if_sr, cfg_rate=None, + cache_dit=None, + dbcache_fn=None, + dbcache_threshold=None, + dbcache_warmup=None, + dbcache_downsample=None, ): if ( refer_wav_path == "" @@ -1163,6 +1201,11 @@ def handle( sample_steps, if_sr, cfg_rate=cfg_rate, + cache_dit=cache_dit, + dbcache_fn=dbcache_fn, + dbcache_threshold=dbcache_threshold, + dbcache_warmup=dbcache_warmup, + dbcache_downsample=dbcache_downsample, ), media_type="audio/" + media_type, ) @@ -1372,6 +1415,11 @@ async def tts_endpoint(request: Request): json_post_raw.get("sample_steps"), json_post_raw.get("if_sr", False), cfg_rate=json_post_raw.get("cfg_rate"), + cache_dit=json_post_raw.get("cache_dit"), + dbcache_fn=json_post_raw.get("dbcache_fn"), + dbcache_threshold=json_post_raw.get("dbcache_threshold"), + dbcache_warmup=json_post_raw.get("dbcache_warmup"), + dbcache_downsample=json_post_raw.get("dbcache_downsample"), ) @@ -1391,6 +1439,11 @@ async def tts_endpoint( sample_steps: int = None, if_sr: bool = False, cfg_rate: float = None, + cache_dit: str = None, + dbcache_fn: int = None, + dbcache_threshold: float = None, + dbcache_warmup: int = None, + dbcache_downsample: int = None, ): return handle( refer_wav_path, @@ -1407,6 +1460,11 @@ async def tts_endpoint( sample_steps, if_sr, cfg_rate=cfg_rate, + cache_dit=cache_dit, + dbcache_fn=dbcache_fn, + dbcache_threshold=dbcache_threshold, + dbcache_warmup=dbcache_warmup, + dbcache_downsample=dbcache_downsample, ) diff --git a/api_v2.py b/api_v2.py index 891ed295..36891ed2 100644 --- a/api_v2.py +++ b/api_v2.py @@ -44,6 +44,11 @@ POST: "streaming_mode": False, # bool or int. return audio chunk by chunk.T he available options are: 0,1,2,3 or True/False (0/False: Disabled | 1/True: Best Quality, Slowest response speed (old version streaming_mode) | 2: Medium Quality, Slow response speed | 3: Lower Quality, Faster response speed ) "overlap_length": 2, # int. overlap length of semantic tokens for streaming mode. "min_chunk_length": 16, # int. The minimum chunk length of semantic tokens for streaming mode. (affects audio chunk size) + "cache_dit": "off", # str.(optional) off|auto|f4|f8|f16. cache-dit/DBCache for the V3/V4/V5 DiT: fewer DiT forwards, near-identical audio. Off by default; 4-step runs are refused. + "dbcache_fn": 8, # int.(optional) front blocks (default 8; auto = F8 when sample_steps >= 16). + "dbcache_threshold": 0.02, # float.(optional) relative residual-diff threshold (default 0.02). + "dbcache_warmup": 8, # int.(optional) forced full-compute steps at the start (default 8). + "dbcache_downsample": 4, # int.(optional) time downsample of the cached Fn residual (default 4). "gpt_streaming": false, # bool. opt in to semantic-token streaming. "streaming_chunk_seconds": 2.0, "streaming_window_mode": "fixed", @@ -201,6 +206,13 @@ class TTS_Request(BaseModel): repetition_penalty: float = 1.35 sample_steps: Optional[int] = None cfg_rate: Optional[float] = None + # cache-dit (DBCache) for the V3/V4/V5 DiT sampler; off by default. Field names match + # the ones the TTS pipeline and the CLI use. + cache_dit: str = "off" # off | auto | f4 | f8 | f16 + dbcache_fn: Optional[int] = None # front blocks (overrides cache_dit) + dbcache_threshold: Optional[float] = None + dbcache_warmup: Optional[int] = None + dbcache_downsample: Optional[int] = None super_sampling: bool = False overlap_length: int = 2 min_chunk_length: int = 16 @@ -613,6 +625,11 @@ async def tts_get_endpoint( repetition_penalty: float = 1.35, sample_steps: Optional[int] = None, cfg_rate: Optional[float] = None, + cache_dit: str = "off", + dbcache_fn: Optional[int] = None, + dbcache_threshold: Optional[float] = None, + dbcache_warmup: Optional[int] = None, + dbcache_downsample: Optional[int] = None, super_sampling: bool = False, streaming_mode: Union[bool, int] = False, overlap_length: int = 2, @@ -649,6 +666,11 @@ async def tts_get_endpoint( "repetition_penalty": float(repetition_penalty), "sample_steps": None if sample_steps is None else int(sample_steps), "cfg_rate": None if cfg_rate is None else float(cfg_rate), + "cache_dit": cache_dit, + "dbcache_fn": dbcache_fn, + "dbcache_threshold": dbcache_threshold, + "dbcache_warmup": dbcache_warmup, + "dbcache_downsample": dbcache_downsample, "super_sampling": super_sampling, "overlap_length": int(overlap_length), "min_chunk_length": int(min_chunk_length),