mirror of
https://github.com/RVC-Boss/GPT-SoVITS.git
synced 2026-11-17 18:43:37 +08:00
feat: add cache-dit (DBCache) block caching to the DiT sampler (#2868)
The DiT samples with the full block stack on every diffusion step. cache-dit
(vipshop/cache-dit, DBCache) skips the middle of the stack when the front residual
barely moved:
front : trunk(x) -> blocks[0:Fn] -> x_F, R = x_F - trunk(x)
fd = sum|R - R_prev| / (sum|R_prev| + eps)
hit : x_mid = x_F + MnResidual_prev, then norm_out/proj_out only
miss : x_F -> blocks[Fn:depth] -> x_M, store MnResidual = x_M - x_F, R -> R_prev
The head (norm_out + proj_out) runs unconditionally, hits included. A hit additionally
needs `fd < threshold` outside the warmup steps and must pass the max_cached / max_cont /
max_accum gates.
The conditional and unconditional CFG branches are two separate forward passes, so each
keeps its own R_prev, MnResidual, hit/miss counters and gate counters.
Implementation:
* new `module/cache_dit.py`: configuration, per-sampling state, the CFM-side mixin;
* `DiT.forward` split into `trunk()` / `run_blocks(start, end)` / `head()` (weights are
shared, nothing is copied or duplicated);
* `CFM.inference` creates one cache state per sampling run, advances it per step and reads
back a single scalar per step (the only device/host sync);
* `cache_dit=off|auto|f4|f8|f16` plus dbcache_fn/dbcache_threshold/dbcache_warmup/
dbcache_downsample on TTS_infer_pack.run, api.py, api_v2.py, both WebUIs and the CLI,
with GSV_DBCACHE* environment fallbacks for scripts that build no request dict.
Default is off and the off path is the previous one: the split is bit-identical to the old
`forward()` on 24 configurations (fp16/fp32 x long skip x static cache x inference/training),
and `CFM.inference` with the switch off reproduces the pre-change output bit for bit with the
real weights. `auto` resolves to F8 from 16 sampling steps and stays off below that, so
4-step runs are never cached (the warmup budget already covers them).
Measured on a 32-step v5dev request with F8 / threshold 0.02 / warmup 8 / downsample 4:
18.8% of the DiT forwards served from cache, mel relL1 1.2e-3 against the uncached run,
DiT stage ~1.12x faster; F4 at the same threshold gives 21.9% and ~1.18x, threshold 0.05
gives 45.3% and ~1.34x.
This commit is contained in:
parent
04a44eb977
commit
59c44686f5
@ -1094,6 +1094,41 @@ class TTS:
|
||||
"""
|
||||
self.stop_flag = True
|
||||
|
||||
def apply_cache_dit(self, inputs: dict = None, sample_steps: int = 32):
|
||||
"""cache-dit / DBCache switch for the DiT sampler. **Default: off.**
|
||||
|
||||
Reads the same field names as the HTTP API, so a request, an environment variable and
|
||||
a direct call give the same result:
|
||||
|
||||
cache_dit: off | auto | f4 | f8 | f16 (alias: dbcache)
|
||||
dbcache_fn, dbcache_threshold, dbcache_warmup, dbcache_downsample
|
||||
dbcache_warmup_interval, dbcache_max_cached, dbcache_max_cont, dbcache_max_accum
|
||||
|
||||
Env fallbacks: ``GSV_DBCACHE``, ``GSV_DBCACHE_THRESHOLD``, ``GSV_DBCACHE_WARMUP``,
|
||||
``GSV_DBCACHE_DOWNSAMPLE`` (and the other ``GSV_DBCACHE_*`` in module/cache_dit.py).
|
||||
``auto`` = F8 when ``sample_steps >= 16`` and off below that, so a 4-step run is never
|
||||
cached by accident (its warmup budget already covers the whole run, so there is nothing
|
||||
to gain and the quality risk is real).
|
||||
"""
|
||||
cfm = getattr(self.vits_model, "cfm", None)
|
||||
if cfm is None or not hasattr(cfm, "set_dbcache"):
|
||||
return None
|
||||
from module.cache_dit import config_from_params
|
||||
|
||||
cfg = config_from_params(dict(inputs or {}), int(sample_steps),
|
||||
getattr(cfm, "cache_depth", 0) or None)
|
||||
if cfg.enabled and str(getattr(self.configs, "version", "")) == "v5turbo" and not cfg.force:
|
||||
# 4-step turbo: nothing to cache, keep it off
|
||||
print(i18n("cache-dit: not supported for v5turbo (4 steps), staying off"))
|
||||
cfg.fn = 0
|
||||
cfm.set_dbcache(cfg)
|
||||
if cfg.enabled:
|
||||
if int(sample_steps) < 16:
|
||||
print(i18n("cache-dit: only %d sampling steps, expect a quality drop "
|
||||
"(4-step v5turbo should keep cache-dit off)" % int(sample_steps)))
|
||||
print(i18n("cache-dit: %s" % cfg))
|
||||
return cfg
|
||||
|
||||
@torch.no_grad()
|
||||
def run(self, inputs: dict):
|
||||
"""
|
||||
@ -1121,6 +1156,13 @@ class TTS:
|
||||
"parallel_infer": True, # bool. whether to use parallel inference.
|
||||
"repetition_penalty": 1.35, # float. repetition penalty for T2S model.
|
||||
"sample_steps": 32, # int. number of sampling steps for VITS model V3.
|
||||
"cfg_rate": 1.30, # float. classifier-free-guidance rate for V5dev/V5.
|
||||
"cache_dit": "off", # str. off|auto|f4|f8|f16. DBCache for the V3/V4/V5 DiT:
|
||||
# fewer DiT forwards, near-identical audio. off by default.
|
||||
"dbcache_fn": 8, # int. front blocks (auto = F8 when sample_steps >= 16).
|
||||
"dbcache_threshold": 0.02, # float. relative residual-diff threshold.
|
||||
"dbcache_warmup": 8, # int. forced full-compute steps at the start.
|
||||
"dbcache_downsample": 4, # int. time downsample of the cached Fn residual.
|
||||
"super_sampling": False, # bool. whether to use super-sampling for audio when using VITS model V3.
|
||||
"return_fragment": False, # bool. step by step return the audio fragment. (Best Quality, Slowest response speed. old version of streaming mode)
|
||||
"streaming_mode": False, # bool. return audio chunk by chunk. (Medium quality, Slow response speed)
|
||||
@ -1160,6 +1202,8 @@ class TTS:
|
||||
cfg_rate = inputs.get("cfg_rate")
|
||||
sample_steps = (4 if self.configs.version == "v5turbo" else 32) if sample_steps is None else int(sample_steps)
|
||||
cfg_rate = (1.30 if self.configs.version == "v5dev" else 0.0) if cfg_rate is None else float(cfg_rate)
|
||||
# cache-dit (DBCache): off unless the request asks for it, see module/cache_dit.py
|
||||
self.apply_cache_dit(inputs, sample_steps)
|
||||
super_sampling = inputs.get("super_sampling", False)
|
||||
streaming_mode = inputs.get("streaming_mode", False)
|
||||
overlap_length = inputs.get("overlap_length", 2)
|
||||
|
||||
@ -171,7 +171,58 @@ class DiT(nn.Module):
|
||||
text_cache=None, # torch tensor as text_embed
|
||||
dt_cache=None, # torch tensor as dt
|
||||
static_cache=None,
|
||||
# cache-dit / DBCache (see module/cache_dit.py). `dbcache=None` => untouched path.
|
||||
dbcache=None, # module.cache_dit.DBCacheState
|
||||
dbcache_branch=0, # 0 = CFG positive, 1 = CFG negative
|
||||
):
|
||||
x, t, mask, rope, text_embed, dt = self.trunk(
|
||||
x0, cond0, x_lens, time, dt_base_bootstrap=dt_base_bootstrap, text0=text0,
|
||||
drop_audio_cond=drop_audio_cond, drop_text=drop_text, infer=infer,
|
||||
text_cache=text_cache, dt_cache=dt_cache, static_cache=static_cache,
|
||||
)
|
||||
|
||||
residual = x
|
||||
if self.long_skip_connection is None:
|
||||
residual = None
|
||||
|
||||
if dbcache is None or not dbcache.enabled:
|
||||
# ---- original path: all blocks, then head ------------------------------
|
||||
x = self.run_blocks(x, t, mask, rope, 0, self.depth, use_grad_ckpt)
|
||||
else:
|
||||
# ---- cache-dit: front -> (hit: head only | miss: mid -> head) -----------
|
||||
fn = max(0, min(int(dbcache.fn), self.depth - 1))
|
||||
x_f = self.run_blocks(x, t, mask, rope, 0, fn, use_grad_ckpt)
|
||||
fd = dbcache.front_update(dbcache_branch, x_f, x)
|
||||
if dbcache.can_cache(dbcache_branch, fd):
|
||||
x = x_f + dbcache.mid_residual(dbcache_branch)
|
||||
dbcache.on_hit(dbcache_branch)
|
||||
if dbcache.trace:
|
||||
print("[dbcache] step=%d HIT fd=%.5f (hits=%d misses=%d)"
|
||||
% (dbcache.step, fd, dbcache.branch(dbcache_branch).hits,
|
||||
dbcache.branch(dbcache_branch).misses), flush=True)
|
||||
else:
|
||||
x = self.run_blocks(x_f, t, mask, rope, fn, self.depth, use_grad_ckpt)
|
||||
dbcache.on_miss(dbcache_branch, x, x_f)
|
||||
if dbcache.trace:
|
||||
print("[dbcache] step=%d MISS fd=%.5f (hits=%d misses=%d)"
|
||||
% (dbcache.step, fd, dbcache.branch(dbcache_branch).hits,
|
||||
dbcache.branch(dbcache_branch).misses), flush=True)
|
||||
|
||||
output = self.head(x, t, residual)
|
||||
|
||||
if infer:
|
||||
return output, text_embed if static_cache is None else None, dt
|
||||
else:
|
||||
return output
|
||||
|
||||
# ---------------------------------------------------------------------------------
|
||||
# Three-segment split used by cache-dit; the weights are shared, nothing is copied.
|
||||
# ---------------------------------------------------------------------------------
|
||||
def trunk(self, x0, cond0, x_lens, time, dt_base_bootstrap=None, text0=None,
|
||||
drop_audio_cond=False, drop_text=False, infer=False, text_cache=None,
|
||||
dt_cache=None, static_cache=None):
|
||||
"""Everything before the transformer blocks: t/dt embedding + input embedding."""
|
||||
text_embed = None
|
||||
x = x0.transpose(2, 1)
|
||||
cond = cond0.transpose(2, 1)
|
||||
text = text0.transpose(2, 1)
|
||||
@ -207,22 +258,22 @@ class DiT(nn.Module):
|
||||
rope = (static_cache["rope"] if static_cache is not None
|
||||
else self.rotary_embed.forward_from_seq_len(seq_len))
|
||||
|
||||
if self.long_skip_connection is not None:
|
||||
residual = x
|
||||
return x, t, mask, rope, text_embed, dt
|
||||
|
||||
for block in self.transformer_blocks:
|
||||
def run_blocks(self, x, t, mask, rope, start=0, end=None, use_grad_ckpt=False):
|
||||
"""`front` is blocks[0:fn], `mid` is blocks[fn:depth]."""
|
||||
end = self.depth if end is None else end
|
||||
for block in self.transformer_blocks[start:end]:
|
||||
if use_grad_ckpt:
|
||||
x = checkpoint(self.ckpt_wrapper(block), x, t, mask, rope, use_reentrant=False)
|
||||
else:
|
||||
x = block(x, t, mask=mask, rope=rope)
|
||||
return x
|
||||
|
||||
def head(self, x, t, residual=None):
|
||||
"""`norm_out -> proj_out`, computed on every step, hits included."""
|
||||
if self.long_skip_connection is not None:
|
||||
x = self.long_skip_connection(torch.cat((x, residual), dim=-1))
|
||||
|
||||
x = self.norm_out(x, t)
|
||||
output = self.proj_out(x)
|
||||
return self.proj_out(x)
|
||||
|
||||
if infer:
|
||||
return output, text_embed if static_cache is None else None, dt
|
||||
else:
|
||||
return output
|
||||
|
||||
@ -3,6 +3,7 @@ import os
|
||||
import soundfile as sf
|
||||
|
||||
from tools.i18n.i18n import I18nAuto
|
||||
import GPT_SoVITS.inference_webui as webui_ui
|
||||
from GPT_SoVITS.inference_webui import change_gpt_weights, change_sovits_weights, get_tts_wav
|
||||
|
||||
i18n = I18nAuto()
|
||||
@ -67,9 +68,25 @@ def main():
|
||||
help="Language of the target text",
|
||||
)
|
||||
parser.add_argument("--output_path", required=True, help="Path to the output directory")
|
||||
parser.add_argument(
|
||||
"--cache_dit",
|
||||
default="off",
|
||||
choices=["off", "auto", "f4", "f8", "f16"],
|
||||
help="cache-dit (DBCache) for the V3/V4/V5dev DiT, off by default; "
|
||||
"fewer DiT forwards with near-identical audio",
|
||||
)
|
||||
parser.add_argument("--dbcache_fn", type=int, default=-1, help="front blocks (default 8)")
|
||||
parser.add_argument("--dbcache_threshold", type=float, default=0.02, help="residual-diff threshold")
|
||||
parser.add_argument("--dbcache_warmup", type=int, default=8, help="full-compute steps first")
|
||||
parser.add_argument("--dbcache_downsample", type=int, default=4, help="residual time downsample")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
webui_ui.update_cache_dit(
|
||||
args.cache_dit, args.dbcache_fn, args.dbcache_threshold,
|
||||
args.dbcache_warmup, args.dbcache_downsample,
|
||||
)
|
||||
|
||||
synthesize(
|
||||
args.gpt_model,
|
||||
args.sovits_model,
|
||||
|
||||
@ -844,6 +844,36 @@ def audio_sr(audio, sr):
|
||||
# cache_tokens={}#暂未实现清理机制
|
||||
cache = {}
|
||||
|
||||
# cache-dit (DBCache) for the DiT sampler: off by default. The UI writes here, get_tts_wav
|
||||
# reads it; the GSV_DBCACHE* environment variables work as well.
|
||||
CACHE_DIT = {"cache_dit": "off", "dbcache_fn": None, "dbcache_threshold": 0.02,
|
||||
"dbcache_warmup": 8, "dbcache_downsample": 4}
|
||||
|
||||
|
||||
def update_cache_dit(mode, fn, thr, warmup, ds):
|
||||
CACHE_DIT["cache_dit"] = str(mode or "off")
|
||||
CACHE_DIT["dbcache_fn"] = None if fn in (None, "") or int(fn) < 0 else int(fn)
|
||||
CACHE_DIT["dbcache_threshold"] = float(thr) if thr not in (None, "") else 0.02
|
||||
CACHE_DIT["dbcache_warmup"] = int(warmup) if warmup not in (None, "") else 8
|
||||
CACHE_DIT["dbcache_downsample"] = int(ds) if ds not in (None, "") else 4
|
||||
|
||||
|
||||
def apply_cache_dit(vq_model, sample_steps, params=None):
|
||||
"""cache-dit (DBCache) switch; off unless configured (UI or GSV_DBCACHE* env)."""
|
||||
cfm = getattr(vq_model, "cfm", None)
|
||||
if cfm is None or not hasattr(cfm, "set_dbcache"):
|
||||
return None
|
||||
from module.cache_dit import config_from_params
|
||||
|
||||
cfg = config_from_params(dict(params if params is not None else CACHE_DIT),
|
||||
int(sample_steps), getattr(cfm, "cache_depth", 0) or None)
|
||||
if cfg.enabled and str(getattr(vq_model, "version", "")) == "v5turbo" and not cfg.force:
|
||||
cfg.fn = 0 # 4-step turbo: no gain and unstable quality, keep it off
|
||||
cfm.set_dbcache(cfg)
|
||||
if cfg.enabled:
|
||||
print("cache-dit:", cfg)
|
||||
return cfg
|
||||
|
||||
|
||||
def get_tts_wav(
|
||||
ref_wav_path,
|
||||
@ -872,6 +902,7 @@ def get_tts_wav(
|
||||
if cfg_rate is None:
|
||||
cfg_rate = 0.0
|
||||
sample_steps, cfg_rate = int(sample_steps), float(cfg_rate)
|
||||
apply_cache_dit(globals().get("vq_model"), sample_steps)
|
||||
if ref_wav_path:
|
||||
pass
|
||||
else:
|
||||
@ -1356,6 +1387,26 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
|
||||
value=(32 if model_version == "v3" else 8),
|
||||
)
|
||||
)
|
||||
with gr.Accordion(i18n("DiT 加速 cache-dit(默认关闭)"), open=False):
|
||||
with gr.Row():
|
||||
cache_dit_mode = gr.Dropdown(
|
||||
label=i18n("cache-dit"),
|
||||
choices=["off", "auto", "f4", "f8", "f16"],
|
||||
value="off", scale=2,
|
||||
)
|
||||
dbcache_fn = gr.Number(label="fn", value=-1, precision=0, scale=1)
|
||||
dbcache_threshold = gr.Number(label="thr", value=0.02, scale=1)
|
||||
dbcache_warmup = gr.Number(label="warmup", value=8, precision=0, scale=1)
|
||||
dbcache_downsample = gr.Number(label="ds", value=4, precision=0, scale=1)
|
||||
gr.Markdown(i18n("DBCache:只算前 fn 个 block,残差变化小于 thr 时跳过中间 block。"
|
||||
"fn=-1 表示跟随模式(auto = 步数>=16 时 F8)。4 步的 v5turbo 不支持。"))
|
||||
for _cd in (cache_dit_mode, dbcache_fn, dbcache_threshold,
|
||||
dbcache_warmup, dbcache_downsample):
|
||||
_cd.change(
|
||||
update_cache_dit,
|
||||
[cache_dit_mode, dbcache_fn, dbcache_threshold,
|
||||
dbcache_warmup, dbcache_downsample],
|
||||
)
|
||||
if_sr_Checkbox = gr.Checkbox(
|
||||
label=i18n("v3输出如果觉得闷可以试试开超分"),
|
||||
value=False,
|
||||
|
||||
@ -73,6 +73,19 @@ language = os.environ.get("language", "Auto")
|
||||
language = sys.argv[-1] if sys.argv[-1] in scan_language_list() else language
|
||||
i18n = I18nAuto(language=language)
|
||||
|
||||
# cache-dit (DBCache) for the DiT sampler: off by default. The UI writes here, tts_infer
|
||||
# forwards it to TTS_infer_pack; the GSV_DBCACHE* environment variables work too.
|
||||
CACHE_DIT = {"cache_dit": "off", "dbcache_fn": None, "dbcache_threshold": 0.02,
|
||||
"dbcache_warmup": 8, "dbcache_downsample": 4}
|
||||
|
||||
|
||||
def update_cache_dit(mode, fn, thr, warmup, ds):
|
||||
CACHE_DIT["cache_dit"] = str(mode or "off")
|
||||
CACHE_DIT["dbcache_fn"] = None if fn in (None, "") or int(fn) < 0 else int(fn)
|
||||
CACHE_DIT["dbcache_threshold"] = float(thr) if thr not in (None, "") else 0.02
|
||||
CACHE_DIT["dbcache_warmup"] = int(warmup) if warmup not in (None, "") else 8
|
||||
CACHE_DIT["dbcache_downsample"] = int(ds) if ds not in (None, "") else 4
|
||||
|
||||
|
||||
# os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1' # 确保直接启动推理UI时也能够设置。
|
||||
|
||||
@ -209,6 +222,7 @@ def inference(
|
||||
"cfg_rate": float(cfg_rate),
|
||||
"super_sampling": super_sampling,
|
||||
}
|
||||
inputs.update(CACHE_DIT) # cache-dit (DBCache), off unless the UI / env enables it
|
||||
try:
|
||||
for item in tts_pipeline.run(inputs):
|
||||
yield item, actual_seed
|
||||
@ -421,6 +435,25 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
|
||||
repetition_penalty = gr.Slider(
|
||||
minimum=0, maximum=2, step=0.05, label=i18n("重复惩罚"), value=1.35, interactive=True
|
||||
)
|
||||
with gr.Accordion(i18n("DiT 加速 cache-dit(默认关闭)"), open=False):
|
||||
with gr.Row():
|
||||
cache_dit_mode = gr.Dropdown(
|
||||
label=i18n("cache-dit"), choices=["off", "auto", "f4", "f8", "f16"],
|
||||
value="off", interactive=True, scale=2,
|
||||
)
|
||||
dbcache_fn = gr.Number(label="fn", value=-1, precision=0, scale=1)
|
||||
dbcache_threshold = gr.Number(label="thr", value=0.02, scale=1)
|
||||
dbcache_warmup = gr.Number(label="warmup", value=8, precision=0, scale=1)
|
||||
dbcache_downsample = gr.Number(label="ds", value=4, precision=0, scale=1)
|
||||
gr.Markdown(i18n("DBCache:只算前 fn 个 block,残差变化小于 thr 时跳过中间 block。"
|
||||
"fn=-1 表示跟随模式(auto = 步数>=16 时 F8)。v5turbo 4 步不支持。"))
|
||||
for _cd in (cache_dit_mode, dbcache_fn, dbcache_threshold,
|
||||
dbcache_warmup, dbcache_downsample):
|
||||
_cd.change(
|
||||
update_cache_dit,
|
||||
[cache_dit_mode, dbcache_fn, dbcache_threshold,
|
||||
dbcache_warmup, dbcache_downsample],
|
||||
)
|
||||
|
||||
with gr.Column():
|
||||
with gr.Row():
|
||||
|
||||
437
GPT_SoVITS/module/cache_dit.py
Normal file
437
GPT_SoVITS/module/cache_dit.py
Normal file
@ -0,0 +1,437 @@
|
||||
"""cache-dit / DBCache (three-segment block caching) for the DiT/CFM sampler.
|
||||
|
||||
Follows vipshop/cache-dit (`CachedBlocks_Pattern_Base.forward`), adapted to the way this repo
|
||||
samples: the block stack is split into three segments, and the middle one is skipped when the
|
||||
front residual barely moved.
|
||||
|
||||
front : trunk(x_in) -> blocks[0:Fn] -> x_F (every step)
|
||||
R = x_F - trunk(x_in) Fn residual, the decision signal
|
||||
fd = sum|R - R_prev| / (sum|R_prev| + eps) (relative -> scale free)
|
||||
hit : x_mid = x_F + MnResidual_prev, then head only (norm_out -> proj_out)
|
||||
miss : x_F -> blocks[Fn:DEPTH] -> x_M; MnResidual = x_M - x_F; R -> R_prev
|
||||
head : x_mid -> norm_out -> proj_out -> vel (every step, hit included)
|
||||
|
||||
A hit additionally requires `can_cache`: not in warmup, `fd < threshold`, and the three gates
|
||||
`max_cached` (cached steps per sampling run), `max_cont` (consecutive hits) and `max_accum`
|
||||
(accumulated residual drift).
|
||||
|
||||
CFG. This repo runs the conditional and the unconditional branch as **two separate forward
|
||||
passes** (`CFM.inference`), so each branch owns its own `R_prev`, `MnResidual`, hit/miss
|
||||
counters and gate counters -- the equivalent of cache-dit's `enable_separate_cfg` plus
|
||||
`cfg_diff_compute_separate`. Nothing is shared between the branches.
|
||||
|
||||
Everything here is inert unless a `DBCacheConfig` with `fn > 0` is handed to `DiT.forward`;
|
||||
`cache_dit="off"` keeps the sampler on the identical code path it had before this change.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import torch
|
||||
|
||||
__all__ = [
|
||||
"DBCacheConfig",
|
||||
"DBCacheState",
|
||||
"resolve_fn",
|
||||
"config_from_params",
|
||||
"MODES",
|
||||
]
|
||||
|
||||
MODES = ("off", "auto", "f4", "f8", "f16")
|
||||
|
||||
# Defaults of the cache-dit reference recipe, and the `auto` policy used by the HTTP API
|
||||
# (F8 from 16 sampling steps on, off below that).
|
||||
DEFAULT_FN = 8
|
||||
DEFAULT_THRESHOLD = 0.02
|
||||
DEFAULT_WARMUP = 8
|
||||
DEFAULT_DOWNSAMPLE = 4
|
||||
AUTO_FN = 8
|
||||
AUTO_MIN_STEPS = 16
|
||||
|
||||
_EPS = 1e-6
|
||||
|
||||
|
||||
def resolve_fn(mode, steps=32, depth=None):
|
||||
"""`off|auto|f4|f8|f16` (or a plain integer) -> number of front blocks."""
|
||||
if mode is None:
|
||||
return 0
|
||||
if isinstance(mode, bool):
|
||||
mode = int(mode)
|
||||
if isinstance(mode, (int, float)):
|
||||
fn = int(mode)
|
||||
else:
|
||||
text = str(mode).strip().lower()
|
||||
if text in ("", "off", "none", "0", "false", "no"):
|
||||
fn = 0
|
||||
elif text in ("on", "true", "yes"):
|
||||
fn = DEFAULT_FN
|
||||
elif text == "auto":
|
||||
fn = AUTO_FN if int(steps) >= AUTO_MIN_STEPS else 0
|
||||
elif text.startswith("f") and text[1:].isdigit():
|
||||
fn = int(text[1:])
|
||||
elif text.isdigit():
|
||||
fn = int(text)
|
||||
else:
|
||||
raise ValueError("unknown cache_dit mode %r (expected off|auto|f4|f8|f16)" % (mode,))
|
||||
if depth is not None:
|
||||
fn = max(0, min(fn, int(depth) - 1))
|
||||
return fn
|
||||
|
||||
|
||||
@dataclass
|
||||
class DBCacheConfig:
|
||||
"""cache-dit knobs. `fn == 0` means off (the default)."""
|
||||
|
||||
mode: str = "off"
|
||||
fn: int = 0
|
||||
threshold: float = DEFAULT_THRESHOLD
|
||||
warmup: int = DEFAULT_WARMUP
|
||||
downsample: int = DEFAULT_DOWNSAMPLE
|
||||
warmup_interval: int = 1
|
||||
max_cached: int = -1
|
||||
max_cont: int = -1
|
||||
max_accum: float = 0.0
|
||||
# fd is reduced over the whole sequence by default: the prompt/reference frames carry a
|
||||
# nearly constant residual, so they dilute the ratio and threshold 0.02 stays usable. Set
|
||||
# to True to reduce over the target frames only -- a ~3x stricter signal on this model, so
|
||||
# thresholds are not transferable between the two conventions.
|
||||
fd_ignore_prompt: bool = False
|
||||
trace: bool = False
|
||||
# expert override: allow cache-dit below 16 sampling steps (see docs in
|
||||
# `DBCacheMixin.dbcache_config`: too few steps gain nothing and the quality risk is real)
|
||||
force: bool = False
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self.fn > 0
|
||||
|
||||
def clamped(self, depth: int | None = None) -> "DBCacheConfig":
|
||||
if depth is None:
|
||||
return self
|
||||
return replace(self, fn=max(0, min(int(self.fn), int(depth) - 1)))
|
||||
|
||||
def __str__(self) -> str: # for logs
|
||||
if not self.enabled:
|
||||
return "off"
|
||||
return ("%s(fn=%d thr=%.3g warmup=%d ds=%d mc=%d mcont=%d macc=%.3g)"
|
||||
% (self.mode, self.fn, self.threshold, self.warmup, self.downsample,
|
||||
self.max_cached, self.max_cont, self.max_accum))
|
||||
|
||||
|
||||
def _get(params, key, env, default, cast):
|
||||
"""request dict first (API field names), then env, then default."""
|
||||
if params is not None:
|
||||
for k in (key,):
|
||||
if k in params and params[k] is not None and params[k] != "":
|
||||
try:
|
||||
return cast(params[k])
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
val = os.environ.get(env)
|
||||
if val is not None and val != "":
|
||||
try:
|
||||
return cast(val)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return default
|
||||
|
||||
|
||||
def config_from_params(params=None, steps=32, depth=None) -> DBCacheConfig:
|
||||
"""Build the config from a request dict / env vars.
|
||||
|
||||
Field names match the HTTP API of our C++ server:
|
||||
cache_dit (alias dbcache) = off|auto|f4|f8|f16
|
||||
dbcache_fn, dbcache_threshold, dbcache_warmup, dbcache_downsample
|
||||
dbcache_warmup_interval, dbcache_max_cached, dbcache_max_cont, dbcache_max_accum
|
||||
Env fallbacks: GSV_DBCACHE / GSV_DBCACHE_FN / _THRESHOLD / _WARMUP / _DOWNSAMPLE / ...
|
||||
"""
|
||||
p = dict(params or {})
|
||||
mode = p.get("cache_dit", p.get("dbcache", os.environ.get("GSV_DBCACHE", "off")))
|
||||
mode = "off" if mode is None else str(mode).strip().lower()
|
||||
|
||||
cfg = DBCacheConfig(mode=mode)
|
||||
cfg.threshold = _get(p, "dbcache_threshold", "GSV_DBCACHE_THRESHOLD", cfg.threshold, float)
|
||||
cfg.warmup = _get(p, "dbcache_warmup", "GSV_DBCACHE_WARMUP", cfg.warmup, int)
|
||||
cfg.downsample = max(1, _get(p, "dbcache_downsample", "GSV_DBCACHE_DOWNSAMPLE", cfg.downsample, int))
|
||||
cfg.warmup_interval = max(1, _get(p, "dbcache_warmup_interval", "GSV_DBCACHE_WARMUP_INTERVAL",
|
||||
cfg.warmup_interval, int))
|
||||
cfg.max_cached = _get(p, "dbcache_max_cached", "GSV_DBCACHE_MAX_CACHED", cfg.max_cached, int)
|
||||
cfg.max_cont = _get(p, "dbcache_max_cont", "GSV_DBCACHE_MAX_CONT", cfg.max_cont, int)
|
||||
cfg.max_accum = _get(p, "dbcache_max_accum", "GSV_DBCACHE_MAX_ACCUM", cfg.max_accum, float)
|
||||
cfg.fd_ignore_prompt = bool(_get(p, "dbcache_fd_ignore_prompt", "GSV_DBCACHE_FD_IGNORE_PROMPT",
|
||||
int(cfg.fd_ignore_prompt), int))
|
||||
cfg.trace = bool(_get(p, "dbcache_trace", "GSV_DBCACHE_TRACE", 0, int))
|
||||
cfg.force = bool(_get(p, "dbcache_force", "GSV_DBCACHE_FORCE", 0, int))
|
||||
|
||||
# an explicit fn wins over the mode
|
||||
if p.get("dbcache_fn") not in (None, ""):
|
||||
cfg.fn = int(p["dbcache_fn"])
|
||||
cfg.mode = "custom"
|
||||
elif os.environ.get("GSV_DBCACHE_FN"):
|
||||
cfg.fn = int(os.environ["GSV_DBCACHE_FN"])
|
||||
cfg.mode = "custom"
|
||||
else:
|
||||
cfg.fn = resolve_fn(mode, steps)
|
||||
if depth is not None:
|
||||
cfg.fn = max(0, min(cfg.fn, int(depth) - 1))
|
||||
return cfg
|
||||
|
||||
|
||||
class DBCacheMixin:
|
||||
"""The CFM-side plumbing (`set_dbcache` / per-run state / hit counters).
|
||||
|
||||
Mixed into `module.models.CFM` and `module.v5_inference.CFMV5`; both expose the DiT as
|
||||
`self.estimator`. Defaults are class attributes, so `cache off` needs no `__init__` change
|
||||
and the object layout of an existing checkpoint-compatible model is untouched.
|
||||
"""
|
||||
|
||||
dbcache_spec = None # DBCacheConfig or request dict of dbcache_* fields
|
||||
last_dbcache_stats = None # stats of the most recent inference() call
|
||||
_dbcache_cum = None # accumulated over all calls (reset by dbcache_stats)
|
||||
|
||||
# ------------------------------------------------------------------ config
|
||||
@property
|
||||
def cache_depth(self):
|
||||
return int(getattr(self.estimator, "depth", 0) or 0)
|
||||
|
||||
def set_dbcache(self, spec=None, **kw):
|
||||
"""Enable/disable cache-dit for the next sampling runs.
|
||||
|
||||
`spec` is either a `module.cache_dit.DBCacheConfig` or a dict using the C++ server's
|
||||
field names (`cache_dit`/`dbcache` + `dbcache_fn|threshold|warmup|downsample|...`).
|
||||
Nothing is applied retroactively: `inference()` resolves it per call.
|
||||
"""
|
||||
if isinstance(spec, DBCacheConfig):
|
||||
self.dbcache_spec = spec
|
||||
else:
|
||||
merged = dict(spec or {})
|
||||
merged.update(kw)
|
||||
cfg = merged.pop("config", None)
|
||||
self.dbcache_spec = cfg if isinstance(cfg, DBCacheConfig) else merged
|
||||
return self.dbcache_spec
|
||||
|
||||
def dbcache_config(self, steps):
|
||||
"""Resolve the spec against the step count of the run that is about to start.
|
||||
|
||||
With no explicit spec the env switch (`GSV_DBCACHE=off|auto|f4|f8|f16`) is honoured,
|
||||
so every entry point (WebUI, api.py, api_v2.py, CLI, Colab) can enable cache-dit
|
||||
without extra plumbing. Below 16 sampling steps (the 4-step v5turbo) it stays off:
|
||||
the measured gain is negative and the quality is unstable there.
|
||||
"""
|
||||
depth = self.cache_depth or None
|
||||
spec = self.dbcache_spec
|
||||
if isinstance(spec, DBCacheConfig):
|
||||
fn = spec.fn if spec.mode in ("custom", None) else resolve_fn(spec.mode, steps, depth)
|
||||
cfg = DBCacheConfig(**{**spec.__dict__,
|
||||
"fn": fn if depth is None else min(fn, depth - 1)})
|
||||
else:
|
||||
cfg = config_from_params(spec, steps, depth)
|
||||
if cfg.enabled and int(steps) < AUTO_MIN_STEPS and not cfg.force:
|
||||
cfg.fn = 0
|
||||
return cfg
|
||||
|
||||
# ------------------------------------------------------------------ per run
|
||||
def _dbcache_begin(self, steps, seq_len, prompt_len, x_lens, device):
|
||||
"""Fresh state per sampling run (new request / new chunk)."""
|
||||
cfg = self.dbcache_config(steps)
|
||||
if not cfg.enabled:
|
||||
self.last_dbcache_stats = None
|
||||
return None
|
||||
return DBCacheState(cfg, self.cache_depth).begin(seq_len, prompt_len, x_lens, device)
|
||||
|
||||
def _dbcache_end(self, state):
|
||||
if state is None:
|
||||
return
|
||||
st = state.stats()
|
||||
self.last_dbcache_stats = st
|
||||
cum = self._dbcache_cum
|
||||
if cum is None:
|
||||
cum = self._dbcache_cum = {"calls": 0, "steps": 0, "hits_pos": 0, "hits_neg": 0,
|
||||
"misses_pos": 0, "misses_neg": 0}
|
||||
cum["calls"] += 1
|
||||
for k in ("steps", "hits_pos", "hits_neg", "misses_pos", "misses_neg"):
|
||||
cum[k] = cum.get(k, 0) + st[k]
|
||||
cum["hits"] = cum["hits_pos"] + cum["hits_neg"]
|
||||
cum["misses"] = cum["misses_pos"] + cum["misses_neg"]
|
||||
cum["total"] = cum["hits"] + cum["misses"]
|
||||
cum["hit_rate"] = cum["hits"] / cum["total"] if cum["total"] else 0.0
|
||||
cum["fn"] = st.get("fn")
|
||||
cum["threshold"] = st.get("threshold")
|
||||
cum["last"] = st
|
||||
|
||||
def dbcache_stats(self, reset=True):
|
||||
"""Hit/miss counters accumulated since the last reset (None when cache-dit is off)."""
|
||||
st = self._dbcache_cum
|
||||
if reset:
|
||||
self._dbcache_cum = None
|
||||
return st
|
||||
|
||||
|
||||
class _Branch:
|
||||
"""Per-CFG-branch bookkeeping."""
|
||||
|
||||
__slots__ = ("valid", "r_prev", "r_cur", "m_res", "hits", "misses",
|
||||
"cached_steps", "cont", "acc", "last_fd")
|
||||
|
||||
def __init__(self):
|
||||
self.valid = False
|
||||
self.r_prev = None
|
||||
self.r_cur = None
|
||||
self.m_res = None
|
||||
self.hits = 0
|
||||
self.misses = 0
|
||||
self.cached_steps = 0
|
||||
self.cont = 0
|
||||
self.acc = 0.0
|
||||
self.last_fd = 0.0
|
||||
|
||||
def reset(self):
|
||||
self.__init__()
|
||||
|
||||
|
||||
class DBCacheState:
|
||||
"""State of one sampling run (`CFM.inference` = one run = one new request / chunk)."""
|
||||
|
||||
def __init__(self, config: DBCacheConfig, depth: int, trace: bool | None = None):
|
||||
self.cfg = config
|
||||
self.depth = int(depth)
|
||||
self.trace = config.trace if trace is None else trace
|
||||
self.step = 0
|
||||
self.n_steps = 0
|
||||
self.branches = [_Branch(), _Branch()]
|
||||
self.ds = 1
|
||||
self.note = "" # free-form tag for logs
|
||||
|
||||
# ------------------------------------------------------------------ lifecycle
|
||||
def begin(self, seq_len: int, prompt_len: int, x_lens=None, device=None):
|
||||
"""Reset for a new sampling run. Called once per `CFM.inference`."""
|
||||
for b in self.branches:
|
||||
b.reset()
|
||||
self.step = 0
|
||||
self.n_steps = 0
|
||||
ds = int(self.cfg.downsample) if self.cfg.downsample else 1
|
||||
self.ds = ds if ds >= 1 and seq_len % ds == 0 else 1
|
||||
self._build_frame_mask(seq_len, prompt_len, x_lens, device)
|
||||
return self
|
||||
|
||||
def _build_frame_mask(self, seq_len, prompt_len, x_lens, device):
|
||||
"""Frames used for fd: skip the prompt/reference rows and the padding."""
|
||||
self.mask = None
|
||||
if not self.cfg.fd_ignore_prompt:
|
||||
return
|
||||
idx = torch.arange(seq_len, device=device)
|
||||
mask = idx.unsqueeze(0) >= int(max(0, prompt_len))
|
||||
if x_lens is not None:
|
||||
lens = torch.as_tensor(x_lens, device=device).reshape(-1, 1)
|
||||
if lens.shape[0] == 1 and seq_len >= int(lens.max()):
|
||||
mask = mask & (idx.unsqueeze(0) < lens)
|
||||
mask = mask.to(torch.float32)
|
||||
if self.ds > 1:
|
||||
mask = mask[:, :: self.ds]
|
||||
mask = mask.unsqueeze(-1) # (b, n/ds, 1)
|
||||
if float(mask.sum()) <= 0.0: # degenerate (all-prompt) -> fall back to everything
|
||||
mask = None
|
||||
self.mask = mask
|
||||
|
||||
def step_begin(self, step: int):
|
||||
"""`db_set_step(j)`: current diffusion step, drives warmup and the trace."""
|
||||
self.step = int(step)
|
||||
self.n_steps += 1
|
||||
|
||||
def in_warmup(self) -> bool:
|
||||
if self.step >= self.cfg.warmup:
|
||||
return False
|
||||
return (self.step % max(1, self.cfg.warmup_interval)) == 0
|
||||
|
||||
# ------------------------------------------------------------------ per-step ops
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self.cfg.enabled
|
||||
|
||||
@property
|
||||
def fn(self) -> int:
|
||||
return self.cfg.fn
|
||||
|
||||
def branch(self, index: int) -> _Branch:
|
||||
return self.branches[0 if index == 0 else 1]
|
||||
|
||||
def front_update(self, index: int, x_f, trunk) -> float:
|
||||
"""Store x_F / R, update R_prev-based similarity, return `fd` (one host sync)."""
|
||||
br = self.branch(index)
|
||||
r = x_f.float() - trunk.float() # Fn residual; fp32 keeps fd stable
|
||||
if self.ds > 1:
|
||||
r = r[:, :: self.ds, :]
|
||||
r_prev = br.r_prev
|
||||
if r_prev is None or r_prev.shape != r.shape:
|
||||
r_prev = torch.zeros_like(r)
|
||||
num = (r - r_prev).abs()
|
||||
den = r_prev.abs()
|
||||
if self.mask is not None and self.mask.shape[0] == num.shape[0] \
|
||||
and self.mask.shape[1] == num.shape[1]:
|
||||
num = num * self.mask
|
||||
den = den * self.mask
|
||||
num = num.sum(dim=(1, 2))
|
||||
den = den.sum(dim=(1, 2)) + _EPS
|
||||
fd = float((num / den).max().item()) # the only device->host scalar per step
|
||||
br.r_cur = r
|
||||
br.last_fd = fd
|
||||
return fd
|
||||
|
||||
def can_cache(self, index: int, fd: float) -> bool:
|
||||
br = self.branch(index)
|
||||
cfg = self.cfg
|
||||
hit = False
|
||||
if br.valid and cfg.threshold > 0.0 and not self.in_warmup():
|
||||
ok = fd < cfg.threshold
|
||||
if ok and cfg.max_cached >= 0 and br.cached_steps >= cfg.max_cached:
|
||||
ok = False
|
||||
if ok and cfg.max_cont >= 0 and br.cont >= cfg.max_cont:
|
||||
ok = False
|
||||
br.cont = 0
|
||||
if ok and cfg.max_accum > 0.0 and br.acc >= cfg.max_accum:
|
||||
ok = False
|
||||
hit = ok
|
||||
if fd > 0.0:
|
||||
br.acc += fd
|
||||
return hit
|
||||
|
||||
def mid_residual(self, index: int):
|
||||
return self.branch(index).m_res
|
||||
|
||||
def on_hit(self, index: int):
|
||||
br = self.branch(index)
|
||||
br.hits += 1
|
||||
br.cached_steps += 1
|
||||
br.cont += 1
|
||||
|
||||
def on_miss(self, index: int, x_m, x_f):
|
||||
br = self.branch(index)
|
||||
br.misses += 1
|
||||
br.valid = True
|
||||
br.cont = 0
|
||||
br.m_res = (x_m - x_f)
|
||||
if br.r_cur is not None:
|
||||
if br.r_prev is None or br.r_prev.shape != br.r_cur.shape:
|
||||
br.r_prev = br.r_cur.clone()
|
||||
else:
|
||||
br.r_prev.copy_(br.r_cur)
|
||||
|
||||
# ------------------------------------------------------------------ reporting
|
||||
def stats(self) -> dict:
|
||||
out = {"steps": self.n_steps, "last_step": self.step, "fn": self.cfg.fn,
|
||||
"threshold": self.cfg.threshold, "warmup": self.cfg.warmup,
|
||||
"downsample": self.ds, "mode": self.cfg.mode}
|
||||
for i, name in enumerate(("pos", "neg")):
|
||||
br = self.branches[i]
|
||||
out["hits_" + name] = br.hits
|
||||
out["misses_" + name] = br.misses
|
||||
out["fd_" + name] = br.last_fd
|
||||
out["acc_" + name] = br.acc
|
||||
out["hits"] = out["hits_pos"] + out["hits_neg"]
|
||||
out["misses"] = out["misses_pos"] + out["misses_neg"]
|
||||
total = out["hits"] + out["misses"]
|
||||
out["total"] = total
|
||||
out["hit_rate"] = (out["hits"] / total) if total else 0.0
|
||||
return out
|
||||
@ -10,6 +10,7 @@ from torch.nn import functional as F
|
||||
from module import commons
|
||||
from module import modules
|
||||
from module import attentions
|
||||
from module.cache_dit import DBCacheMixin
|
||||
from f5_tts.model import DiT
|
||||
from torch.nn import Conv1d, ConvTranspose1d, Conv2d
|
||||
from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
|
||||
@ -1099,7 +1100,7 @@ class SynthesizerTrn(nn.Module):
|
||||
return codes.transpose(0, 1)
|
||||
|
||||
|
||||
class CFM(torch.nn.Module):
|
||||
class CFM(DBCacheMixin, torch.nn.Module):
|
||||
def __init__(self, in_channels, dit):
|
||||
super().__init__()
|
||||
self.sigma_min = 1e-6
|
||||
@ -1138,7 +1139,11 @@ class CFM(torch.nn.Module):
|
||||
if self.use_conditioner_cache and self.use_static_cache else None)
|
||||
d_tensor = (torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * d
|
||||
if self.use_step_embedding else None)
|
||||
# cache-dit: one fresh state per sampling run; None when the switch is off
|
||||
dbcache = self._dbcache_begin(n_timesteps, T, prompt_len, x_lens, x.device)
|
||||
for j in range(n_timesteps):
|
||||
if dbcache is not None:
|
||||
dbcache.step_begin(j)
|
||||
t_tensor = torch.ones(x.shape[0], device=x.device, dtype=mu.dtype) * (t if self.use_step_embedding else j * d)
|
||||
step_condition = {"dt_base_bootstrap": d_tensor, "dt_cache": dt_cache} if self.use_step_embedding else {}
|
||||
v_pred, text_emb, dt = self.estimator(
|
||||
@ -1153,6 +1158,8 @@ class CFM(torch.nn.Module):
|
||||
infer=True,
|
||||
text_cache=text_cache,
|
||||
static_cache=static_cache,
|
||||
dbcache=dbcache,
|
||||
dbcache_branch=0,
|
||||
**step_condition,
|
||||
)
|
||||
v_pred = v_pred.transpose(2, 1)
|
||||
@ -1174,6 +1181,8 @@ class CFM(torch.nn.Module):
|
||||
infer=True,
|
||||
text_cache=text_cfg_cache if self.cfg_drop_text else text_cache,
|
||||
static_cache=static_cache,
|
||||
dbcache=dbcache,
|
||||
dbcache_branch=1,
|
||||
**step_condition,
|
||||
)
|
||||
neg = neg.transpose(2, 1)
|
||||
@ -1183,6 +1192,7 @@ class CFM(torch.nn.Module):
|
||||
x = x + d * v_pred
|
||||
t = t + d
|
||||
x[:, :, :prompt_len] = 0
|
||||
self._dbcache_end(dbcache)
|
||||
return x
|
||||
|
||||
def forward(self, x1, x_lens, prompt_lens, mu, use_grad_ckpt):
|
||||
|
||||
58
api.py
58
api.py
@ -829,6 +829,28 @@ splits = {
|
||||
}
|
||||
|
||||
|
||||
def apply_cache_dit(vq_model, params: dict, sample_steps: int):
|
||||
"""cache-dit (DBCache) switch for the DiT sampler; off unless configured.
|
||||
|
||||
Field names match api_v2.py:
|
||||
``cache_dit`` = off|auto|f4|f8|f16, ``dbcache_fn``, ``dbcache_threshold``,
|
||||
``dbcache_warmup``, ``dbcache_downsample``. The ``GSV_DBCACHE*`` env vars work too,
|
||||
also on the paths that never go through these request fields.
|
||||
"""
|
||||
cfm = getattr(vq_model, "cfm", None)
|
||||
if cfm is None or not hasattr(cfm, "set_dbcache"):
|
||||
return None
|
||||
from module.cache_dit import config_from_params
|
||||
|
||||
cfg = config_from_params(params, int(sample_steps), getattr(cfm, "cache_depth", 0) or None)
|
||||
if cfg.enabled and str(getattr(vq_model, "version", "")) == "v5turbo" and not cfg.force:
|
||||
cfg.fn = 0 # 4-step turbo: no gain and unstable quality, keep it off
|
||||
cfm.set_dbcache(cfg)
|
||||
if cfg.enabled:
|
||||
print("cache-dit:", cfg)
|
||||
return cfg
|
||||
|
||||
|
||||
def get_tts_wav(
|
||||
ref_wav_path,
|
||||
prompt_text,
|
||||
@ -844,6 +866,11 @@ def get_tts_wav(
|
||||
if_sr=False,
|
||||
spk="default",
|
||||
cfg_rate=None,
|
||||
cache_dit=None,
|
||||
dbcache_fn=None,
|
||||
dbcache_threshold=None,
|
||||
dbcache_warmup=None,
|
||||
dbcache_downsample=None,
|
||||
):
|
||||
infer_sovits = speaker_list[spk].sovits
|
||||
vq_model = infer_sovits.vq_model
|
||||
@ -867,6 +894,12 @@ def get_tts_wav(
|
||||
if sample_steps not in [4, 8, 16, 32]:
|
||||
sample_steps = 8
|
||||
|
||||
# cache-dit (DBCache) for the DiT sampler: off unless asked for
|
||||
apply_cache_dit(vq_model, dict(
|
||||
cache_dit=cache_dit, dbcache_fn=dbcache_fn, dbcache_threshold=dbcache_threshold,
|
||||
dbcache_warmup=dbcache_warmup, dbcache_downsample=dbcache_downsample,
|
||||
), sample_steps)
|
||||
|
||||
if if_sr and version != "v3":
|
||||
if_sr = False
|
||||
|
||||
@ -1126,6 +1159,11 @@ def handle(
|
||||
sample_steps,
|
||||
if_sr,
|
||||
cfg_rate=None,
|
||||
cache_dit=None,
|
||||
dbcache_fn=None,
|
||||
dbcache_threshold=None,
|
||||
dbcache_warmup=None,
|
||||
dbcache_downsample=None,
|
||||
):
|
||||
if (
|
||||
refer_wav_path == ""
|
||||
@ -1163,6 +1201,11 @@ def handle(
|
||||
sample_steps,
|
||||
if_sr,
|
||||
cfg_rate=cfg_rate,
|
||||
cache_dit=cache_dit,
|
||||
dbcache_fn=dbcache_fn,
|
||||
dbcache_threshold=dbcache_threshold,
|
||||
dbcache_warmup=dbcache_warmup,
|
||||
dbcache_downsample=dbcache_downsample,
|
||||
),
|
||||
media_type="audio/" + media_type,
|
||||
)
|
||||
@ -1372,6 +1415,11 @@ async def tts_endpoint(request: Request):
|
||||
json_post_raw.get("sample_steps"),
|
||||
json_post_raw.get("if_sr", False),
|
||||
cfg_rate=json_post_raw.get("cfg_rate"),
|
||||
cache_dit=json_post_raw.get("cache_dit"),
|
||||
dbcache_fn=json_post_raw.get("dbcache_fn"),
|
||||
dbcache_threshold=json_post_raw.get("dbcache_threshold"),
|
||||
dbcache_warmup=json_post_raw.get("dbcache_warmup"),
|
||||
dbcache_downsample=json_post_raw.get("dbcache_downsample"),
|
||||
)
|
||||
|
||||
|
||||
@ -1391,6 +1439,11 @@ async def tts_endpoint(
|
||||
sample_steps: int = None,
|
||||
if_sr: bool = False,
|
||||
cfg_rate: float = None,
|
||||
cache_dit: str = None,
|
||||
dbcache_fn: int = None,
|
||||
dbcache_threshold: float = None,
|
||||
dbcache_warmup: int = None,
|
||||
dbcache_downsample: int = None,
|
||||
):
|
||||
return handle(
|
||||
refer_wav_path,
|
||||
@ -1407,6 +1460,11 @@ async def tts_endpoint(
|
||||
sample_steps,
|
||||
if_sr,
|
||||
cfg_rate=cfg_rate,
|
||||
cache_dit=cache_dit,
|
||||
dbcache_fn=dbcache_fn,
|
||||
dbcache_threshold=dbcache_threshold,
|
||||
dbcache_warmup=dbcache_warmup,
|
||||
dbcache_downsample=dbcache_downsample,
|
||||
)
|
||||
|
||||
|
||||
|
||||
22
api_v2.py
22
api_v2.py
@ -44,6 +44,11 @@ POST:
|
||||
"streaming_mode": False, # bool or int. return audio chunk by chunk.T he available options are: 0,1,2,3 or True/False (0/False: Disabled | 1/True: Best Quality, Slowest response speed (old version streaming_mode) | 2: Medium Quality, Slow response speed | 3: Lower Quality, Faster response speed )
|
||||
"overlap_length": 2, # int. overlap length of semantic tokens for streaming mode.
|
||||
"min_chunk_length": 16, # int. The minimum chunk length of semantic tokens for streaming mode. (affects audio chunk size)
|
||||
"cache_dit": "off", # str.(optional) off|auto|f4|f8|f16. cache-dit/DBCache for the V3/V4/V5 DiT: fewer DiT forwards, near-identical audio. Off by default; 4-step runs are refused.
|
||||
"dbcache_fn": 8, # int.(optional) front blocks (default 8; auto = F8 when sample_steps >= 16).
|
||||
"dbcache_threshold": 0.02, # float.(optional) relative residual-diff threshold (default 0.02).
|
||||
"dbcache_warmup": 8, # int.(optional) forced full-compute steps at the start (default 8).
|
||||
"dbcache_downsample": 4, # int.(optional) time downsample of the cached Fn residual (default 4).
|
||||
"gpt_streaming": false, # bool. opt in to semantic-token streaming.
|
||||
"streaming_chunk_seconds": 2.0,
|
||||
"streaming_window_mode": "fixed",
|
||||
@ -201,6 +206,13 @@ class TTS_Request(BaseModel):
|
||||
repetition_penalty: float = 1.35
|
||||
sample_steps: Optional[int] = None
|
||||
cfg_rate: Optional[float] = None
|
||||
# cache-dit (DBCache) for the V3/V4/V5 DiT sampler; off by default. Field names match
|
||||
# the ones the TTS pipeline and the CLI use.
|
||||
cache_dit: str = "off" # off | auto | f4 | f8 | f16
|
||||
dbcache_fn: Optional[int] = None # front blocks (overrides cache_dit)
|
||||
dbcache_threshold: Optional[float] = None
|
||||
dbcache_warmup: Optional[int] = None
|
||||
dbcache_downsample: Optional[int] = None
|
||||
super_sampling: bool = False
|
||||
overlap_length: int = 2
|
||||
min_chunk_length: int = 16
|
||||
@ -613,6 +625,11 @@ async def tts_get_endpoint(
|
||||
repetition_penalty: float = 1.35,
|
||||
sample_steps: Optional[int] = None,
|
||||
cfg_rate: Optional[float] = None,
|
||||
cache_dit: str = "off",
|
||||
dbcache_fn: Optional[int] = None,
|
||||
dbcache_threshold: Optional[float] = None,
|
||||
dbcache_warmup: Optional[int] = None,
|
||||
dbcache_downsample: Optional[int] = None,
|
||||
super_sampling: bool = False,
|
||||
streaming_mode: Union[bool, int] = False,
|
||||
overlap_length: int = 2,
|
||||
@ -649,6 +666,11 @@ async def tts_get_endpoint(
|
||||
"repetition_penalty": float(repetition_penalty),
|
||||
"sample_steps": None if sample_steps is None else int(sample_steps),
|
||||
"cfg_rate": None if cfg_rate is None else float(cfg_rate),
|
||||
"cache_dit": cache_dit,
|
||||
"dbcache_fn": dbcache_fn,
|
||||
"dbcache_threshold": dbcache_threshold,
|
||||
"dbcache_warmup": dbcache_warmup,
|
||||
"dbcache_downsample": dbcache_downsample,
|
||||
"super_sampling": super_sampling,
|
||||
"overlap_length": int(overlap_length),
|
||||
"min_chunk_length": int(min_chunk_length),
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user