fix(streaming): 支持API v2语义流式文本切分 (#2850)

This commit is contained in:
游手好闲的言柒 2026-10-19 18:05:29 +08:00 committed by GitHub
parent 1d9fe63d28
commit b4570b876f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 116 additions and 22 deletions

View File

@ -1321,9 +1321,14 @@ class TTS:
)
else:
print(f"############ {i18n('切分文本')} ############")
# Preserve cut0's single-session behavior; other methods use the
# frontend's normal punctuation/newline and long-text handling.
texts = self.text_preprocessor.pre_seg_text(
text.replace("\n", " ") if gpt_streaming else text, text_lang, text_split_method)
if gpt_streaming and len(texts) > 1:
text.replace("\n", " ") if gpt_streaming and text_split_method == "cut0" else text,
text_lang,
text_split_method,
)
if gpt_streaming and text_split_method == "cut0" and len(texts) > 1:
raise ValueError("Text exceeds the single-session frontend limit; shorten the input")
data = []
for i in range(len(texts)):
@ -1361,6 +1366,7 @@ class TTS:
t2 = time.perf_counter()
streaming_tokens = streaming_audio = semantic_token_generator = None
streaming_reference = None
try:
print("############ 推理 ############")
###### inference ######
@ -1370,6 +1376,8 @@ class TTS:
is_first_package = True
output_sr = self.configs.sampling_rate if not self.configs.use_vocoder else self.vocoder_configs["sr"]
for item in data:
if gpt_streaming and stream_stop.is_set():
break
t3 = time.perf_counter()
if return_fragment or streaming_mode:
item = make_batch(item)
@ -1529,7 +1537,9 @@ class TTS:
batch_audio_fragment.append(audio_fragment)
elif gpt_streaming and self.configs.version in V5_VERSIONS:
reference = v5_reference(self)
if streaming_reference is None:
streaming_reference = v5_reference(self)
reference = streaming_reference
# Validate the acoustic context before starting the AR owner thread.
frames = round(stream_options["streaming_chunk_seconds"] * 100)
future = round(stream_options["streaming_lookahead_seconds"] * 100)
@ -1545,7 +1555,8 @@ class TTS:
streaming_audio.close()
streaming_tokens.close()
streaming_audio = streaming_tokens = None
yield output_sr, np.zeros(int(output_sr * fragment_interval), dtype=np.int16)
if not stream_stop.is_set() and fragment_interval:
yield output_sr, np.zeros(int(output_sr * fragment_interval), dtype=np.int16)
t4 = time.perf_counter()
else:
# refer_audio_spec: torch.Tensor = [
@ -1688,8 +1699,9 @@ class TTS:
if gpt_streaming:
semantic_token_generator.close()
streaming_tokens.close()
streaming_tokens = None
yield output_sr, np.zeros(int(output_sr*fragment_interval), dtype=np.int16)
semantic_token_generator = streaming_tokens = None
if not gpt_streaming or not stream_stop.is_set():
yield output_sr, np.zeros(int(output_sr*fragment_interval), dtype=np.int16)
t5 = time.perf_counter()
t_45 += t5 - t4

View File

@ -14,8 +14,10 @@ def validate_streaming_options(req, version, api_mode=False):
mode = req.get("streaming_mode", False)
if (api_mode and (type(mode) is not int or mode not in (2, 3))) or (not api_mode and mode not in (True, 2, 3)):
raise ValueError("gpt_streaming requires streaming_mode=2 or 3")
if req.get("text_split_method", "cut5") != "cut0" or req.get("batch_size", 1) != 1:
raise ValueError("gpt_streaming requires text_split_method=cut0 and batch_size=1")
# Text methods are validated by the existing frontend/API registry.
# Each resulting segment uses its own semantic stream, served sequentially.
if req.get("batch_size", 1) != 1:
raise ValueError("gpt_streaming requires batch_size=1")
if req.get("if_freeze", False):
raise ValueError("gpt_streaming cannot freeze semantic tokens")
if req.get("streaming_window_mode", "fixed") not in {"fixed", "hybrid"}:
@ -60,6 +62,7 @@ class PipelineStop:
def semantic_updates(pipeline, x, prompt, bert, req, stop):
"""Stream one text segment without cancelling the enclosing request."""
# A separate decoder leaves the ordinary context and cache unchanged.
acceleration = pipeline.streaming_accel
if acceleration is None:
@ -84,7 +87,8 @@ def semantic_updates(pipeline, x, prompt, bert, req, stop):
try:
yield from updates
finally:
stop.set()
# infer_stream closes/cancels this segment's handle. The TTS caller owns
# the request-wide stop signal, which must survive normal segment EOS.
updates.close()

View File

@ -54,7 +54,15 @@ POST:
}
```
When `gpt_streaming` is enabled, use model `v2ProPlus`, `v5turbo` or `v5dev`, integer `streaming_mode` 2 or 3, `text_split_method="cut0"`, `batch_size=1`, and `media_type="raw"` or `"wav"`. `streaming_window_mode` defaults to `fixed`; `hybrid` is supported for these models. On v2ProPlus, the first chunk retains `min_chunk_length` and later chunks can reach four times that token count; on V5, `streaming_chunk_seconds` controls the first chunk.
When `gpt_streaming` is enabled, use model `v2ProPlus`, `v5turbo` or `v5dev`,
integer `streaming_mode` 2 or 3, `batch_size=1`, and `media_type="raw"` or `"wav"`.
All existing text split methods (`cut0` through `cut5`) are supported: each text
segment streams through its own GPT session within one HTTP response.
`streaming_window_mode` defaults to `fixed`; `hybrid` is supported for these models.
On v2ProPlus, `min_chunk_length` sets the initial semantic block size; hybrid
windows can grow later blocks up to four times that token count. On V5,
`streaming_chunk_seconds` sets every fixed window, or the first hybrid window
of each text segment.
RESP:
成功: 直接返回 wav 音频流, http code 200

View File

@ -29,8 +29,9 @@ Use the existing `GET /tts` or `POST /tts`, with these options:
}
```
Both HTTP methods expose the same new defaults. `streaming_mode` must be the
integer 2 or 3; its existing meaning is preserved when the new switch is off.
Both HTTP methods expose the same defaults. With `gpt_streaming=true`,
`streaming_mode` must be the integer 2 or 3. With `gpt_streaming=false`,
`streaming_mode` uses the standard API behavior.
V2ProPlus uses `min_chunk_length` (16 semantic tokens by default),
`overlap_length` and the existing VITS streaming decoder. Mode 2 chooses a
silence boundary, while mode 3 uses fixed semantic chunks. Deltas never repeat
@ -43,9 +44,9 @@ Auxiliary references remain supported by the V2ProPlus path.
### Fixed and progressive hybrid windows
`streaming_window_mode` accepts `fixed` (the unchanged default) or `hybrid`.
Both keep the complete text in one GPT session and preserve the requested
sampling steps, CFG and speed.
`streaming_window_mode` accepts `fixed` (default) or `hybrid`.
Both preserve the selected text split method, sampling steps, CFG and speed.
Window scheduling starts afresh for each text segment.
For V5, `streaming_chunk_seconds` sets every acoustic window in fixed mode,
or only the first window in hybrid mode. Hybrid gradually grows later windows
@ -71,11 +72,23 @@ must still consume PCM promptly and use a persistent playback queue. This
reduces scheduling-induced stalls; it is not a guarantee against arbitrary
device or network pauses. All block decisions are local to the request.
The complete text enters one GPT session. Newlines are joined; inputs exceeding
the frontend single-session limit are rejected rather than silently split.
This is not incremental LLM text input. A separate AR decoder has 2048 context
positions; generated tokens are limited to 1500 and the remaining context.
Reaching the limit is an error rather than successful truncation.
### Text segmentation
All existing `text_split_method` options (`cut0` through `cut5`) work with
semantic streaming. `cut0` keeps the complete text in one GPT session, replacing
newlines with spaces. Other methods use the text frontend and open one GPT
session per resulting segment, sequentially within a single HTTP response.
Every segment emits its audio windows as they become available; the next segment
does not cancel or restart the HTTP stream. `fragment_interval` sets the
pause after each segment, including the last one. The request seeds
once and reuses the same reference and sampling options for all segments.
This is not incremental LLM text input. Each GPT session still has 2048 context
positions and at most 1500 generated tokens, further limited by the remaining
context. `cut0` rejects input exceeding the single-session frontend limit;
choose a splitting method for long text. Reaching a segment's token limit is an
error rather than successful truncation. Disconnect cleanup cancels the active
segment and prevents later segments from starting.
## Audio and resource lifetime

View File

@ -270,6 +270,21 @@ class APIV2StreamingTests(unittest.TestCase):
self.assertFalse(self.api.tts_pipeline.calls[-1]["gpt_streaming"])
self.assertEqual(self.api.tts_pipeline.calls[-1]["streaming_window_mode"], "hybrid")
def test_all_text_splitters_work_with_both_windows_over_get_and_post(self):
for version in ("v5turbo", "v5dev", "v2ProPlus"):
self.api.tts_pipeline.configs.version = version
for method in (f"cut{i}" for i in range(6)):
for window in ("fixed", "hybrid"):
with self.subTest(version=version, method=method, window=window):
request = self.request(text_split_method=method, streaming_mode=2,
gpt_streaming=True, streaming_window_mode=window)
post = self.client.post("/tts", json=request)
self.assertEqual(post.status_code, 200, post.text)
get = self.client.get("/tts", params=request)
self.assertEqual(get.status_code, 200, get.text)
self.assertEqual(self.api.tts_pipeline.calls[-1], self.api.tts_pipeline.calls[-2])
self.assertEqual(self.api.tts_pipeline.calls[-1]["text_split_method"], method)
def test_old_streaming_modes_keep_their_mapping(self):
expected = {
0: (False, False, False),
@ -301,7 +316,7 @@ class APIV2StreamingTests(unittest.TestCase):
{"streaming_mode": 4},
{"streaming_mode": 2.0},
{"streaming_mode": "2"},
{"streaming_mode": 2, "text_split_method": "cut5"},
{"streaming_mode": 2, "text_split_method": "not-a-method"},
{"streaming_mode": 2, "media_type": "ogg"},
{"streaming_mode": 2, "streaming_chunk_seconds": 0},
):

View File

@ -478,7 +478,7 @@ class ValidationTests(unittest.TestCase):
for version in ["v5turbo", "v5dev", "v2ProPlus"]:
validate_streaming_options(self.req(), version, api_mode=True)
for patch in [dict(streaming_mode=True), dict(streaming_mode=0), dict(streaming_mode=2.0), dict(batch_size=2),
dict(text_split_method="cut5"), dict(media_type="aac"),
dict(media_type="aac"),
dict(streaming_chunk_seconds=float("nan")), dict(speed_factor=0),
dict(cfg_rate=float("inf")), dict(overlap_length=0)]:
with self.subTest(patch=patch), self.assertRaises(ValueError):
@ -489,6 +489,48 @@ class ValidationTests(unittest.TestCase):
with self.assertRaises(ValueError):
validate_streaming_options(self.req(), "v4", api_mode=True)
def test_semantic_streaming_preserves_the_registered_text_splitters(self):
for version in ("v5turbo", "v5dev", "v2ProPlus"):
for method in (f"cut{i}" for i in range(6)):
with self.subTest(version=version, method=method):
validate_streaming_options(self.req(text_split_method=method), version, api_mode=True)
def test_finishing_one_segment_does_not_cancel_the_next(self):
closed = []
def infer_stream(*args, **kwargs):
index = len(closed)
try:
yield torch.tensor([index, 1]), False
yield torch.tensor([index, 1, 2]), True
finally:
closed.append(index)
pipeline = SimpleNamespace(streaming_accel=SimpleNamespace(infer_stream=infer_stream),
configs=SimpleNamespace(hz=50, max_sec=54))
stop = threading.Event()
for segment in range(3):
updates = list(_streaming_module.semantic_updates(pipeline, None, None, None, {}, stop))
self.assertEqual(updates[-1][0].tolist(), [segment, 1, 2])
self.assertFalse(stop.is_set())
self.assertEqual(closed, [0, 1, 2])
def test_closing_a_segment_releases_its_iterator(self):
closed = threading.Event()
def infer_stream(*args, **kwargs):
try:
yield torch.tensor([1]), False
yield torch.tensor([1, 2]), True
finally:
closed.set()
pipeline = SimpleNamespace(streaming_accel=SimpleNamespace(infer_stream=infer_stream),
configs=SimpleNamespace(hz=50, max_sec=54))
stop = threading.Event()
iterator = _streaming_module.semantic_updates(pipeline, None, None, None, {}, stop)
next(iterator)
stop.set()
iterator.close()
self.assertTrue(closed.is_set())
self.assertTrue(stop.is_set())
def test_pipeline_stop_and_disconnect_share_a_stop_signal(self):
pipeline = SimpleNamespace(stop_flag=False)
stop = PipelineStop(pipeline)