diff --git a/.github/build_windows_packages.ps1 b/.github/build_windows_packages.ps1 index 2e4acb2a..d527c5bd 100644 --- a/.github/build_windows_packages.ps1 +++ b/.github/build_windows_packages.ps1 @@ -115,12 +115,17 @@ Remove-Item $ffDir.FullName -Recurse -Force Write-Host "[INFO] Installing PyTorch..." & ".\runtime\python.exe" -m ensurepip & ".\runtime\python.exe" -m pip install --upgrade pip --no-warn-script-location + switch ($cuda) { "cu124" { - & ".\runtime\python.exe" -m pip install torch==2.6 torchaudio --index-url https://download.pytorch.org/whl/cu124 --no-warn-script-location + & ".\runtime\python.exe" -m pip install psutil ninja packaging wheel "setuptools>=42" --no-warn-script-location + & ".\runtime\python.exe" -m pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu124 --no-warn-script-location + & ".\runtime\python.exe" -m pip install flash-attn -i https://xxxxrt666.github.io/PIP-Index/ --no-build-isolation } "cu128" { + & ".\runtime\python.exe" -m pip install psutil ninja packaging wheel "setuptools>=42" --no-warn-script-location & ".\runtime\python.exe" -m pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu128 --no-warn-script-location + & ".\runtime\python.exe" -m pip install flash-attn -i https://xxxxrt666.github.io/PIP-Index/ --no-build-isolation } default { Write-Error "Unsupported CUDA version: $cuda" diff --git a/.github/workflows/build_windows_packages.yaml b/.github/workflows/build_windows_packages.yaml index 32861463..95e9d0ff 100644 --- a/.github/workflows/build_windows_packages.yaml +++ b/.github/workflows/build_windows_packages.yaml @@ -31,6 +31,15 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Install Windows CUDA 12.9 + if: ${{ runner.os == 'Windows' && matrix.torch_cuda == '12.8' }} + uses: Jimver/cuda-toolkit + id: cuda-toolkit-win-129 + with: + cuda: 12.9.1 + method: "network" + sub-packages: '["nvcc", "cudart", "visual_studio_integration"]' + - name: Run Build and Upload Script shell: pwsh run: | diff --git a/Docker/miniconda_install.sh b/Docker/miniconda_install.sh index 001a2a46..cf2e3d6f 100644 --- a/Docker/miniconda_install.sh +++ b/Docker/miniconda_install.sh @@ -23,8 +23,10 @@ fi if [ "$TARGETPLATFORM" = "linux/amd64" ]; then "${WGET_CMD[@]}" -O miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py311_25.3.1-1-Linux-x86_64.sh + SYSROOT_PKG="sysroot_linux-64>=2.28" elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then "${WGET_CMD[@]}" -O miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py311_25.3.1-1-Linux-aarch64.sh + SYSROOT_PKG="sysroot_linux-aarch64>=2.28" else exit 1 fi @@ -45,20 +47,36 @@ rm miniconda.sh source "$HOME/miniconda3/etc/profile.d/conda.sh" +"$HOME/miniconda3/bin/conda" init bash + +source "$HOME/.bashrc" + "$HOME/miniconda3/bin/conda" config --add channels conda-forge "$HOME/miniconda3/bin/conda" update -q --all -y 1>/dev/null "$HOME/miniconda3/bin/conda" install python=3.11 -q -y -"$HOME/miniconda3/bin/conda" install gcc=14 gxx ffmpeg cmake make unzip -q -y +"$HOME/miniconda3/bin/conda" install gcc=11 gxx ffmpeg cmake make unzip $SYSROOT_PKG "libstdcxx-ng>=11" -q -y if [ "$CUDA_VERSION" = "12.8" ]; then "$HOME/miniconda3/bin/pip" install torch torchaudio --no-cache-dir --index-url https://download.pytorch.org/whl/cu128 + "$HOME/miniconda3/bin/conda" install cuda-nvcc=12.8 -c nvidia elif [ "$CUDA_VERSION" = "12.6" ]; then - "$HOME/miniconda3/bin/pip" install torch==2.6 torchaudio --no-cache-dir --index-url https://download.pytorch.org/whl/cu126 + "$HOME/miniconda3/bin/pip" install torch torchaudio --no-cache-dir --index-url https://download.pytorch.org/whl/cu126 + "$HOME/miniconda3/bin/conda" install cuda-nvcc=12.6 -c nvidia fi +CUDA_PATH=$(echo "$HOME/miniconda3/targets/"*-linux | awk '{print $1}') + +export CUDA_HOME=$CUDA_PATH +export PATH="$HOME/miniconda3/bin:$PATH" +export PATH="$CUDA_HOME/bin:$PATH" +export PATH="$CUDA_HOME/nvvm/bin:$PATH" + +"$HOME/miniconda3/bin/pip" install psutil ninja packaging wheel "setuptools>=42" +"$HOME/miniconda3/bin/pip" install flash-attn -i https://xxxxrt666.github.io/PIP-Index/ --no-build-isolation + "$HOME/miniconda3/bin/pip" cache purge rm $LOG_PATH diff --git a/GPT_SoVITS/AR/text_processing/__init__.py b/GPT_SoVITS/AR/text_processing/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/GPT_SoVITS/AR/text_processing/phonemizer.py b/GPT_SoVITS/AR/text_processing/phonemizer.py deleted file mode 100644 index 1003040e..00000000 --- a/GPT_SoVITS/AR/text_processing/phonemizer.py +++ /dev/null @@ -1,72 +0,0 @@ -# modified from https://github.com/yangdongchao/SoundStorm/blob/master/soundstorm/s1/AR/text_processing/phonemizer.py -# reference: https://github.com/lifeiteng/vall-e -import itertools -import re -from typing import Dict -from typing import List - -import regex -from gruut import sentences -from gruut.const import Sentence -from gruut.const import Word -from AR.text_processing.symbols import SYMBOL_TO_ID - - -class GruutPhonemizer: - def __init__(self, language: str): - self._phonemizer = sentences - self.lang = language - self.symbol_to_id = SYMBOL_TO_ID - self._special_cases_dict: Dict[str] = { - r"\.\.\.": "... ", - ";": "; ", - ":": ": ", - ",": ", ", - r"\.": ". ", - "!": "! ", - r"\?": "? ", - "—": "—", - "…": "… ", - "«": "«", - "»": "»", - } - self._punctuation_regexp: str = rf"([{''.join(self._special_cases_dict.keys())}])" - - def _normalize_punctuation(self, text: str) -> str: - text = regex.sub(rf"\pZ+{self._punctuation_regexp}", r"\1", text) - text = regex.sub(rf"{self._punctuation_regexp}(\pL)", r"\1 \2", text) - text = regex.sub(r"\pZ+", r" ", text) - return text.strip() - - def _convert_punctuation(self, word: Word) -> str: - if not word.phonemes: - return "" - if word.phonemes[0] in ["‖", "|"]: - return word.text.strip() - - phonemes = "".join(word.phonemes) - # remove modifier characters ˈˌː with regex - phonemes = re.sub(r"[ˈˌː͡]", "", phonemes) - return phonemes.strip() - - def phonemize(self, text: str, espeak: bool = False) -> str: - text_to_phonemize: str = self._normalize_punctuation(text) - sents: List[Sentence] = [sent for sent in self._phonemizer(text_to_phonemize, lang="en-us", espeak=espeak)] - words: List[str] = [self._convert_punctuation(word) for word in itertools.chain(*sents)] - return " ".join(words) - - def transform(self, phonemes): - # convert phonemes to ids - # dictionary is in symbols.py - return [self.symbol_to_id[p] for p in phonemes if p in self.symbol_to_id.keys()] - - -if __name__ == "__main__": - phonemizer = GruutPhonemizer("en-us") - # text -> IPA - phonemes = phonemizer.phonemize("Hello, wor-ld ?") - print("phonemes:", phonemes) - print("len(phonemes):", len(phonemes)) - phoneme_ids = phonemizer.transform(phonemes) - print("phoneme_ids:", phoneme_ids) - print("len(phoneme_ids):", len(phoneme_ids)) diff --git a/GPT_SoVITS/AR/text_processing/symbols.py b/GPT_SoVITS/AR/text_processing/symbols.py deleted file mode 100644 index f7ef57fa..00000000 --- a/GPT_SoVITS/AR/text_processing/symbols.py +++ /dev/null @@ -1,12 +0,0 @@ -# modified from https://github.com/yangdongchao/SoundStorm/blob/master/soundstorm/s1/AR/text_processing/symbols.py -# reference: https://github.com/lifeiteng/vall-e -PAD = "_" -PUNCTUATION = ';:,.!?¡¿—…"«»“” ' -LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" -IPA_LETTERS = ( - "ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ" -) -SYMBOLS = [PAD] + list(PUNCTUATION) + list(LETTERS) + list(IPA_LETTERS) -SPACE_ID = SYMBOLS.index(" ") -SYMBOL_TO_ID = {s: i for i, s in enumerate(SYMBOLS)} -ID_TO_SYMBOL = {i: s for i, s in enumerate(SYMBOLS)} diff --git a/GPT_SoVITS/Accelerate/MLX/__init__.py b/GPT_SoVITS/Accelerate/MLX/__init__.py new file mode 100644 index 00000000..4042c0f1 --- /dev/null +++ b/GPT_SoVITS/Accelerate/MLX/__init__.py @@ -0,0 +1,11 @@ +import importlib.util + +if importlib.util.find_spec("mlx") is not None: + from .sample_funcs_mlx import sample_naive as sample_naive_mlx + from .t2s_engine_mlx import T2SEngine as T2SEngineMLX + + backends = ["mlx_static", "mlx_quantized", "mlx_varlen"] +else: + backends = [] + +__all__ = ["T2SEngineMLX", "sample_naive_mlx", "backends"] diff --git a/GPT_SoVITS/Accelerate/MLX/backends/mlx_quantized.py b/GPT_SoVITS/Accelerate/MLX/backends/mlx_quantized.py new file mode 100644 index 00000000..e66a37d8 --- /dev/null +++ b/GPT_SoVITS/Accelerate/MLX/backends/mlx_quantized.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +from typing import cast + +import mlx.core as mx +import mlx.nn as nn + +from ..structs_mlx import KVCacheQ +from ..t2s_model_abc import ( + AttentionABC, + KVCache, + KVCacheHND, + T2SDecoderABC, + TransformerBlockABC, + TransformerDecoderABC, +) + +Array = mx.array + + +class Attention(AttentionABC): + def __init__(self, n_head: int, hidden_dim: int, max_seq_length: int): + super().__init__(n_head, hidden_dim, max_seq_length) + self.kc_class = KVCacheHND + + @staticmethod + def quantized_scaled_dot_product_attention( + queries: Array, + q_keys: tuple[Array, Array, Array], + q_values: tuple[Array, Array, Array], + scale: float, + mask: Array, + group_size: int = 32, + bits: int = 8, + ) -> Array: + queries *= scale + + scores = mx.quantized_matmul(queries, *q_keys, transpose=True, group_size=group_size, bits=bits) + scores = mx.where(mask, scores, -mx.inf) + scores = mx.softmax(scores, axis=-1, precise=True) # type: ignore + out = mx.quantized_matmul(scores, *q_values, transpose=False, group_size=group_size, bits=bits) + + return out + + def __call__(self, x: Array, input_pos: Array, kv_cache: KVCache | KVCacheQ, cache_idx: Array, attn_mask: Array): + bsz, seqlen, _ = cast(tuple[int, ...], x.shape) + + q, k, v = self.in_proj(x).split(3, axis=-1) + + q, k, v = map(lambda x: x.reshape(bsz, seqlen, self.n_head, self.head_dim), (q, k, v)) + + q, k, v = map(lambda x: x.swapaxes(1, 2), (q, k, v)) + + kv_cache = self.kc_class.update_cache(input_pos, k, v, kv_cache, cache_idx) + assert len(kv_cache) == 2 + + max_idx = int(input_pos.max()) + + q, k, v = map(lambda x: x[..., :max_idx, :], (q, *kv_cache)) + + mask = attn_mask[..., :max_idx] + + attn = mx.fast.scaled_dot_product_attention(q, k, v, scale=self.scale, mask=mask) + + attn = attn.swapaxes(1, 2).reshape(bsz, seqlen, self.hidden_dim) + + attn = self.out_proj(attn) + + return attn + + # def __call__(self, x: Array, input_pos: Array, kv_cache: KVCache | KVCacheQ, cache_idx: Array, attn_mask: Array): + # bsz, seqlen, _ = cast(tuple[int, ...], x.shape) + + # q, k, v = self.in_proj(x).split(3, axis=-1) + + # q, k, v = map(lambda x: x.reshape(bsz, seqlen, self.n_head, self.head_dim), (q, k, v)) + + # q, k, v = map(lambda x: x.swapaxes(1, 2), (q, k, v)) + + # kv_cache = self.kc_class.update_cache(input_pos, k, v, kv_cache, cache_idx) + + # assert len(kv_cache) == 3 + # (k_q, k_s, k_b), (v_q, v_s, v_b), (group_size, bits) = kv_cache + + # k_q, k_s, k_b, v_q, v_s, v_b = map(lambda x: x[..., : int(input_pos.max()), :], (k_q, k_s, k_b, v_q, v_s, v_b)) + + # mask = attn_mask[..., : int(input_pos.max())] + + # attn = Attention.quantized_scaled_dot_product_attention( + # q, + # (k_q, k_s, k_b), + # (v_q, v_s, v_b), + # self.scale, + # mask, + # group_size, + # bits, + # ) + + # attn = attn.swapaxes(1, 2).reshape(bsz, seqlen, self.hidden_dim) + + # output = self.out_proj(attn) + + # return output + + +class TransformerBlock(TransformerBlockABC): + def __init__(self, n_head: int, ffn_dim: int, hidden_dim: int, max_seq_length: int, *args, **kwds) -> None: + super().__init__(n_head, ffn_dim, hidden_dim, max_seq_length, *args, **kwds) + + self.attention = Attention(n_head, hidden_dim, max_seq_length, *args, **kwds) + + +class TransformerDecoder(TransformerDecoderABC): + def __init__( + self, + hidden_dim: int, + n_layer: int, + n_head: int, + ffn_dim: int, + vocab_size: int, + max_seq_length: int, + max_batch_size: int, + *args, + **kwds, + ) -> None: + super().__init__( + hidden_dim, + n_layer, + n_head, + ffn_dim, + vocab_size, + max_seq_length, + max_batch_size, + *args, + **kwds, + ) + + self.layers = [ + TransformerBlock( + n_head, + ffn_dim, + hidden_dim, + max_seq_length, + *args, + **kwds, + ) + for _ in range(n_layer) + ] + + +class T2SDecoder(T2SDecoderABC): + def __init__( + self, + config: dict, + max_seq_length: int = 1800, + max_batch_size: int = 10, + ) -> None: + super().__init__(config, max_seq_length, max_batch_size) + + self.h = TransformerDecoder( + self.hidden_dim, self.n_layer, self.n_head, self.ffn_dim, self.vocab_size, max_seq_length, max_batch_size + ) + + self.kv_class = KVCacheHND + self.group_size = 32 + self.bits = 8 + + # def init_cache(self, bsz: int = 0): + # return super().init_cache(bsz, group_size=self.group_size, bits=self.bits) + + def quantized(self): + for layer in self.h.layers: + # nn.quantize(layer.feed_forward, self.group_size, self.bits) + nn.quantize(layer.attention, self.group_size, self.bits) diff --git a/GPT_SoVITS/Accelerate/MLX/backends/mlx_static.py b/GPT_SoVITS/Accelerate/MLX/backends/mlx_static.py new file mode 100644 index 00000000..6716fb32 --- /dev/null +++ b/GPT_SoVITS/Accelerate/MLX/backends/mlx_static.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from typing import cast + +import mlx.core as mx + +from ..structs_mlx import KVCache, KVCacheQ +from ..t2s_model_abc import ( + AttentionABC, + KVCacheHND, + T2SDecoderABC, + TransformerBlockABC, + TransformerDecoderABC, +) + +Array = mx.array + + +class Attention(AttentionABC): + def __init__(self, n_head: int, hidden_dim: int, max_seq_length: int): + super().__init__(n_head, hidden_dim, max_seq_length) + self.kc_class = KVCacheHND + + def __call__(self, x: Array, input_pos: Array, kv_cache: KVCache | KVCacheQ, cache_idx: Array, attn_mask: Array): + bsz, seqlen, _ = cast(tuple[int, ...], x.shape) + + q, k, v = self.in_proj(x).split(3, axis=-1) + + q, k, v = map(lambda x: x.reshape(bsz, seqlen, self.n_head, self.head_dim), (q, k, v)) + + q, k, v = map(lambda x: x.swapaxes(1, 2), (q, k, v)) + + kv_cache = self.kc_class.update_cache(input_pos, k, v, kv_cache, cache_idx) + assert len(kv_cache) == 2 + + k, v = kv_cache + + attn = mx.fast.scaled_dot_product_attention(q, k, v, scale=self.scale, mask=attn_mask) + + attn = attn.swapaxes(1, 2).reshape(bsz, seqlen, self.hidden_dim) + + attn = self.out_proj(attn) + + return attn + + +class TransformerBlock(TransformerBlockABC): + def __init__(self, n_head: int, ffn_dim: int, hidden_dim: int, max_seq_length: int) -> None: + super().__init__(n_head, ffn_dim, hidden_dim, max_seq_length) + + self.attention = Attention(n_head, hidden_dim, max_seq_length) + + +class TransformerDecoder(TransformerDecoderABC): + def __init__( + self, + hidden_dim: int, + n_layer: int, + n_head: int, + ffn_dim: int, + vocab_size: int, + max_seq_length: int, + max_batch_size: int, + ) -> None: + super().__init__( + hidden_dim, + n_layer, + n_head, + ffn_dim, + vocab_size, + max_seq_length, + max_batch_size, + ) + + self.layers = [ + TransformerBlock( + n_head, + ffn_dim, + hidden_dim, + max_seq_length, + ) + for _ in range(n_layer) + ] + + +class T2SDecoder(T2SDecoderABC): + def __init__( + self, + config: dict, + max_seq_length: int = 1800, + max_batch_size: int = 10, + ) -> None: + super().__init__(config, max_seq_length, max_batch_size) + + self.h = TransformerDecoder( + self.hidden_dim, self.n_layer, self.n_head, self.ffn_dim, self.vocab_size, max_seq_length, max_batch_size + ) + + self.kv_class = KVCacheHND diff --git a/GPT_SoVITS/Accelerate/MLX/backends/mlx_varlen.py b/GPT_SoVITS/Accelerate/MLX/backends/mlx_varlen.py new file mode 100644 index 00000000..3f07f6e2 --- /dev/null +++ b/GPT_SoVITS/Accelerate/MLX/backends/mlx_varlen.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from typing import cast + +import mlx.core as mx + +from ..structs_mlx import KVCache, KVCacheQ +from ..t2s_model_abc import ( + AttentionABC, + KVCacheHND, + T2SDecoderABC, + TransformerBlockABC, + TransformerDecoderABC, +) + +Array = mx.array + + +class Attention(AttentionABC): + def __init__(self, n_head: int, hidden_dim: int, max_seq_length: int): + super().__init__(n_head, hidden_dim, max_seq_length) + self.kc_class = KVCacheHND + + def __call__(self, x: Array, input_pos: Array, kv_cache: KVCache | KVCacheQ, cache_idx: Array, attn_mask: Array): + bsz, seqlen, _ = cast(tuple[int, ...], x.shape) + + q, k, v = self.in_proj(x).split(3, axis=-1) + + q, k, v = map(lambda x: x.reshape(bsz, seqlen, self.n_head, self.head_dim), (q, k, v)) + + q, k, v = map(lambda x: x.swapaxes(1, 2), (q, k, v)) + + kv_cache = self.kc_class.update_cache(input_pos, k, v, kv_cache, cache_idx) + assert len(kv_cache) == 2 + + max_idx = int(input_pos.max()) + + q, k, v = map(lambda x: x[..., :max_idx, :], (q, *kv_cache)) + + mask = attn_mask[..., :max_idx] + + attn = mx.fast.scaled_dot_product_attention(q, k, v, scale=self.scale, mask=mask) + + attn = attn.swapaxes(1, 2).reshape(bsz, seqlen, self.hidden_dim) + + attn = self.out_proj(attn) + + return attn + + +class TransformerBlock(TransformerBlockABC): + def __init__(self, n_head: int, ffn_dim: int, hidden_dim: int, max_seq_length: int) -> None: + super().__init__(n_head, ffn_dim, hidden_dim, max_seq_length) + + self.attention = Attention(n_head, hidden_dim, max_seq_length) + + +class TransformerDecoder(TransformerDecoderABC): + def __init__( + self, + hidden_dim: int, + n_layer: int, + n_head: int, + ffn_dim: int, + vocab_size: int, + max_seq_length: int, + max_batch_size: int, + ) -> None: + super().__init__( + hidden_dim, + n_layer, + n_head, + ffn_dim, + vocab_size, + max_seq_length, + max_batch_size, + ) + + self.layers = [ + TransformerBlock( + n_head, + ffn_dim, + hidden_dim, + max_seq_length, + ) + for _ in range(n_layer) + ] + + +class T2SDecoder(T2SDecoderABC): + def __init__( + self, + config: dict, + max_seq_length: int = 1800, + max_batch_size: int = 10, + ) -> None: + super().__init__(config, max_seq_length, max_batch_size) + + self.h = TransformerDecoder( + self.hidden_dim, self.n_layer, self.n_head, self.ffn_dim, self.vocab_size, max_seq_length, max_batch_size + ) + + self.kv_class = KVCacheHND diff --git a/GPT_SoVITS/Accelerate/MLX/sample_funcs_mlx.py b/GPT_SoVITS/Accelerate/MLX/sample_funcs_mlx.py new file mode 100644 index 00000000..5b0c5efd --- /dev/null +++ b/GPT_SoVITS/Accelerate/MLX/sample_funcs_mlx.py @@ -0,0 +1,64 @@ +from functools import partial +from typing import Protocol, cast + +import mlx.core as mx + +Array = mx.array + + +class SampleProtocolMLX(Protocol): + @staticmethod + def __call__( + logits: Array, + previous_tokens: Array, + temperature: float, + top_k: int, + top_p: float, + repetition_penalty: float, + ) -> Array: ... + + +class sample_naive(SampleProtocolMLX): + # @partial(mx.compile) + @staticmethod + def __call__( + logits, + previous_tokens, + temperature, + top_k, + top_p, + repetition_penalty, + ): + if temperature <= 1e-5: + probs = mx.softmax(logits, axis=-1) + return mx.argmax(probs, axis=-1, keepdims=True).astype(mx.int32) + + if repetition_penalty != 1.0: + batch_idx = mx.arange(cast(tuple[int, ...], previous_tokens.shape)[0]) + previous_tokens = previous_tokens.astype(mx.int64) + selected_logists = logits[batch_idx, previous_tokens] + selected_logists = mx.where( + selected_logists < 0, selected_logists * repetition_penalty, selected_logists / repetition_penalty + ) + logits[batch_idx, previous_tokens] = selected_logists + + sorted_indices = mx.argsort(-logits, axis=-1) + sorted_logits = mx.take_along_axis(logits, sorted_indices, axis=-1) + cum_probs = mx.cumsum(mx.softmax(sorted_logits, axis=-1), axis=-1) + sorted_indices_to_remove = cum_probs > top_p + sorted_indices_to_remove[:, -1] = False + indices_to_remove = mx.zeros_like(logits).astype(mx.bool_) + batch_indices = mx.arange(cast(tuple[int, ...], logits.shape)[0])[:, None] + indices_to_remove[batch_indices, sorted_indices] = sorted_indices_to_remove + logits = mx.where(indices_to_remove, -mx.inf, logits) + + logits = logits / temperature + + v = mx.topk(logits, top_k) + pivot = mx.expand_dims(v[:, 0], -1) + logits = mx.where(logits < pivot, -mx.inf, logits) + + gumbel_noise = mx.random.gumbel(shape=cast(tuple[int, ...], logits.shape), dtype=logits.dtype) + idx_next = mx.argmax(logits + gumbel_noise, axis=-1, keepdims=True).astype(mx.int32) + + return idx_next diff --git a/GPT_SoVITS/Accelerate/MLX/structs_mlx.py b/GPT_SoVITS/Accelerate/MLX/structs_mlx.py new file mode 100644 index 00000000..ab6643a0 --- /dev/null +++ b/GPT_SoVITS/Accelerate/MLX/structs_mlx.py @@ -0,0 +1,164 @@ +""" +Modified From https://github.com/XXXXRT666/GPT-SoVITS +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import List, MutableSequence, Protocol, TypeAlias, cast + +import mlx.core as mx +import torch + +from ..PyTorch.structs import T2SRequest, T2SResult +from .sample_funcs_mlx import SampleProtocolMLX, sample_naive + +Tensor = torch.Tensor +Array = mx.array + + +@dataclass(slots=True) +class T2SRequestMLX: + x: List[Array] + x_lens: Array + prompts: Array + bert_feature: List[Array] + valid_length: int + top_k: int = 5 + top_p: float = 1 + early_stop_num: int = -1 + temperature: float = 1.0 + repetition_penalty: float = 1.35 + + @classmethod + def from_torch(cls, request: T2SRequest) -> T2SRequestMLX: + x = list(map(lambda tensor: mx.array(tensor.cpu()), request.x)) + x_lens = mx.array(request.x_lens.cpu()) + prompts = mx.array(request.prompts.cpu()) + bert_feature = list(map(lambda tensor: mx.array(tensor.cpu()), request.bert_feature)) + + return cls( + x, + x_lens, + prompts, + bert_feature, + request.valid_length, + request.top_k, + request.top_p, + request.early_stop_num, + request.temperature, + request.repetition_penalty, + ) + + +KVCache: TypeAlias = tuple[Array, Array] +KVCacheQ: TypeAlias = tuple[tuple[Array, Array, Array], tuple[Array, Array, Array], tuple[int, int]] + + +class KVCacheProtocol(Protocol): + @staticmethod + def empty(kv_cache: KVCache | KVCacheQ) -> None: ... + + @staticmethod + def update_cache( + input_pos: Array, k_val: Array, v_val: Array, kv_cache: KVCache | KVCacheQ, cache_idx: Array + ) -> KVCache | KVCacheQ: ... + + @staticmethod + def prefill_kv(k_val: Array, v_val: Array, kv_cache: KVCache | KVCacheQ) -> None: ... + + @staticmethod + def init_cache( + batch_size: int, max_seq_length: int, n_heads: int, head_dim: int, dtype: mx.Dtype, *args, **kwds + ) -> KVCache | KVCacheQ: ... + + +class T2SDecoderProtocol(Protocol): + max_seq_length: int + EOS: int + n_head: int + + def embed(self, x: list[Array], y: Array, bert_features: list[Array]) -> Array: ... + + +class T2SEngineProtocol(Protocol): + def _handle_request(self, request: T2SRequest) -> tuple[list[Array], float]: ... + + def generate(self, request: T2SRequest) -> T2SResult: ... + + @staticmethod + def load_decoder( + weights_path: os.PathLike, max_batch_size: int = 1, implement: str = "MLX" + ) -> T2SDecoderProtocol: ... + + +class T2SSessionMLX: + def __init__( + self, + decoder: T2SDecoderProtocol, + request_torch: T2SRequest, + sample_func: type[SampleProtocolMLX] = sample_naive, + device: mx.Device = mx.Device(mx.cpu), + dtype: mx.Dtype = mx.float32, + ): + with mx.stream(device): + request = T2SRequestMLX.from_torch(request_torch) + + self.decoder = decoder + self.request = request + self.device = device + self.dtype = dtype + + bsz = len(request.x) + y_len: int = cast(tuple[int, ...], request.prompts.shape)[-1] + self.bsz = bsz + self.y_len = y_len + + # Cache + self.kv_cache: MutableSequence[KVCache | KVCacheQ] + self.sample = sample_func() + + # Forward args + self.x = [i.astype(mx.int32) for i in request.x] + self.x_lens = request.x_lens.astype(mx.int32) + self.y = mx.zeros((bsz, decoder.max_seq_length)).astype(mx.int32) + self.y[:, : cast(tuple[int, ...], request.prompts.shape)[-1]] = request.prompts.astype(mx.int32) + self.bert_feature = [i.astype(dtype) for i in request.bert_feature] + + self.prefill_len = self.x_lens + cast(tuple[int, ...], request.prompts.shape)[1] + + self.input_pos = mx.zeros_like(self.prefill_len) + self.input_pos += self.prefill_len + + # EOS + self.completed = mx.array([False] * len(self.x)).astype(mx.bool_) + self.y_results: List[Array] = [None] * len(self.x) # type: ignore + + self.xy_pos = decoder.embed(self.x, request.prompts, self.bert_feature) + + max_len = int(self.prefill_len.max(-1)) + attn_mask = mx.zeros(shape=(bsz, max_len, max_len), dtype=mx.bool_) + + for bs in range(bsz): + pos = int(self.x_lens[bs]) + seq_len = pos + y_len + + attn_mask[bs, :seq_len, :pos] = True + + ar_mask = ~mx.triu( + x=mx.ones( + shape=( + y_len, + y_len, + ), + dtype=mx.bool_, + ), + k=1, + ) + attn_mask[bs, pos:seq_len, pos:seq_len] = ar_mask + + attn_mask = mx.repeat(mx.expand_dims(attn_mask, 1), decoder.n_head, 1) + self.attn_mask = attn_mask + + mx.eval(self.attn_mask) diff --git a/GPT_SoVITS/Accelerate/MLX/t2s_engine_mlx.py b/GPT_SoVITS/Accelerate/MLX/t2s_engine_mlx.py new file mode 100644 index 00000000..98e4ec07 --- /dev/null +++ b/GPT_SoVITS/Accelerate/MLX/t2s_engine_mlx.py @@ -0,0 +1,232 @@ +import gc +import os +import time +import traceback +from typing import cast + +import mlx.core as mx +import torch +from rich.progress import BarColumn, Progress, TextColumn, TimeRemainingColumn + +from ..logger import console, logger +from ..PyTorch.structs import T2SEngineProtocol, T2SRequest +from .backends import mlx_quantized, mlx_static, mlx_varlen +from .structs_mlx import T2SResult, T2SSessionMLX +from .t2s_model_abc import T2SDecoderABC + +Array = mx.array +Tensor = torch.Tensor + + +class T2SEngine(T2SEngineProtocol): + def __init__( + self, + decoder_model: T2SDecoderABC, + device: mx.Device | str = mx.Device(mx.cpu), + dtype: torch.dtype | mx.Dtype = torch.float32, + ) -> None: + if isinstance(device, str): + match device: + case "mx.cpu": + device = mx.Device(mx.cpu) + case "mx.gpu": + device = mx.Device(mx.gpu) + + match dtype: + case torch.float32: + dtype = mx.float32 + case torch.float16: + dtype = mx.float16 + case torch.bfloat16: + dtype = mx.bfloat16 + + device = cast(mx.Device, device) + dtype = cast(mx.Dtype, dtype) + + assert device.type.value in {0, 1} + assert dtype in {mx.float16, mx.bfloat16, mx.float32} + + self.device = device + self.dtype = dtype + + mx.set_default_device(device) + decoder_model.set_dtype(self.dtype) + + self.decoder_model: T2SDecoderABC = decoder_model + # self.decoder_model.compile() + + def _handle_request(self, request: T2SRequest): + decoder = self.decoder_model + session = T2SSessionMLX(decoder, request, device=self.device, dtype=self.dtype) + batch_idx = mx.arange(session.bsz) + + t1 = 0.0 + infer_speed = 0.0 + infer_time = 0.0 + + with ( + mx.stream(session.device), + Progress( + TextColumn("[cyan]{task.description}"), + BarColumn(), + TextColumn("{task.completed}/{task.total}"), + TimeRemainingColumn(), + console=console, + transient=True, + ) as progress, + ): + max_token = min(1800 - int(session.input_pos.max()), 1500) + + task = progress.add_task("T2S Decoding", total=max_token) + for idx in range(1500): + progress.update(task, advance=1) + if idx == 0: + session.kv_cache = decoder.init_cache(session.bsz) + xy_dec = decoder.h.prefill( + session.xy_pos, + session.attn_mask, + session.kv_cache, + ) # bs, seq_len, embed_dim + xy_dec = xy_dec[None, batch_idx, session.input_pos - 1] + else: + args, kwds = decoder.pre_forward(session) + xy_dec = decoder.h( + session.input_pos, + session.xy_pos, + session.kv_cache, + batch_idx, + *args, + **kwds, + ) + + decoder.post_forward(idx, session) + logits = decoder.ar_predict_layer(xy_dec[:, -1]) + session.input_pos += 1 + + if idx == 0: + logits[:, -1] = -mx.inf + + samples = session.sample( + logits=logits, + previous_tokens=session.y[:, : session.y_len + idx], + top_k=request.top_k, + top_p=request.top_p, + repetition_penalty=request.repetition_penalty, + temperature=request.temperature, + ) + + session.y[batch_idx, session.y_len + idx] = samples + + argmax_token = mx.argmax(logits, axis=-1) + sample_token = samples.squeeze(1) + EOS_mask = (cast(Array, argmax_token == decoder.EOS)) | (sample_token == decoder.EOS) + + newly_done_mask = EOS_mask & (~session.completed) + newly_done_indices = mx.where(newly_done_mask, batch_idx, -1) + pos = mx.where(newly_done_indices != -1, batch_idx, session.bsz) + pos_sorted = mx.sort(pos, axis=0) + valid_count = session.bsz - mx.sum(cast(Array, pos_sorted == session.bsz)) + pos_final = pos_sorted[: int(valid_count)] + newly_done_indices = mx.expand_dims(newly_done_indices[pos_final], 0) + + if newly_done_indices.size > 0: + for i in newly_done_indices: + session.y_results[int(i)] = session.y[i, session.y_len : session.y_len + idx] + session.completed[newly_done_indices] = True + + if mx.all(session.completed).item(): + if session.y[:, session.y_len :].sum() == 0: + session.y_results = [mx.array([0]) for _ in range(session.bsz)] + logger.error("Bad Zero Prediction") + else: + logger.info( + f"T2S Decoding EOS {session.prefill_len.tolist().__str__().strip('[]')} -> {[cast(tuple[int, ...], i.shape)[-1] for i in session.y_results].__str__().strip('[]')}" + ) + logger.info(f"Infer Speed: {(idx - 1) / (time.perf_counter() - t1):.2f} token/s") + infer_time = time.perf_counter() - t1 + infer_speed = (idx - 1) / infer_time + break + + if (request.early_stop_num != -1 and idx >= request.early_stop_num) or idx == max_token - 1: + for j in range(session.bsz): + if not session.completed[j].item(): + session.y_results[j] = session.y[[j], session.y_len : session.y_len + 1499] + session.completed[j] = True + logger.error("Bad Full Prediction") + logger.info(f"Infer Speed: {(idx - 1) / (time.perf_counter() - t1):.2f} token/s") + infer_time = time.perf_counter() - t1 + infer_speed = (idx - 1) / infer_time + break + + y_emb = decoder.ar_audio_embedding(samples) + session.xy_pos = decoder.ar_audio_position(session.input_pos - session.x_lens, y_emb) + mx.eval(session.xy_pos, session.y) + + if idx == 1: + t1 = time.perf_counter() + + if idx % 100 == 0: + mx.clear_cache() + + match session.device: + case mx.gpu: + mx.clear_cache() + case mx.cpu: + gc.collect() + + result_mlx = session.y_results[: request.valid_length] + mx.eval(result_mlx) + result = [torch.tensor(k) for k in result_mlx] + return result, infer_speed, infer_time + + def generate(self, request: T2SRequest): + try: + result, infer_speed, infer_time = self._handle_request(request) + t2s_result = T2SResult(result=result, infer_speed=(infer_speed, infer_time), status="Success") + except Exception as e: + t2s_result = T2SResult(status="Error", exception=e, traceback=traceback.format_exc()) + return t2s_result + + @staticmethod + def replace_key(state_dict: dict[str, Tensor]): + state_dict_mlx: list[tuple[str, Array]] = [] + for key, value in state_dict.items(): + key = ( + key.replace("model.", "") + .replace("in_proj_", "in_proj.") + .replace("self_attn", "attention") + .replace("linear", "feed_forward.linear") + .replace("norm1", "attention_norm") + .replace("norm2", "ffn_norm") + ) + value_mlx = mx.array(value) + state_dict_mlx.append((key, value_mlx)) + return state_dict_mlx + + @staticmethod + def load_decoder(weights_path: os.PathLike, max_batch_size: int = 1, backend: str = "MLX-Varlen"): + logger.info(f"Loading Text2Semantic Weights from {weights_path} with {backend} Backend") + dict_s1 = torch.load(weights_path, map_location="cpu", weights_only=False, mmap=True) + config = dict_s1["config"] + match backend: + case "MLX-Varlen": + decoder_cls: type[T2SDecoderABC] = mlx_varlen.T2SDecoder + case "MLX-Static": + decoder_cls = mlx_static.T2SDecoder + case "MLX-Quantized": + decoder_cls = mlx_quantized.T2SDecoder + case _: + raise RuntimeError(f"Backend {backend} Not Found") + + decoder: T2SDecoderABC = decoder_cls(config, max_batch_size=max_batch_size) + state_dict = dict_s1["weight"] + state_dict_mlx = T2SEngine.replace_key(state_dict) + decoder.load_weights(state_dict_mlx) + decoder.eval() + mx.eval(decoder) + + if "Quantized" in backend and isinstance(decoder, mlx_quantized.T2SDecoder): + decoder.quantized() + mx.eval(decoder) + + return decoder diff --git a/GPT_SoVITS/Accelerate/MLX/t2s_model_abc.py b/GPT_SoVITS/Accelerate/MLX/t2s_model_abc.py new file mode 100644 index 00000000..fb295179 --- /dev/null +++ b/GPT_SoVITS/Accelerate/MLX/t2s_model_abc.py @@ -0,0 +1,530 @@ +from __future__ import annotations + +import math +from abc import ABC, abstractmethod +from typing import MutableSequence, cast + +import mlx.core as mx +import mlx.nn as nn + +from .structs_mlx import KVCache, KVCacheProtocol, KVCacheQ, T2SDecoderProtocol, T2SSessionMLX + +Array = mx.array + + +class TokenEmbedding(nn.Module): + def __init__( + self, + embedding_dim: int, + vocab_size: int, + ): + super().__init__() + + self.vocab_size = vocab_size + self.embedding_dim = embedding_dim + + self.word_embeddings = nn.Embedding(self.vocab_size, self.embedding_dim) + + @property + def weight(self): + return self.word_embeddings.weight + + def embedding(self, index: int): + return self.word_embeddings.weight[index : index + 1] + + def __call__(self, x: Array): + x = self.word_embeddings(x) + return x + + +class SinePositionalEmbedding(nn.Module): + def __init__( + self, + embedding_dim: int, + scale: bool = False, + max_batch_size: int = 10, + max_seq_len: int = 1800, + ): + super().__init__() + self.embedding_dim = embedding_dim + self.x_scale = math.sqrt(embedding_dim) if scale else 1.0 + self.alpha = mx.ones(1) + self.max_batch_size = max_batch_size + self.max_seq_len = max_seq_len + + self.reverse = False + self._pe = mx.zeros((max_batch_size, max_seq_len, embedding_dim)) + self.compute_pe() + + def compute_pe(self): + """Reset the positional encodings.""" + + if self.reverse: + position = mx.expand_dims(mx.arange(self.max_seq_len - 1, -1, -1.0), axis=1) + else: + position = mx.expand_dims(mx.arange(self.max_seq_len), axis=1) + div_term = mx.exp( + mx.arange( + 0, + self.embedding_dim, + 2, + ) + * -(math.log(10000.0) / self.embedding_dim) + ) + pe = self._pe + pe[:, :, 0::2] = mx.sin(position * div_term) + pe[:, :, 1::2] = mx.cos(position * div_term) + + def __call__(self, input_pos: Array, x: Array): + """ + Args: + input_pos (Array): [batch_size, ] + x (Array): [batch_size, 1, embed_dim] + + Returns: + embedded_x (Array): [batch_size, 1, embed_dim] + """ + + batch_size = cast(tuple[int, ...], x.shape)[0] + pe_values = self._pe[mx.arange(batch_size), input_pos - 1] # (batch_size, embed_dim) + + return x * self.x_scale + self.alpha * mx.expand_dims(pe_values, 1) # (batch_size, 1, embed_dim) + + def prefill(self, x: Array): + """ + Args: + x (Array): [batch_size, seq_len, embed_dim] + + Returns: + embedded_x (Array): [batch_size, seq_len, embed_dim] + """ + pe_values = self._pe[:, : cast(tuple[int, ...], x.shape)[-2]] + return x * self.x_scale + self.alpha * pe_values + + +class KVCacheHND(KVCacheProtocol): + @staticmethod + def empty(kv_cache): + assert len(kv_cache) == 2 + k_cache, v_cache = kv_cache + + k_cache[:] = 0 + v_cache[:] = 0 + + @staticmethod + def update_cache(input_pos, k_val, v_val, kv_cache, cache_idx): + # input_pos: [B, ], k_val: [B, H, 1, D] + assert len(kv_cache) == 2 + k_out, v_out = kv_cache + ip0 = input_pos - 1 + + k_out[cache_idx, :, ip0, None] = k_val + v_out[cache_idx, :, ip0, None] = v_val + + return k_out, v_out + + @staticmethod + def prefill_kv(k_val, v_val, kv_cache): + # k_val: [B, S, H, D] + assert len(kv_cache) == 2 + k_cache, v_cache = kv_cache + + k_cache[..., : cast(tuple[int, ...], k_val.shape)[1], :] = k_val.swapaxes(1, 2) + v_cache[..., : cast(tuple[int, ...], v_val.shape)[1], :] = v_val.swapaxes(1, 2) + + @staticmethod + def init_cache(batch_size: int, max_seq_length: int, n_heads: int, head_dim: int, dtype: mx.Dtype) -> KVCache: + cache_shape = (batch_size, n_heads, max_seq_length, head_dim) + + return (mx.zeros(cache_shape, dtype=dtype), mx.zeros(cache_shape, dtype=dtype)) + + +class KVCacheHNDQuantized(KVCacheProtocol): + @staticmethod + def _el_per_int(bits: int) -> int: + return 32 // bits + + @staticmethod + def _packed_dim(head_dim: int, bits: int = 8) -> int: + el_per_int = KVCacheHNDQuantized._el_per_int(bits) + if head_dim % el_per_int != 0: + raise ValueError(f"{head_dim=} is not divisible by {el_per_int=} ({bits=})") + return head_dim // el_per_int + + @staticmethod + def _group_count(head_dim: int, group_size: int = 32) -> int: + assert group_size in {32, 64, 128} + if head_dim % group_size != 0: + raise ValueError(f"{head_dim} is not divisible by {group_size=}") + return head_dim // group_size + + @staticmethod + def empty(kv_cache) -> None: + assert len(kv_cache) == 3 + (k_q, k_s, k_b), (v_q, v_s, v_b), (_, __) = kv_cache + + k_q[:] = 0 + k_s[:] = 0 + k_b[:] = 0 + v_q[:] = 0 + v_s[:] = 0 + v_b[:] = 0 + + @staticmethod + def update_cache( + input_pos, + k_val, + v_val, + kv_cache, + cache_idx, + ): + # input_pos: [B, ], k_val: [B, H, 1, D] + + assert len(kv_cache) == 3 + (k_q_out, k_s_out, k_b_out), (v_q_out, v_s_out, v_b_out), (group_size, bits) = kv_cache + + k_q, k_s, k_b = mx.quantize(k_val, group_size=group_size, bits=bits) + v_q, v_s, v_b = mx.quantize(v_val, group_size=group_size, bits=bits) + + ip0 = input_pos - 1 + + k_q_out[cache_idx, :, ip0, None] = k_q + k_s_out[cache_idx, :, ip0, None] = k_s + k_b_out[cache_idx, :, ip0, None] = k_b + + v_q_out[cache_idx, :, ip0, None] = v_q + v_s_out[cache_idx, :, ip0, None] = v_s + v_b_out[cache_idx, :, ip0, None] = v_b + + return (k_q_out, k_s_out, k_b_out), (v_q_out, v_s_out, v_b_out), (group_size, bits) + + @staticmethod + def prefill_kv( + k_val, + v_val, + kv_cache, + ) -> None: + assert len(kv_cache) == 3 + (k_q_out, k_s_out, k_b_out), (v_q_out, v_s_out, v_b_out), (group_size, bits) = kv_cache + + S = cast(tuple[int, ...], k_val.shape)[1] + + k_sw = k_val.swapaxes(1, 2) + v_sw = v_val.swapaxes(1, 2) + + k_q, k_s, k_b = mx.quantize(k_sw, group_size=group_size, bits=bits) + v_q, v_s, v_b = mx.quantize(v_sw, group_size=group_size, bits=bits) + + k_q_out[..., :S, :] = k_q + k_s_out[..., :S, :] = k_s + k_b_out[..., :S, :] = k_b + + v_q_out[..., :S, :] = v_q + v_s_out[..., :S, :] = v_s + v_b_out[..., :S, :] = v_b + + @staticmethod + def init_cache( + batch_size: int, + max_seq_length: int, + n_heads: int, + head_dim: int, + dtype: mx.Dtype, + *, + group_size: int = 32, + bits: int = 8, + ) -> KVCacheQ: + packed_dim = KVCacheHNDQuantized._packed_dim(head_dim, bits=bits) + group_cnt = KVCacheHNDQuantized._group_count(head_dim, group_size=group_size) + + packed_shape = (batch_size, n_heads, max_seq_length, packed_dim) + group_shape = (batch_size, n_heads, max_seq_length, group_cnt) + + k_q = mx.zeros(packed_shape, dtype=mx.uint32) + k_s = mx.zeros(group_shape, dtype=dtype) + k_b = mx.zeros(group_shape, dtype=dtype) + + v_q = mx.zeros(packed_shape, dtype=mx.uint32) + v_s = mx.zeros(group_shape, dtype=dtype) + v_b = mx.zeros(group_shape, dtype=dtype) + + return (k_q, k_s, k_b), (v_q, v_s, v_b), (group_size, bits) + + +class AttentionABC(ABC, nn.Module): + def __init__(self, n_head: int, hidden_dim: int, max_seq_length: int, *args, **kwds): + super().__init__() + + self.n_head = n_head + self.hidden_dim = hidden_dim + assert hidden_dim % n_head == 0 + self.head_dim = hidden_dim // n_head + + self.max_seq_length = max_seq_length + + # key, query, value projections for all heads, but in a batch + self.in_proj = nn.Linear(hidden_dim, hidden_dim * 3, bias=True) + self.out_proj = nn.Linear(hidden_dim, hidden_dim, bias=True) + + self.scale = 1 / math.sqrt(self.head_dim) + + self.kc_class: KVCacheProtocol + + @abstractmethod + def __call__( + self, x: Array, input_pos: Array, kv_cache: KVCache | KVCacheQ, cache_idx: Array, attn_mask: Array + ) -> Array: ... + + def prefill(self, x: Array, kv_cache: KVCache | KVCacheQ, attn_mask: Array): + bsz, seqlen, _ = cast(tuple[int, ...], x.shape) + + q, k, v = self.in_proj(mx.expand_dims(x, 0)).split(3, axis=-1) + + q, k, v = map(lambda x: x.reshape(bsz, seqlen, self.n_head, self.head_dim), (q, k, v)) + + self.kc_class.prefill_kv(k, v, kv_cache) + + q, k, v = map(lambda x: x.swapaxes(1, 2), (q, k, v)) + + attn = mx.fast.scaled_dot_product_attention(q, k, v, mask=attn_mask, scale=self.scale) + + attn = mx.nan_to_num(attn) + + attn = attn.swapaxes(1, 2).reshape(1, -1, self.hidden_dim) + + output = self.out_proj(attn) + + return output + + +class FeedForward(nn.Module): + def __init__(self, dim: int, hidden_dim: int) -> None: + super().__init__() + + self.linear1 = nn.Linear(dim, hidden_dim, bias=True) + self.linear2 = nn.Linear(hidden_dim, dim, bias=True) + + def __call__(self, x: Array): + return self.linear2(nn.relu(self.linear1(x))) + + +class TransformerBlockABC(nn.Module): + def __init__(self, n_head: int, ffn_dim: int, hidden_dim: int, max_seq_length: int, *args, **kwds) -> None: + super().__init__() + + self.hidden_dim = hidden_dim + self.max_seq_length = max_seq_length + + self.attention: AttentionABC + + self.feed_forward = FeedForward(hidden_dim, ffn_dim) + self.attention_norm = nn.LayerNorm(self.hidden_dim) + self.ffn_norm = nn.LayerNorm(self.hidden_dim) + + def __call__(self, x: Array, input_pos: Array, kv_cache: KVCache | KVCacheQ, cache_idx: Array, attn_mask: Array): + h = self.attention_norm( + x + + self.attention( + x, + input_pos, + kv_cache, + cache_idx, + attn_mask, + ) + ) + out = self.ffn_norm(h + self.feed_forward(h)) + return out + + def prefill(self, x: Array, attn_mask: Array, kv_cache: KVCache | KVCacheQ): + h = self.attention_norm( + x + + self.attention.prefill( + x, + kv_cache, + attn_mask, + ) + ) + out = self.ffn_norm(h + self.feed_forward(h)) + + return out + + +class TransformerDecoderABC(nn.Module): + def __init__( + self, + hidden_dim: int, + n_layer: int, + n_head: int, + ffn_dim: int, + vocab_size: int, + max_seq_length: int, + max_batch_size: int, + *args, + **kwds, + ) -> None: + super().__init__() + + self.hidden_dim = hidden_dim + self.n_head = n_head + assert hidden_dim % n_head == 0 + + self.head_dim = hidden_dim // n_head + self.vocab_size = vocab_size + + self.n_layer = n_layer + + self.layers: MutableSequence[TransformerBlockABC] + + self.max_seq_length = max_seq_length + self.max_batch_size = max_batch_size + + def __call__( + self, + input_pos: Array, + x: Array, + kv_caches: MutableSequence[KVCache | KVCacheQ], + cache_idx: Array, + *args, + **kwds, + ): + for layer, kv_cache in zip(self.layers, kv_caches): + x = layer( + x, + input_pos, + kv_cache, + cache_idx, + *args, + **kwds, + ) + + return x + + def prefill(self, x: Array, mask: Array, kv_caches: MutableSequence[KVCache | KVCacheQ]): + for layer, kv_cache in zip(self.layers, kv_caches): + x = layer.prefill( + x, + mask, + kv_cache, + ) + return x + + +class T2SDecoderABC(nn.Module, T2SDecoderProtocol): + def __init__( + self, + config: dict, + max_seq_length: int = 1800, + max_batch_size: int = 10, + ) -> None: + super().__init__() + + hidden_dim: int = config["model"]["hidden_dim"] + embedding_dim: int = config["model"]["embedding_dim"] + n_head: int = config["model"]["head"] + n_layer: int = config["model"]["n_layer"] + vocab_size: int = config["model"]["vocab_size"] + phoneme_vocab_size: int = config["model"]["phoneme_vocab_size"] + EOS: int = config["model"]["EOS"] + ffn_dim: int = hidden_dim * 4 + + self.n_layer = int(n_layer) + self.hidden_dim = int(hidden_dim) + self.n_head = int(n_head) + assert hidden_dim % n_head == 0 + + self.head_dim = int(hidden_dim // n_head) + self.embedding_dim = int(embedding_dim) + self.ffn_dim = int(ffn_dim) + self.vocab_size = int(vocab_size) + self.phoneme_vocab_size = int(phoneme_vocab_size) + self.max_seq_length = max_seq_length + self.max_batch_size = max_batch_size + self.EOS = EOS + assert self.EOS == self.vocab_size - 1 + + self.bert_proj = nn.Linear(1024, self.embedding_dim) + self.ar_predict_layer = nn.Linear(self.hidden_dim, self.vocab_size, bias=False) + self.h: TransformerDecoderABC + + self.ar_text_embedding = TokenEmbedding(self.embedding_dim, self.phoneme_vocab_size) + self.ar_text_position = SinePositionalEmbedding( + self.embedding_dim, + scale=False, + max_batch_size=max_batch_size, + max_seq_len=max_seq_length, + ) + self.ar_audio_embedding = TokenEmbedding(self.embedding_dim, self.vocab_size) + self.ar_audio_position = SinePositionalEmbedding( + self.embedding_dim, + scale=False, + max_batch_size=max_batch_size, + max_seq_len=max_seq_length, + ) + + self.kv_class: KVCacheProtocol + + def init_cache(self, bsz: int = 0, *args, **kwds) -> MutableSequence[KVCache | KVCacheQ]: + bsz = bsz or self.h.max_batch_size + assert bsz <= self.h.max_batch_size + seq_lens = self.h.max_seq_length + dtype = self.bert_proj.bias.dtype + cache: MutableSequence[KVCache | KVCacheQ] = [ + self.kv_class.init_cache(bsz, seq_lens, self.n_head, self.head_dim, dtype, *args, **kwds) + for _ in range(self.n_layer) + ] + mx.eval(cache) + return cache + + def embed( + self, + x: list[Array], + y: Array, + bert_features: list[Array], + ): + x_len: list[int] = [cast(tuple[int, ...], i.shape)[0] for i in x] + x_len_max = max(x_len) + xy_pos = mx.zeros((len(x), x_len_max + cast(tuple[int, ...], y.shape)[1], self.embedding_dim)).astype( + bert_features[0].dtype + ) + + bert_features = list(map(lambda x: x.swapaxes(0, 1), bert_features)) + + y_len = cast(tuple[int, ...], y.shape)[1] + y_emb = self.ar_audio_embedding(y) + y_pos = self.ar_audio_position.prefill(y_emb) + + for bs, (x_, len_, bert_feature) in enumerate(zip(x, x_len, bert_features)): + x_emb = self.ar_text_embedding(x_) + bert = self.bert_proj(bert_feature) + x_emb = x_emb + bert + x_pos = self.ar_text_position.prefill(mx.expand_dims(x_emb, 0)) + xy_pos[[bs], :len_] = x_pos + xy_pos[[bs], len_ : len_ + y_len] = y_pos + + mx.eval(xy_pos) + return xy_pos + + def compile(self): + setattr(self.h, "__call__", mx.compile(self.h.__call__)) + # setattr(self.h, "prefill", mx.compile(self.h.prefill, shapeless=True)) + + def pre_forward(self, session: T2SSessionMLX): + attn_mask = session.attn_mask + return list(), dict(attn_mask=attn_mask) + + def post_forward(self, idx: int, session: T2SSessionMLX) -> None: + if idx == 0: + prefill_len = session.prefill_len + bsz = session.bsz + + range_tensor = mx.arange(self.max_seq_length).reshape(1, 1, 1, self.max_seq_length) + prefill_len_expanded = prefill_len.reshape(bsz, 1, 1, 1) + attn_mask = range_tensor < prefill_len_expanded + attn_mask = mx.repeat(attn_mask, self.n_head, 1) + + session.attn_mask = attn_mask + + attn_mask = session.attn_mask + input_pos = session.input_pos + attn_mask[mx.arange(session.bsz), :, :, input_pos] = True + mx.eval(attn_mask) diff --git a/GPT_SoVITS/Accelerate/PyTorch/__init__.py b/GPT_SoVITS/Accelerate/PyTorch/__init__.py new file mode 100644 index 00000000..2997b7ff --- /dev/null +++ b/GPT_SoVITS/Accelerate/PyTorch/__init__.py @@ -0,0 +1,28 @@ +import importlib.util + +import torch + +from .sample_funcs import sample_naive +from .structs import T2SRequest, T2SResult +from .t2s_engine import T2SEngine as T2SEngineTorch + +backends = ["torch_varlen"] +if torch.cuda.is_available(): + backends.append("torch_static_cuda_graph") + if importlib.util.find_spec("sageattention") is not None: + for i in range(torch.cuda.device_count()): + major, minor = torch.cuda.get_device_capability(i) + sm_version = major + minor / 10.0 + if sm_version >= 7.0: + backends.append("sage_attn_varlen_cuda_graph") + if importlib.util.find_spec("flash_attn") is not None: + for i in range(torch.cuda.device_count()): + major, minor = torch.cuda.get_device_capability(i) + sm_version = major + minor / 10.0 + if sm_version >= 7.5: + backends.append("flash_attn_varlen_cuda_graph") +if torch.mps.is_available(): + backends.append("mps_flash_attn_varlen") + + +__all__ = ["T2SEngineTorch", "T2SRequest", "sample_naive", "T2SResult", "backends"] diff --git a/GPT_SoVITS/Accelerate/PyTorch/backends/flash_attn_varlen_cuda_graph.py b/GPT_SoVITS/Accelerate/PyTorch/backends/flash_attn_varlen_cuda_graph.py new file mode 100644 index 00000000..666c1b63 --- /dev/null +++ b/GPT_SoVITS/Accelerate/PyTorch/backends/flash_attn_varlen_cuda_graph.py @@ -0,0 +1,157 @@ +""" +Modified From https://github.com/XXXXRT666/GPT-SoVITS +""" + +from typing import Dict, List, Tuple + +import kernels +import torch + +from .. import nn +from ..structs import T2SSession +from ..t2s_model_abc import ( + AttentionABC, + CUDAGraphCacheABC, + FeedForward, + KVCacheNHD, + KVCacheProtocol, + T2SDecoderABC, + TransformerBlockABC, + TransformerDecoderABC, +) + +flash_attn_kernel = None +try: + import flash_attn_interface as flash_attn # type: ignore + + flash_attn_kernel = flash_attn.flash_attn_with_kvcache +except ModuleNotFoundError: + try: + import flash_attn # type: ignore + + flash_attn_kernel = flash_attn.flash_attn_with_kvcache + + except ModuleNotFoundError: + pass + +if flash_attn_kernel is None: + flash_attn_kernel = kernels.get_kernel("kernels-community/flash-attn").flash_attn_with_kvcache + + +Tensor = torch.Tensor + + +class Attention(AttentionABC): + def __init__(self, n_head, hidden_dim, max_seq_length): + super().__init__(n_head, hidden_dim, max_seq_length) + + self.in_proj = nn.Linear(hidden_dim, hidden_dim * 3, bias=True) + self.out_proj = nn.Linear(hidden_dim, hidden_dim, bias=True) + + def __call__(self, x: Tensor, input_pos: Tensor, kv_cache: KVCacheProtocol, *args, **kwds) -> Tensor: + bsz, seqlen, _ = x.shape + + q, k, v = self.in_proj(x).chunk(3, dim=-1) + + q = q.view(bsz, seqlen, self.n_head, self.head_dim) + k = k.view(bsz, seqlen, self.n_head, self.head_dim) + v = v.view(bsz, seqlen, self.n_head, self.head_dim) + + attn: Tensor = flash_attn.flash_attn_with_kvcache( # type: ignore + q, kv_cache.k_cache, kv_cache.v_cache, k, v, cache_seqlens=input_pos - 1 + ) + + attn = attn.view(bsz, seqlen, self.hidden_dim) + + attn = self.out_proj(attn) + + return attn + + +class TransformerBlock(TransformerBlockABC): + def __init__(self, n_head, ffn_dim, hidden_dim, max_seq_length) -> None: + super().__init__(n_head, ffn_dim, hidden_dim, max_seq_length) + + self.attention = Attention(n_head, hidden_dim, max_seq_length) + self.feed_forward = FeedForward(hidden_dim, ffn_dim) + self.attention_norm = nn.LayerNorm([self.hidden_dim]) + self.ffn_norm = nn.LayerNorm([self.hidden_dim]) + + +class TransformerDecoder(TransformerDecoderABC): + def __init__( + self, + hidden_dim, + n_layer, + n_head, + ffn_dim, + vocab_size, + max_seq_length, + max_batch_size, + ) -> None: + super().__init__(hidden_dim, n_layer, n_head, ffn_dim, vocab_size, max_seq_length, max_batch_size) + + self.layers = nn.ModuleList( # type: ignore + TransformerBlock(n_head, ffn_dim, hidden_dim, max_seq_length) for _ in range(n_layer) + ) + + +class T2SDecoder(T2SDecoderABC): + def __init__( + self, + config, + max_seq_length=1800, + max_batch_size=10, + ) -> None: + assert torch.cuda.is_available() + super().__init__(config, max_seq_length, max_batch_size) + + self.bert_proj = nn.Linear(1024, self.embedding_dim) + self.ar_predict_layer = nn.Linear(self.hidden_dim, self.vocab_size, bias=False) + self.h: TransformerDecoderABC = TransformerDecoder( + self.hidden_dim, self.n_layer, self.n_head, self.ffn_dim, self.vocab_size, max_seq_length, max_batch_size + ) + + self.kv_class = KVCacheNHD + + def post_forward(self, idx: int, session: T2SSession) -> None: + return super().post_forward(idx, session) + + def pre_forward(self, session: T2SSession) -> Tuple[List, Dict]: + return super().pre_forward(session) + + +class CUDAGraphCache(CUDAGraphCacheABC): + def __init__( + self, + decoder: T2SDecoder, + ) -> None: + super().__init__(decoder) + + def release_graph(self, session: T2SSession): + if session.id != self.id: + self.assigned = False + else: + del session.graph, session.xy_pos_, session.xy_dec_, session.input_pos, session.kv_cache + + def get_cache_graph(self, session: T2SSession): + assert self.graph + session.graph = self.graph + session.stream = self.stream + + session.xy_pos_ = self.xy_pos + session.xy_dec_ = self.xy_dec + session.input_pos = self.input_pos.copy_(session.input_pos) + + for cache, cache_ in zip(self.kv_cache, session.kv_cache): + cache.sync_cache(cache_) + + def capture_new_graph(self, session: T2SSession): + session.xy_pos_ = self.xy_pos.clone() + session.xy_dec_ = self.xy_dec.clone() + session.input_pos = self.input_pos.clone().copy_(session.input_pos) + + args, kwds = self.decoder.pre_forward(session) + graph = self.decoder.capture(self.input_pos, self.xy_pos, self.xy_dec, self.kv_cache, *args, **kwds) + session.graph = graph + session.stream = torch.cuda.Stream() # type: ignore diff --git a/GPT_SoVITS/Accelerate/PyTorch/backends/mps_flash_attn_varlen.py b/GPT_SoVITS/Accelerate/PyTorch/backends/mps_flash_attn_varlen.py new file mode 100644 index 00000000..6f15f51e --- /dev/null +++ b/GPT_SoVITS/Accelerate/PyTorch/backends/mps_flash_attn_varlen.py @@ -0,0 +1,165 @@ +import torch +from torch.nn import functional as F + +from .. import nn +from ..structs import KVCacheProtocol, T2SSession +from ..t2s_model_abc import ( + AttentionABC, + CUDAGraphCacheABC, + FeedForward, + KVCacheHND, + T2SDecoderABC, + TransformerBlockABC, + TransformerDecoderABC, +) + +Tensor = torch.Tensor + + +class Attention(AttentionABC): + def __init__(self, n_head, hidden_dim, max_seq_length): + super().__init__(n_head, hidden_dim, max_seq_length) + + # key, query, value projections for all heads, but in a batch + self.in_proj = nn.Linear(hidden_dim, hidden_dim * 3, bias=True) + self.out_proj = nn.Linear(hidden_dim, hidden_dim, bias=True) + + def __call__(self, x: Tensor, input_pos: Tensor, kv_cache: KVCacheProtocol, attn_mask: Tensor): + bsz, seqlen, _ = x.shape + + q, k, v = self.in_proj(x).chunk(3, dim=-1) + + q = q.view(bsz, seqlen, self.n_head, self.head_dim) + k = k.view(bsz, seqlen, self.n_head, self.head_dim) + v = v.view(bsz, seqlen, self.n_head, self.head_dim) + + q, k, v = map(lambda x: x.transpose(1, 2), (q, k, v)) + + k, v = kv_cache.update(input_pos, k, v) + + attn = F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask) + + attn = attn.transpose(1, 2).contiguous().view(bsz, seqlen, self.hidden_dim) + + attn = self.out_proj(attn) + + return attn + + +class TransformerBlock(TransformerBlockABC): + def __init__(self, n_head: int, ffn_dim: int, hidden_dim: int, max_seq_length: int) -> None: + super().__init__(n_head, ffn_dim, hidden_dim, max_seq_length) + + self.attention = Attention(n_head, hidden_dim, max_seq_length) + self.feed_forward = FeedForward(hidden_dim, ffn_dim) + self.attention_norm = nn.LayerNorm([self.hidden_dim]) + self.ffn_norm = nn.LayerNorm([self.hidden_dim]) + + +class TransformerDecoder(TransformerDecoderABC): + def __init__( + self, + hidden_dim, + n_layer, + n_head, + ffn_dim, + vocab_size, + max_seq_length, + max_batch_size, + ) -> None: + super().__init__(hidden_dim, n_layer, n_head, ffn_dim, vocab_size, max_seq_length, max_batch_size) + + self.layers = nn.ModuleList( # type: ignore + TransformerBlock(n_head, ffn_dim, hidden_dim, max_seq_length) for _ in range(n_layer) + ) + + +class T2SDecoder(T2SDecoderABC): + def __init__( + self, + config, + max_seq_length=1800, + max_batch_size=10, + ) -> None: + super().__init__(config, max_seq_length, max_batch_size) + + self.bert_proj = nn.Linear(1024, self.embedding_dim) + self.ar_predict_layer = nn.Linear(self.hidden_dim, self.vocab_size, bias=False) + self.h: TransformerDecoderABC = TransformerDecoder( + self.hidden_dim, self.n_layer, self.n_head, self.ffn_dim, self.vocab_size, max_seq_length, max_batch_size + ) + + self.kv_class = KVCacheHND + + def pre_forward(self, session: T2SSession): + attn_mask = session.attn_mask + return list(), dict(attn_mask=attn_mask) + + def post_forward(self, idx: int, session: T2SSession) -> None: + if idx == 0: + prefill_len = session.prefill_len + bsz = session.bsz + + range_tensor = torch.arange(self.max_seq_length).view(1, 1, 1, self.max_seq_length) + prefill_len_expanded = prefill_len.view(bsz, 1, 1, 1) + attn_mask = range_tensor < prefill_len_expanded + attn_mask = attn_mask.expand(-1, self.n_head, -1, -1) + + session.attn_mask = attn_mask + + attn_mask = session.attn_mask + input_pos = session.input_pos + attn_mask[torch.arange(session.bsz), :, :, input_pos] = True + + +class CUDAGraphCache(CUDAGraphCacheABC): + def __init__( + self, + decoder, + ) -> None: + super().__init__(decoder) + if torch.cuda.is_available(): + self.attn_mask = ( + torch.randint(0, 2, (decoder.max_batch_size, decoder.n_head, 1, decoder.max_seq_length)) + .bool() + .to(self.device, self.dtype) + ) + + def release_graph(self, session: T2SSession): + if session.id != self.id: + self.assigned = False + else: + del ( + session.graph, + session.xy_pos_, + session.xy_dec_, + session.input_pos, + session.kv_cache, + session.attn_mask, + ) + + def get_cache_graph(self, session: T2SSession): + assert self.graph + session.graph = self.graph + session.stream = self.stream + + session.xy_pos_ = self.xy_pos + session.xy_dec_ = self.xy_dec + session.input_pos = self.input_pos.copy_(session.input_pos) + + session.attn_mask = self.attn_mask + + for cache, cache_ in zip(self.kv_cache, session.kv_cache): + cache.sync_cache(cache_) + + def capture_new_graph(self, session: T2SSession): + session.xy_pos_ = self.xy_pos.clone() + session.xy_dec_ = self.xy_dec.clone() + session.input_pos = self.input_pos.clone().copy_(session.input_pos) + + session.attn_mask = self.attn_mask.clone().copy_(session.attn_mask) + + args, kwds = self.decoder.pre_forward(session) + graph = self.decoder.capture(self.input_pos, self.xy_pos, self.xy_dec, self.kv_cache, *args, **kwds) + session.graph = graph + session.stream = torch.cuda.Stream() # type: ignore diff --git a/GPT_SoVITS/Accelerate/PyTorch/backends/sage_attn_varlen_cuda_graph.py b/GPT_SoVITS/Accelerate/PyTorch/backends/sage_attn_varlen_cuda_graph.py new file mode 100644 index 00000000..fd363147 --- /dev/null +++ b/GPT_SoVITS/Accelerate/PyTorch/backends/sage_attn_varlen_cuda_graph.py @@ -0,0 +1,176 @@ +from typing import MutableSequence + +import sageattention # type: ignore +import torch + +from .. import nn +from ..structs import T2SSession +from ..t2s_model_abc import ( + AttentionABC, + CUDAGraphCacheABC, + FeedForward, + KVCacheHND, + KVCacheProtocol, + T2SDecoderABC, + TransformerBlockABC, + TransformerDecoderABC, +) + +Tensor = torch.Tensor + + +class Attention(AttentionABC): + def __init__(self, n_head, hidden_dim, max_seq_length): + super().__init__(n_head, hidden_dim, max_seq_length) + + # key, query, value projections for all heads, but in a batch + self.in_proj = nn.Linear(hidden_dim, hidden_dim * 3, bias=True) + self.out_proj = nn.Linear(hidden_dim, hidden_dim, bias=True) + + def __call__( + self, + x: Tensor, + input_pos: Tensor, + kv_cache: KVCacheProtocol, + cu_seqlens_q: Tensor, + cu_seqlens_kv: Tensor, + ) -> Tensor: + bsz, seqlen, _ = x.shape + + q, k, v = self.in_proj(x).chunk(3, dim=-1) + + q = q.view(bsz, seqlen, self.n_head, self.head_dim) + k = k.view(bsz, seqlen, self.n_head, self.head_dim) + v = v.view(bsz, seqlen, self.n_head, self.head_dim) + + q, k, v = map(lambda x: x.transpose(1, 2), (q, k, v)) + + k, v = kv_cache.update(input_pos, k, v) + + attn: Tensor = sageattention.sageattn_varlen( + q, + k, + v, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_kv=cu_seqlens_kv, + max_seqlen_q=1, + max_seqlen_k=self.max_seq_length, + ) + + attn = attn.transpose(1, 2).contiguous().view(bsz, seqlen, self.hidden_dim) + + attn = self.out_proj(attn) + + return attn + + +class TransformerBlock(TransformerBlockABC): + def __init__(self, n_head, ffn_dim, hidden_dim, max_seq_length) -> None: + super().__init__(n_head, ffn_dim, hidden_dim, max_seq_length) + + self.attention = Attention(n_head, hidden_dim, max_seq_length) + self.feed_forward = FeedForward(hidden_dim, ffn_dim) + self.attention_norm = nn.LayerNorm([self.hidden_dim]) + self.ffn_norm = nn.LayerNorm([self.hidden_dim]) + + +class TransformerDecoder(TransformerDecoderABC): + def __init__( + self, + hidden_dim, + n_layer, + n_head, + ffn_dim, + vocab_size, + max_seq_length, + max_batch_size, + ) -> None: + super().__init__(hidden_dim, n_layer, n_head, ffn_dim, vocab_size, max_seq_length, max_batch_size) + + self.layers = nn.ModuleList( # type: ignore + TransformerBlock(n_head, ffn_dim, hidden_dim, max_seq_length) for _ in range(n_layer) + ) + + +class T2SDecoder(T2SDecoderABC): + def __init__( + self, + config, + max_seq_length=1800, + max_batch_size=10, + ) -> None: + super().__init__(config, max_seq_length, max_batch_size) + + self.bert_proj = nn.Linear(1024, self.embedding_dim) + self.ar_predict_layer = nn.Linear(self.hidden_dim, self.vocab_size, bias=False) + self.h: TransformerDecoderABC = TransformerDecoder( + self.hidden_dim, self.n_layer, self.n_head, self.ffn_dim, self.vocab_size, max_seq_length, max_batch_size + ) + + self.kv_class = KVCacheHND + + def pre_forward(self, session: T2SSession) -> tuple[list[Tensor], dict[str, Tensor]]: + return list(), dict(cu_seqlens_q=session.cu_seqlens_q, cu_seqlens_kv=session.cu_seqlens_kv) + + def post_forward(self, idx: int, session: T2SSession): + if idx == 0: + session.cu_seqlens_q = torch.arange(0, session.bsz + 1, dtype=torch.int32) + session.cu_seqlens_kv = torch.cat([torch.tensor(0, dtype=torch.int32), session.input_pos]) + else: + cu_seqlens_q = session.cu_seqlens_q + cu_seqlens_kv = session.cu_seqlens_kv + cu_seqlens_kv.add_(cu_seqlens_q) + + +class CUDAGraphCache(CUDAGraphCacheABC): + def __init__( + self, + decoder: T2SDecoder, + ) -> None: + super().__init__(decoder) + + if torch.cuda.is_available(): + self.cu_seqlens_q = torch.arange(0, decoder.max_batch_size + 1, dtype=torch.int32).to(self.device) + self.cu_seqlens_kv = torch.cat([torch.tensor(0, dtype=torch.int32), self.input_pos]).to(self.device) + + def release_graph(self, session: T2SSession): + if session.id != self.id: + self.assigned = False + else: + del ( + session.graph, + session.xy_pos_, + session.xy_dec_, + session.input_pos, + session.kv_cache, + session.cu_seqlens_q, + session.cu_seqlens_kv, + ) + + def get_cache_graph(self, session: T2SSession): + assert self.graph + session.graph = self.graph + session.stream = self.stream + + session.xy_pos_ = self.xy_pos + session.xy_dec_ = self.xy_dec + session.input_pos = self.input_pos.copy_(session.input_pos) + + session.cu_seqlens_q = self.cu_seqlens_q + session.cu_seqlens_kv = self.cu_seqlens_kv + + for cache, cache_ in zip(self.kv_cache, session.kv_cache): + cache.sync_cache(cache_) + + def capture_new_graph(self, session: T2SSession): + session.xy_pos_ = self.xy_pos.clone() + session.xy_dec_ = self.xy_dec.clone() + session.input_pos = self.input_pos.clone().copy_(session.input_pos) + + session.cu_seqlens_q = self.cu_seqlens_q.clone().copy_(session.cu_seqlens_q) + session.cu_seqlens_kv = self.cu_seqlens_kv.clone().copy_(session.cu_seqlens_kv) + + args, kwds = self.decoder.pre_forward(session) + graph = self.decoder.capture(self.input_pos, self.xy_pos, self.xy_dec, self.kv_cache, *args, **kwds) + session.graph = graph + session.stream = torch.cuda.Stream() # type: ignore diff --git a/GPT_SoVITS/Accelerate/PyTorch/backends/torch_static_cuda_graph.py b/GPT_SoVITS/Accelerate/PyTorch/backends/torch_static_cuda_graph.py new file mode 100644 index 00000000..6f15f51e --- /dev/null +++ b/GPT_SoVITS/Accelerate/PyTorch/backends/torch_static_cuda_graph.py @@ -0,0 +1,165 @@ +import torch +from torch.nn import functional as F + +from .. import nn +from ..structs import KVCacheProtocol, T2SSession +from ..t2s_model_abc import ( + AttentionABC, + CUDAGraphCacheABC, + FeedForward, + KVCacheHND, + T2SDecoderABC, + TransformerBlockABC, + TransformerDecoderABC, +) + +Tensor = torch.Tensor + + +class Attention(AttentionABC): + def __init__(self, n_head, hidden_dim, max_seq_length): + super().__init__(n_head, hidden_dim, max_seq_length) + + # key, query, value projections for all heads, but in a batch + self.in_proj = nn.Linear(hidden_dim, hidden_dim * 3, bias=True) + self.out_proj = nn.Linear(hidden_dim, hidden_dim, bias=True) + + def __call__(self, x: Tensor, input_pos: Tensor, kv_cache: KVCacheProtocol, attn_mask: Tensor): + bsz, seqlen, _ = x.shape + + q, k, v = self.in_proj(x).chunk(3, dim=-1) + + q = q.view(bsz, seqlen, self.n_head, self.head_dim) + k = k.view(bsz, seqlen, self.n_head, self.head_dim) + v = v.view(bsz, seqlen, self.n_head, self.head_dim) + + q, k, v = map(lambda x: x.transpose(1, 2), (q, k, v)) + + k, v = kv_cache.update(input_pos, k, v) + + attn = F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask) + + attn = attn.transpose(1, 2).contiguous().view(bsz, seqlen, self.hidden_dim) + + attn = self.out_proj(attn) + + return attn + + +class TransformerBlock(TransformerBlockABC): + def __init__(self, n_head: int, ffn_dim: int, hidden_dim: int, max_seq_length: int) -> None: + super().__init__(n_head, ffn_dim, hidden_dim, max_seq_length) + + self.attention = Attention(n_head, hidden_dim, max_seq_length) + self.feed_forward = FeedForward(hidden_dim, ffn_dim) + self.attention_norm = nn.LayerNorm([self.hidden_dim]) + self.ffn_norm = nn.LayerNorm([self.hidden_dim]) + + +class TransformerDecoder(TransformerDecoderABC): + def __init__( + self, + hidden_dim, + n_layer, + n_head, + ffn_dim, + vocab_size, + max_seq_length, + max_batch_size, + ) -> None: + super().__init__(hidden_dim, n_layer, n_head, ffn_dim, vocab_size, max_seq_length, max_batch_size) + + self.layers = nn.ModuleList( # type: ignore + TransformerBlock(n_head, ffn_dim, hidden_dim, max_seq_length) for _ in range(n_layer) + ) + + +class T2SDecoder(T2SDecoderABC): + def __init__( + self, + config, + max_seq_length=1800, + max_batch_size=10, + ) -> None: + super().__init__(config, max_seq_length, max_batch_size) + + self.bert_proj = nn.Linear(1024, self.embedding_dim) + self.ar_predict_layer = nn.Linear(self.hidden_dim, self.vocab_size, bias=False) + self.h: TransformerDecoderABC = TransformerDecoder( + self.hidden_dim, self.n_layer, self.n_head, self.ffn_dim, self.vocab_size, max_seq_length, max_batch_size + ) + + self.kv_class = KVCacheHND + + def pre_forward(self, session: T2SSession): + attn_mask = session.attn_mask + return list(), dict(attn_mask=attn_mask) + + def post_forward(self, idx: int, session: T2SSession) -> None: + if idx == 0: + prefill_len = session.prefill_len + bsz = session.bsz + + range_tensor = torch.arange(self.max_seq_length).view(1, 1, 1, self.max_seq_length) + prefill_len_expanded = prefill_len.view(bsz, 1, 1, 1) + attn_mask = range_tensor < prefill_len_expanded + attn_mask = attn_mask.expand(-1, self.n_head, -1, -1) + + session.attn_mask = attn_mask + + attn_mask = session.attn_mask + input_pos = session.input_pos + attn_mask[torch.arange(session.bsz), :, :, input_pos] = True + + +class CUDAGraphCache(CUDAGraphCacheABC): + def __init__( + self, + decoder, + ) -> None: + super().__init__(decoder) + if torch.cuda.is_available(): + self.attn_mask = ( + torch.randint(0, 2, (decoder.max_batch_size, decoder.n_head, 1, decoder.max_seq_length)) + .bool() + .to(self.device, self.dtype) + ) + + def release_graph(self, session: T2SSession): + if session.id != self.id: + self.assigned = False + else: + del ( + session.graph, + session.xy_pos_, + session.xy_dec_, + session.input_pos, + session.kv_cache, + session.attn_mask, + ) + + def get_cache_graph(self, session: T2SSession): + assert self.graph + session.graph = self.graph + session.stream = self.stream + + session.xy_pos_ = self.xy_pos + session.xy_dec_ = self.xy_dec + session.input_pos = self.input_pos.copy_(session.input_pos) + + session.attn_mask = self.attn_mask + + for cache, cache_ in zip(self.kv_cache, session.kv_cache): + cache.sync_cache(cache_) + + def capture_new_graph(self, session: T2SSession): + session.xy_pos_ = self.xy_pos.clone() + session.xy_dec_ = self.xy_dec.clone() + session.input_pos = self.input_pos.clone().copy_(session.input_pos) + + session.attn_mask = self.attn_mask.clone().copy_(session.attn_mask) + + args, kwds = self.decoder.pre_forward(session) + graph = self.decoder.capture(self.input_pos, self.xy_pos, self.xy_dec, self.kv_cache, *args, **kwds) + session.graph = graph + session.stream = torch.cuda.Stream() # type: ignore diff --git a/GPT_SoVITS/Accelerate/PyTorch/backends/torch_varlen.py b/GPT_SoVITS/Accelerate/PyTorch/backends/torch_varlen.py new file mode 100644 index 00000000..d079e9af --- /dev/null +++ b/GPT_SoVITS/Accelerate/PyTorch/backends/torch_varlen.py @@ -0,0 +1,144 @@ +from typing import NoReturn + +import torch +from torch.nn import functional as F + +from .. import nn +from ..structs import KVCacheProtocol, T2SSession +from ..t2s_model_abc import ( + AttentionABC, + CUDAGraphCacheABC, + FeedForward, + KVCacheHNDVarlen, + T2SDecoderABC, + TransformerBlockABC, + TransformerDecoderABC, +) + +Tensor = torch.Tensor + + +class Attention(AttentionABC): + def __init__(self, n_head, hidden_dim, max_seq_length): + super().__init__(n_head, hidden_dim, max_seq_length) + + # key, query, value projections for all heads, but in a batch + self.in_proj = nn.Linear(hidden_dim, hidden_dim * 3, bias=True) + self.out_proj = nn.Linear(hidden_dim, hidden_dim, bias=True) + + def __call__(self, x: Tensor, input_pos: Tensor, kv_cache: KVCacheProtocol, attn_mask: Tensor): + bsz, seqlen, _ = x.shape + + q, k, v = self.in_proj(x).chunk(3, dim=-1) + + q = q.view(bsz, seqlen, self.n_head, self.head_dim) + k = k.view(bsz, seqlen, self.n_head, self.head_dim) + v = v.view(bsz, seqlen, self.n_head, self.head_dim) + + q, k, v = map(lambda x: x.transpose(1, 2), (q, k, v)) + + k, v = kv_cache.update(input_pos, k, v) + + max_idx = input_pos.max() + + q, k, v = map(lambda x: x[..., :max_idx, :], (q, k, v)) + + mask = attn_mask[..., :max_idx] + + attn = F.scaled_dot_product_attention(q, k, v, mask) + + attn = attn.transpose(1, 2).contiguous().view(bsz, seqlen, self.hidden_dim) + + attn = self.out_proj(attn) + + return attn + + +class TransformerBlock(TransformerBlockABC): + def __init__(self, n_head: int, ffn_dim: int, hidden_dim: int, max_seq_length: int) -> None: + super().__init__(n_head, ffn_dim, hidden_dim, max_seq_length) + + self.attention = Attention(n_head, hidden_dim, max_seq_length) + self.feed_forward = FeedForward(hidden_dim, ffn_dim) + self.attention_norm = nn.LayerNorm([self.hidden_dim]) + self.ffn_norm = nn.LayerNorm([self.hidden_dim]) + + +class TransformerDecoder(TransformerDecoderABC): + def __init__( + self, + hidden_dim, + n_layer, + n_head, + ffn_dim, + vocab_size, + max_seq_length, + max_batch_size, + ) -> None: + super().__init__(hidden_dim, n_layer, n_head, ffn_dim, vocab_size, max_seq_length, max_batch_size) + + self.layers = nn.ModuleList( # type: ignore + TransformerBlock(n_head, ffn_dim, hidden_dim, max_seq_length) for _ in range(n_layer) + ) + + +class T2SDecoder(T2SDecoderABC): + def __init__( + self, + config, + max_seq_length=1800, + max_batch_size=10, + ) -> None: + super().__init__(config, max_seq_length, max_batch_size) + + self.bert_proj = nn.Linear(1024, self.embedding_dim) + self.ar_predict_layer = nn.Linear(self.hidden_dim, self.vocab_size, bias=False) + self.h: TransformerDecoderABC = TransformerDecoder( + self.hidden_dim, self.n_layer, self.n_head, self.ffn_dim, self.vocab_size, max_seq_length, max_batch_size + ) + + self.kv_class = KVCacheHNDVarlen + + def capture( + self, + *args, + **kwds, + ) -> NoReturn: + raise NotImplementedError("Cuda Graph Is Not Supported For Varlen Model") + + def pre_forward(self, session: T2SSession): + attn_mask = session.attn_mask + return list(), dict(attn_mask=attn_mask) + + def post_forward(self, idx: int, session: T2SSession) -> None: + if idx == 0: + prefill_len = session.prefill_len + bsz = session.bsz + + range_tensor = torch.arange(self.max_seq_length).view(1, 1, 1, self.max_seq_length) + prefill_len_expanded = prefill_len.view(bsz, 1, 1, 1) + attn_mask = range_tensor < prefill_len_expanded + attn_mask = attn_mask.expand(-1, self.n_head, -1, -1) + + session.attn_mask = attn_mask + + attn_mask = session.attn_mask + input_pos = session.input_pos + attn_mask[torch.arange(session.bsz), :, :, input_pos] = True + + +class CUDAGraphCache(CUDAGraphCacheABC): + def __init__( + self, + decoder, + ) -> None: + super().__init__(decoder, False) + + def release_graph(self, session: T2SSession): + raise NotImplementedError("Cuda Graph Is Not Supported For Varlen Model") + + def get_cache_graph(self, session: T2SSession): + raise NotImplementedError("Cuda Graph Is Not Supported For Varlen Model") + + def capture_new_graph(self, session: T2SSession): + raise NotImplementedError("Cuda Graph Is Not Supported For Varlen Model") diff --git a/GPT_SoVITS/Accelerate/PyTorch/nn.py b/GPT_SoVITS/Accelerate/PyTorch/nn.py new file mode 100644 index 00000000..1e69e056 --- /dev/null +++ b/GPT_SoVITS/Accelerate/PyTorch/nn.py @@ -0,0 +1,69 @@ +""" +Enhanced Type Hint nn.Module +Modified From https://github.com/labmlai/labml/blob/master/helpers/labml_helpers/module.py +""" + +from typing import Any + +import torch.nn +from torch.nn import ( + functional as functional, +) +from torch.nn import ( + utils as utils, +) +from torch.nn.modules import * # type: ignore # noqa: F403 +from torch.nn.parameter import ( + Parameter as Parameter, +) + +Tensor = torch.Tensor + + +class Module(torch.nn.Module): + r""" + Wraps ``torch.nn.Module`` to overload ``__call__`` instead of + ``forward`` for better type checking. + + `PyTorch Github issue for clarification `_ + """ + + def _forward_unimplemented(self, *input: Any) -> None: + # To stop PyTorch from giving abstract methods warning + pass + + def __init_subclass__(cls, **kwargs): + if cls.__dict__.get("__call__", None) is None: + return + + setattr(cls, "forward", cls.__dict__["__call__"]) + delattr(cls, "__call__") + + @property + def device(self) -> torch.device: + params = self.parameters() + try: + sample_param = next(params) + return sample_param.device + except StopIteration: + raise RuntimeError(f"Unable to determine device of {self.__class__.__name__}") from None + + +class Linear(torch.nn.Linear): + def __call__(self, input: Tensor) -> Tensor: + return super().__call__(input) + + +class Dropout(torch.nn.Dropout): + def __call__(self, input: Tensor) -> Tensor: + return super().__call__(input) + + +class Embedding(torch.nn.Embedding): + def __call__(self, input: Tensor) -> Tensor: + return super().__call__(input) + + +class LayerNorm(torch.nn.LayerNorm): + def __call__(self, input: Tensor) -> Tensor: + return super().__call__(input) diff --git a/GPT_SoVITS/Accelerate/PyTorch/sample_funcs.py b/GPT_SoVITS/Accelerate/PyTorch/sample_funcs.py new file mode 100644 index 00000000..3cfeb7ff --- /dev/null +++ b/GPT_SoVITS/Accelerate/PyTorch/sample_funcs.py @@ -0,0 +1,63 @@ +from typing import Protocol + +import torch +import torch.nn.functional as F + +Tensor = torch.Tensor + + +class SampleProtocol(Protocol): + @staticmethod + def __call__( + logits: Tensor, + previous_tokens: Tensor, + temperature: float, + top_k: int, + top_p: float, + repetition_penalty: float, + ) -> Tensor: ... + + +class sample_naive(SampleProtocol): + @staticmethod + def __call__( + logits: Tensor, + previous_tokens: Tensor, + temperature: float, + top_k: int, + top_p: float, + repetition_penalty: float, + ): + if temperature <= 1e-5: + probs = F.softmax(logits, dim=-1) + return torch.argmax(probs, dim=-1, keepdim=True).to(dtype=torch.int32) + + if repetition_penalty != 1.0: + previous_tokens = previous_tokens.long() + score = torch.gather(logits, dim=1, index=previous_tokens) + score = torch.where( + score < 0, + score * repetition_penalty, + score / repetition_penalty, + ) + logits.scatter_(dim=1, index=previous_tokens, src=score) + + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cum_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) + cum_probs[cum_probs > 1] = 1 + sorted_indices_to_remove = cum_probs > top_p + sorted_indices_to_remove[:, 0] = False # keep at least one option + indices_to_remove = sorted_indices_to_remove.scatter(dim=1, index=sorted_indices, src=sorted_indices_to_remove) + logits = logits.masked_fill(indices_to_remove, -float("Inf")) + + logits /= temperature + + v, _ = torch.topk(logits, top_k) + pivot = v[:, -1].unsqueeze(-1) + logits = torch.where(logits < pivot, -float("Inf"), logits) + + probs = F.softmax(logits, dim=-1) + q = torch.empty_like(probs).exponential_(1.0) + idx_next = torch.argmax(probs / q, dim=-1, keepdim=True).to(dtype=torch.int32) + + return idx_next diff --git a/GPT_SoVITS/Accelerate/PyTorch/structs.py b/GPT_SoVITS/Accelerate/PyTorch/structs.py new file mode 100644 index 00000000..1822acdc --- /dev/null +++ b/GPT_SoVITS/Accelerate/PyTorch/structs.py @@ -0,0 +1,151 @@ +""" +Modified From https://github.com/XXXXRT666/GPT-SoVITS +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal, MutableSequence, Optional, Protocol + +import torch + +from .sample_funcs import SampleProtocol, sample_naive + +Tensor = torch.Tensor + + +@dataclass +class T2SResult: + result: list[Tensor] | None = None + infer_speed: tuple[float, float] = (0.0, 0.0) + status: Literal["Success", "Error"] = "Success" + exception: Optional[Exception] = None + traceback: Optional[str] = None + + +@dataclass +class T2SRequest: + x: list[torch.Tensor] + x_lens: Tensor + prompts: torch.Tensor + bert_feature: list[Tensor] + valid_length: int + top_k: int = 5 + top_p: float = 1 + early_stop_num: int = -1 + temperature: float = 1.0 + repetition_penalty: float = 1.35 + use_cuda_graph: bool = False + debug: bool = False + + +class KVCacheProtocol(Protocol): + k_cache: Tensor + v_cache: Tensor + + def __init__(self, batch_size: int, max_seq_length: int, n_heads: int, head_dim: int) -> None: ... + + def empty(self) -> None: ... + + def update(self, input_pos: Tensor, k_val: Tensor, v_val: Tensor, *args, **kwds) -> tuple[Tensor, Tensor]: ... + + def prefill_kv(self, k_val: Tensor, v_val: Tensor) -> None: ... + + def sync_cache(self, kv_cache: KVCacheProtocol) -> None: ... + + +class T2SDecoderProtocol(Protocol): + max_seq_length: int + EOS: int + n_head: int + + @property + def device(self) -> torch.device: ... + + def embed(self, x: list[Tensor], y: Tensor, bert_features: list[Tensor]) -> Tensor: ... + + +class T2SEngineProtocol(Protocol): + def _handle_request(self, request: T2SRequest) -> tuple[list[Tensor], float, float]: ... + + def generate(self, request: T2SRequest) -> T2SResult: ... + + +class T2SSession: + def __init__( + self, + decoder: T2SDecoderProtocol, + request: T2SRequest, + sapmle_func: type[SampleProtocol] = sample_naive, + device: torch.device = torch.device("cpu"), + dtype: torch.dtype = torch.float32, + ): + with device: + self.decoder = decoder + self.request = request + self.device = device + self.dtype = dtype + + bsz = len(request.x) + y_len = request.prompts.size(-1) + self.bsz = bsz + self.y_len = y_len + request.prompts = request.prompts.to(device, torch.int32) + + # Cache + self.kv_cache: MutableSequence[KVCacheProtocol] + self.sample = sapmle_func() + + # Forward args + self.x = [i.to(device) for i in request.x] + self.x_lens = request.x_lens.to(torch.int32) + self.y = torch.zeros((bsz, decoder.max_seq_length)).to(torch.int32) + self.y[:, : request.prompts.shape[-1]] = request.prompts + self.bert_feature = [i.to(device, dtype) for i in request.bert_feature] + + self.prefill_len = self.x_lens + request.prompts.size(1) + + self.input_pos = torch.zeros_like(self.prefill_len) + self.input_pos.add_(self.prefill_len) + + # CUDA Graph + self.stream: Optional[torch.cuda.Stream] = None + self.graph: Optional[torch.cuda.CUDAGraph] = None + self.xy_pos_: Tensor + self.xy_dec_: Tensor + + # EOS + self.completed = torch.Tensor([False] * len(self.x)).bool().to(device) + self.y_results: list[Tensor] = [None] * len(self.x) # type: ignore + + self.xy_pos = decoder.embed(self.x, request.prompts, self.bert_feature) + + max_len = int(self.prefill_len.max().item()) + attn_mask = torch.zeros(size=(bsz, max_len, max_len), dtype=torch.bool) + + for bs in range(bsz): + pos = int(self.x_lens[bs]) + seq_len = pos + y_len + + attn_mask[bs, :seq_len, :pos] = True + + ar_mask = ~torch.triu( + input=torch.ones( + size=( + y_len, + y_len, + ), + dtype=torch.bool, + ), + diagonal=1, + ) + attn_mask[bs, pos:seq_len, pos:seq_len] = ar_mask + + self.attn_mask = attn_mask + self.attn_mask = attn_mask.unsqueeze(0).expand(-1, decoder.n_head, -1, -1) + + self.id: int = -1 + + # Sage Attn & Transformer Engine Impl + self.cu_seqlens_q: Tensor + self.cu_seqlens_kv: Tensor diff --git a/GPT_SoVITS/Accelerate/PyTorch/t2s_engine.py b/GPT_SoVITS/Accelerate/PyTorch/t2s_engine.py new file mode 100644 index 00000000..53864e74 --- /dev/null +++ b/GPT_SoVITS/Accelerate/PyTorch/t2s_engine.py @@ -0,0 +1,220 @@ +import contextlib +import gc +import os +import sys +import time +import traceback +from importlib import import_module + +import torch +from rich.progress import BarColumn, Progress, TextColumn, TimeRemainingColumn + +from ..logger import console, logger +from .structs import T2SEngineProtocol, T2SRequest, T2SResult, T2SSession +from .t2s_model_abc import ( + CUDAGraphCacheABC, + T2SDecoderABC, + TorchProfiler, +) + +torch.set_grad_enabled(False) + + +class T2SEngine(T2SEngineProtocol): + def __init__( + self, + decoder_model: T2SDecoderABC, + device: torch.device = torch.device("cpu"), + dtype: torch.dtype = torch.float32, + ) -> None: + assert device.type in {"cpu", "cuda", "mps", "xpu", "mtia"} + assert dtype in {torch.float16, torch.bfloat16, torch.float32} + + self.device = device + self.dtype = dtype + + self.decoder_model: T2SDecoderABC = decoder_model.to(self.device, self.dtype) + + self.graphcache: CUDAGraphCacheABC = self.init_cache() + + def _handle_request(self, request: T2SRequest): + with self.device: + decoder = self.decoder_model + session = T2SSession(decoder, request, device=self.device, dtype=self.dtype) + batch_idx = torch.arange(session.bsz) + + t1 = 0.0 + infer_speed = 0.0 + infer_time = 0.0 + + torch_profiler = TorchProfiler(request.debug) + with ( + torch_profiler.profiler(), + Progress( + TextColumn("[cyan]{task.description}"), + BarColumn(), + TextColumn("{task.completed}/{task.total}"), + TimeRemainingColumn(), + console=console, + transient=True, + ) as progress, + ): + max_token = min(1800 - session.input_pos.max(), 1500) + task = progress.add_task("T2S Decoding", total=max_token) + + for idx in range(max_token): + progress.update(task, advance=1) + if idx == 0: + session.kv_cache = decoder.init_cache(session.bsz) + xy_dec = decoder.h.prefill(session.xy_pos, session.kv_cache, session.attn_mask) + xy_dec = xy_dec[None, batch_idx, session.input_pos - 1] + else: + if request.use_cuda_graph and session.graph is None and torch.cuda.is_available(): + self.graphcache.assign_graph(session) + + with torch_profiler.record("AR"): + if session.graph: + assert session.stream + session.stream.wait_stream(torch.cuda.default_stream()) + with torch.cuda.stream(session.stream): + session.xy_pos_.copy_(session.xy_pos) + session.graph.replay() + xy_dec = session.xy_dec_.clone() + else: + args, kwds = decoder.pre_forward(session) + xy_dec = decoder.h( + session.input_pos, + session.xy_pos, + session.kv_cache, + *args, + **kwds, + ) + + with torch.cuda.stream(session.stream) if session.stream is not None else contextlib.nullcontext(): + decoder.post_forward(idx, session) + logits = decoder.ar_predict_layer(xy_dec[:, -1]) + + if idx == 0: + logits[:, -1] = float("-inf") + + with torch_profiler.record("Sampling"): + samples = session.sample( + logits=logits, + previous_tokens=session.y[:, : session.y_len + idx], + top_k=request.top_k, + top_p=request.top_p, + repetition_penalty=request.repetition_penalty, + temperature=request.temperature, + ) + session.y[batch_idx, session.y_len + idx] = samples + session.input_pos.add_(1) + + with torch_profiler.record("EOS"): + argmax_token = torch.argmax(logits, dim=-1) + sample_token = samples.squeeze(1) + EOS_mask = (argmax_token == decoder.EOS) | (sample_token == decoder.EOS) + + newly_done_mask = EOS_mask & (~session.completed) + newly_done_indices = newly_done_mask.nonzero() + + if newly_done_indices.numel() > 0: + for i in newly_done_indices: + session.y_results[i] = session.y[i, session.y_len : session.y_len + idx] + session.completed[newly_done_indices] = True + + if torch.all(session.completed).item(): + if session.y[:, session.y_len :].sum() == 0: + session.y_results = [torch.tensor(0) for _ in range(session.bsz)] + logger.error("Bad Zero Prediction") + else: + logger.info( + f"T2S Decoding EOS {session.prefill_len.tolist().__str__().strip('[]')} -> {[i.size(-1) for i in session.y_results].__str__().strip('[]')}" + ) + logger.info(f"Infer Speed: {(idx - 1) / (time.perf_counter() - t1):.2f} token/s") + infer_time = time.perf_counter() - t1 + infer_speed = (idx - 1) / infer_time + break + + if (request.early_stop_num != -1 and idx >= request.early_stop_num) or idx == max_token - 1: + for i in range(session.bsz): + if not session.completed[i].item(): + session.y_results[i] = session.y[i, session.y_len : session.y_len + 1499] + session.completed[i] = True + logger.error("Bad Full Prediction") + break + + with torch_profiler.record("NextPos"): + y_emb = decoder.ar_audio_embedding(samples) + session.xy_pos = decoder.ar_audio_position(session.input_pos - session.x_lens, y_emb) + + if idx == 1: + torch_profiler.start() + t1 = time.perf_counter() + + if idx == 51: + torch_profiler.end() + + if idx % 100 == 0: + match session.device.type: + case "cuda": + torch.cuda.empty_cache() + case "mps": + torch.mps.empty_cache() + case "xpu": + torch.xpu.empty_cache() + case "mtia": + torch.mtia.empty_cache() + + match session.device.type: + case "cuda": + if session.stream is not None: + torch.cuda.current_stream().wait_stream(session.stream) + torch.cuda.empty_cache() + case "mps": + torch.mps.empty_cache() + case "xpu": + torch.xpu.empty_cache() + case "mtia": + torch.mtia.empty_cache() + case "cpu": + gc.collect() + + torch_profiler.end() + if request.use_cuda_graph and torch.cuda.is_available(): + self.graphcache.release_graph(session) + + return session.y_results[: request.valid_length], infer_speed, infer_time + + def generate(self, request: T2SRequest): + try: + result, infer_speed, infer_time = self._handle_request(request) + t2s_result = T2SResult(result=result, infer_speed=(infer_speed, infer_time), status="Success") + except Exception as e: + t2s_result = T2SResult(status="Error", exception=e, traceback=traceback.format_exc()) + return t2s_result + + @staticmethod + def load_decoder(weights_path: os.PathLike, max_batch_size: int = 1, backend: str = "Flash Attn CUDAGraph"): + logger.info(f"Loading Text2Semantic Weights from {weights_path} with {backend} Backend") + module_path = f".backends.{backend.lower().replace('-', '_')}" + decoder_cls_name = "T2SDecoder" + decoder_mod = import_module(module_path, package=__package__) + decoder_cls: type[T2SDecoderABC] = getattr(decoder_mod, decoder_cls_name) + dict_s1 = torch.load(weights_path, map_location="cpu", weights_only=False, mmap=True) + config = dict_s1["config"] + decoder: T2SDecoderABC = decoder_cls(config, max_batch_size=max_batch_size) + state_dict = dict_s1["weight"] + decoder.load_state_dict(state_dict) + + return decoder.eval() + + def init_cache(self): + assert self.decoder_model + + module_name = self.decoder_model.__class__.__module__ + module = sys.modules.get(module_name) + assert module + + target_class: type[CUDAGraphCacheABC] = getattr(module, "CUDAGraphCache") + + return target_class(self.decoder_model) diff --git a/GPT_SoVITS/Accelerate/PyTorch/t2s_model_abc.py b/GPT_SoVITS/Accelerate/PyTorch/t2s_model_abc.py new file mode 100644 index 00000000..ad2bddcd --- /dev/null +++ b/GPT_SoVITS/Accelerate/PyTorch/t2s_model_abc.py @@ -0,0 +1,670 @@ +""" +Modified From https://github.com/XXXXRT666/GPT-SoVITS +""" + +from __future__ import annotations + +import math +import os +import random +from abc import ABC, abstractmethod +from contextlib import nullcontext +from typing import MutableSequence + +import torch +import torch._inductor.config +import torch.nn.functional as F +from torch.cuda.graphs import CUDAGraph +from torch.profiler import ProfilerAction, tensorboard_trace_handler + +from . import nn +from .structs import KVCacheProtocol, T2SDecoderProtocol, T2SSession + +Tensor = torch.Tensor + + +class TokenEmbedding(nn.Module): + def __init__( + self, + embedding_dim: int, + vocab_size: int, + ): + super().__init__() + + self.vocab_size = vocab_size + self.embedding_dim = embedding_dim + + self.word_embeddings = nn.Embedding(self.vocab_size, self.embedding_dim) + + @property + def weight(self) -> Tensor: + return self.word_embeddings.weight + + def embedding(self, index: int) -> Tensor: + return self.word_embeddings.weight[index : index + 1] + + def __call__(self, x: Tensor): + x = self.word_embeddings(x) + return x + + +class SinePositionalEmbedding(nn.Module): + def __init__( + self, + embedding_dim: int, + scale: bool = False, + alpha: bool = False, + max_batch_size: int = 10, + max_seq_len: int = 1800, + ): + super().__init__() + self.embedding_dim = embedding_dim + self.x_scale = math.sqrt(embedding_dim) if scale else 1.0 + self.alpha = nn.Parameter(torch.ones(1), requires_grad=alpha) + self.max_batch_size = max_batch_size + self.max_seq_len = max_seq_len + + self.reverse = False + self.register_buffer("pe", torch.zeros(max_batch_size, max_seq_len, embedding_dim), persistent=False) + self.pe: torch.Tensor + self.compute_pe() + + def compute_pe(self): + """Reset the positional encodings.""" + if self.reverse: + position = torch.arange(self.max_seq_len - 1, -1, -1.0, dtype=torch.float32).unsqueeze(1) + else: + position = torch.arange(self.max_seq_len, dtype=torch.float32).unsqueeze(1) + div_term = torch.exp( + torch.arange(0, self.embedding_dim, 2, dtype=torch.float32) * -(math.log(10000.0) / self.embedding_dim) + ) + pe = self.pe + pe[:, :, 0::2] = torch.sin(position * div_term) + pe[:, :, 1::2] = torch.cos(position * div_term) + + def __call__(self, input_pos: Tensor, x: Tensor) -> Tensor: + """ + Args: + input_pos (Tensor): [batch_size, ] + x (Tensor): [batch_size, 1, embed_dim] + + Returns: + embedded_x (Tensor): [batch_size, 1, embed_dim] + """ + + batch_size = x.shape[0] + pe_values = self.pe[torch.arange(batch_size), input_pos - 1] # (batch_size, embed_dim) + + return x * self.x_scale + self.alpha * pe_values.unsqueeze(1) # (batch_size, 1, embed_dim) + + def prefill(self, x: Tensor) -> Tensor: + """ + Args: + x (Tensor): [batch_size, seq_len, embed_dim] + + Returns: + embedded_x (Tensor): [batch_size, seq_len, embed_dim] + """ + + pe_values = self.pe[:, : x.shape[-2]] + return x * self.x_scale + self.alpha.item() * pe_values + + +class KVCacheABC(nn.Module, ABC, KVCacheProtocol): + def __init__(self, batch_size: int, max_seq_length: int, n_heads: int, head_dim: int) -> None: + super().__init__() + + self.n_head = n_heads + self.head_dim = head_dim + self.batch_size = batch_size + self.max_seq_length = max_seq_length + + self.k_cache: Tensor + self.v_cache: Tensor + + def empty(self): + self.k_cache.zero_() + self.v_cache.zero_() + + @abstractmethod + def update(self, input_pos: Tensor, k_val: Tensor, v_val: Tensor, *args, **kwds) -> tuple[Tensor, Tensor]: ... + + @abstractmethod + def prefill_kv(self, k_val: Tensor, v_val: Tensor) -> None: ... + + def sync_cache(self, kv_cache: KVCacheProtocol): + self.k_cache.copy_(kv_cache.k_cache) + self.v_cache.copy_(kv_cache.v_cache) + + +class KVCacheNHD(KVCacheABC): + def __init__(self, batch_size, max_seq_length, n_heads, head_dim): + super().__init__(batch_size, max_seq_length, n_heads, head_dim) + + assert batch_size > 0 + cache_shape = (batch_size, max_seq_length, n_heads, head_dim) + + self.register_buffer("k_cache", torch.zeros(size=cache_shape), persistent=False) + self.register_buffer("v_cache", torch.zeros(size=cache_shape), persistent=False) + + def update(self, input_pos: Tensor, k_val: Tensor, v_val: Tensor): + # input_pos: [B, ], k_val: [B, 1, H, D] + + index = ( + (input_pos - 1) + .unsqueeze(-1) + .unsqueeze(-1) + .unsqueeze(-1) + .expand( + -1, + -1, + self.n_head, + self.head_dim, + ) + .to(torch.int64) + ) # (bs, 1, num_head, head_dim) + + k_out = self.k_cache + v_out = self.v_cache + k_out.scatter_(1, index, k_val) + v_out.scatter_(1, index, v_val) + + return k_out, v_out + + def empty(self): + self.k_cache.zero_() + self.v_cache.zero_() + + def prefill_kv(self, k_val: Tensor, v_val: Tensor): + # input_pos: int, k_val: [B, S, H, D] + + self.k_cache[:, : k_val.shape[1]] = k_val + self.v_cache[:, : v_val.shape[1]] = v_val + + +class KVCacheHND(KVCacheABC): + def __init__(self, batch_size, max_seq_length, n_heads, head_dim): + super().__init__(batch_size, max_seq_length, n_heads, head_dim) + + cache_shape = (batch_size, n_heads, max_seq_length, head_dim) + + self.register_buffer("k_cache", torch.zeros(size=cache_shape), persistent=False) + self.register_buffer("v_cache", torch.zeros(size=cache_shape), persistent=False) + + def update(self, input_pos: Tensor, k_val: Tensor, v_val: Tensor): + # input_pos: [B, ], k_val: [B, H, 1, D] + + index = ( + (input_pos - 1) + .unsqueeze(-1) + .unsqueeze(-1) + .unsqueeze(-1) + .expand( + -1, + self.n_head, + -1, + self.head_dim, + ) + .to(torch.int64) + ) # (bs, num_head, 1, head_dim) + + k_out = self.k_cache + v_out = self.v_cache + k_out.scatter_(2, index, k_val) + v_out.scatter_(2, index, v_val) + + return k_out, v_out + + def empty(self): + self.k_cache.zero_() + self.v_cache.zero_() + + def prefill_kv(self, k_val: Tensor, v_val: Tensor): + # input_pos: int, k_val: [B, S, H, D] + + self.k_cache[..., : k_val.shape[1], :] = k_val.transpose(1, 2) + self.v_cache[..., : v_val.shape[1], :] = v_val.transpose(1, 2) + + +class KVCacheHNDVarlen(KVCacheABC): + def __init__(self, batch_size, max_seq_length, n_heads, head_dim): + super().__init__(batch_size, max_seq_length, n_heads, head_dim) + + cache_shape = (batch_size, n_heads, max_seq_length, head_dim) + self.cache_idx: Tensor + + self.register_buffer("cache_idx", torch.arange(batch_size), persistent=False) + self.register_buffer("k_cache", torch.zeros(size=cache_shape), persistent=False) + self.register_buffer("v_cache", torch.zeros(size=cache_shape), persistent=False) + + def update(self, input_pos: Tensor, k_val: Tensor, v_val: Tensor): + # input_pos: [B, ], k_val: [B, H, 1, D] + + k_out = self.k_cache + v_out = self.v_cache + + ip0 = input_pos - 1 + + k_out[self.cache_idx, :, ip0, None] = k_val + v_out[self.cache_idx, :, ip0, None] = v_val + + return k_out, v_out + + def empty(self): + self.k_cache.zero_() + self.v_cache.zero_() + + def prefill_kv(self, k_val: Tensor, v_val: Tensor): + # input_pos: int, k_val: [B, S, H, D] + + self.k_cache[..., : k_val.shape[1], :] = k_val.transpose(1, 2) + self.v_cache[..., : v_val.shape[1], :] = v_val.transpose(1, 2) + + +class AttentionABC(nn.Module, ABC): + def __init__(self, n_head: int, hidden_dim: int, max_seq_length: int): + super().__init__() + + self.n_head = n_head + self.hidden_dim = hidden_dim + assert hidden_dim % n_head == 0 + self.head_dim = hidden_dim // n_head + + self.max_seq_length = max_seq_length + + # key, query, value projections for all heads, but in a batch + self.in_proj: nn.Linear + self.out_proj: nn.Linear + + self._register_load_state_dict_pre_hook(self.load_hook) + + def load_hook(self, state_dict: dict[str, Tensor], prefix, *args): + keys_to_modify = [key for key in state_dict if "in_proj_" in key] + for key in keys_to_modify: + new_key = key.replace("in_proj_", "in_proj.") # in_proj_ -> in_proj. + state_dict[new_key] = state_dict.pop(key) + + @abstractmethod + def __call__(self, x: Tensor, input_pos: Tensor, kv_cache: KVCacheProtocol, *args, **kwds) -> Tensor: ... + + def prefill(self, x: Tensor, kv_cache: KVCacheProtocol, attn_mask: Tensor) -> Tensor: + bsz, seqlen, _ = x.shape + + q, k, v = self.in_proj(x.unsqueeze(0)).chunk(3, dim=-1) + + q, k, v = map(lambda x: x.contiguous().view(bsz, seqlen, self.n_head, self.head_dim), (q, k, v)) + + kv_cache.prefill_kv(k, v) + + q, k, v = map(lambda x: x.transpose(1, 2), (q, k, v)) + + attn = F.scaled_dot_product_attention(q, k, v, attn_mask) + + attn = attn.transpose(1, 2).contiguous().view(1, -1, self.hidden_dim) + + output = self.out_proj(attn) + + return output + + +class FeedForward(nn.Module): + def __init__(self, dim: int, hidden_dim: int) -> None: + super().__init__() + + self.linear1 = nn.Linear(dim, hidden_dim, bias=True) + self.linear2 = nn.Linear(hidden_dim, dim, bias=True) + + def __call__(self, x: Tensor): + return self.linear2(F.relu(self.linear1(x), inplace=True)) + + +class TransformerBlockABC(nn.Module, ABC): + def __init__(self, n_head: int, ffn_dim: int, hidden_dim: int, max_seq_length: int) -> None: + super().__init__() + + self.hidden_dim = hidden_dim + self.max_seq_length = max_seq_length + + self.attention: AttentionABC + self.feed_forward: FeedForward + self.attention_norm: nn.LayerNorm + self.ffn_norm: nn.LayerNorm + + self._register_load_state_dict_pre_hook(self.load_hook) + + def load_hook(self, state_dict: dict[str, Tensor], prefix, *args): + for key in list(state_dict.keys()): + new_key = ( + key.replace("self_attn", "attention") + .replace("linear", "feed_forward.linear") + .replace("norm1", "attention_norm") + .replace("norm2", "ffn_norm") + ) + state_dict[new_key] = state_dict.pop(key) + + def __call__(self, x: Tensor, input_pos: Tensor, kv_cache: KVCacheProtocol, *args, **kwds): + h = self.attention_norm( + x + + self.attention( + x, + input_pos, + kv_cache, + *args, + **kwds, + ) + ) + out = self.ffn_norm(h + self.feed_forward(h)) + return out + + def prefill( + self, + x: Tensor, + kv_cache: KVCacheProtocol, + attn_mask: Tensor, + ) -> Tensor: + h = self.attention_norm( + x + + self.attention.prefill( + x, + kv_cache, + attn_mask, + ) + ) + out = self.ffn_norm(h + self.feed_forward(h)) + return out + + +class TransformerDecoderABC(nn.Module, ABC): + def __init__( + self, + hidden_dim: int, + n_layer: int, + n_head: int, + ffn_dim: int, + vocab_size: int, + max_seq_length: int, + max_batch_size: int, + ) -> None: + super().__init__() + + self.hidden_dim = hidden_dim + self.n_head = n_head + assert hidden_dim % n_head == 0 + + self.head_dim = hidden_dim // n_head + self.vocab_size = vocab_size + + self.n_layer = n_layer + + self.layers: MutableSequence[TransformerBlockABC] + + self.max_seq_length = max_seq_length + self.max_batch_size = max_batch_size + + def __call__(self, input_pos: Tensor, x: Tensor, kv_caches: MutableSequence[KVCacheProtocol], *args, **kwds): + for layer, kv_cache in zip(self.layers, kv_caches): + x = layer(x, input_pos, kv_cache, *args, **kwds) + return x + + def prefill(self, x: Tensor, kv_caches: MutableSequence[KVCacheProtocol], attn_mask: Tensor): + for layer, kv_cache in zip(self.layers, kv_caches): + x = layer.prefill(x, kv_cache, attn_mask) + return x + + +class T2SDecoderABC(nn.Module, ABC, T2SDecoderProtocol): + def __init__( + self, + config: dict, + max_seq_length: int = 1800, + max_batch_size: int = 10, + ) -> None: + super().__init__() + + hidden_dim: int = config["model"]["hidden_dim"] + embedding_dim: int = config["model"]["embedding_dim"] + n_head: int = config["model"]["head"] + n_layer: int = config["model"]["n_layer"] + vocab_size: int = config["model"]["vocab_size"] + phoneme_vocab_size: int = config["model"]["phoneme_vocab_size"] + EOS: int = config["model"]["EOS"] + ffn_dim: int = hidden_dim * 4 + + self.n_layer = int(n_layer) + self.hidden_dim = int(hidden_dim) + self.n_head = int(n_head) + assert hidden_dim % n_head == 0 + + self.head_dim = int(hidden_dim // n_head) + self.embedding_dim = int(embedding_dim) + self.ffn_dim = int(ffn_dim) + self.vocab_size = int(vocab_size) + self.phoneme_vocab_size = int(phoneme_vocab_size) + self.max_seq_length = max_seq_length + self.max_batch_size = max_batch_size + self.EOS = EOS + assert self.EOS == self.vocab_size - 1 + + self.bert_proj: nn.Linear + self.ar_predict_layer: nn.Linear + self.h: TransformerDecoderABC + + self.kv_class: type[KVCacheABC] + + self.GraphCache: CUDAGraphCacheABC | None + + self.ar_text_embedding = TokenEmbedding(self.embedding_dim, self.phoneme_vocab_size) + self.ar_text_position = SinePositionalEmbedding( + self.embedding_dim, + scale=False, + alpha=True, + max_batch_size=max_batch_size, + max_seq_len=max_seq_length, + ) + self.ar_audio_embedding = TokenEmbedding(self.embedding_dim, self.vocab_size) + self.ar_audio_position = SinePositionalEmbedding( + self.embedding_dim, + scale=False, + alpha=True, + max_batch_size=max_batch_size, + max_seq_len=max_seq_length, + ) + + self._register_load_state_dict_pre_hook(self.load_hook) + + def load_hook(self, state_dict: dict[str, Tensor], prefix, *args): + model_keys = [key for key in state_dict if key.startswith("model.")] + for key in model_keys: + new_key = key[len("model.") :] + state_dict[new_key] = state_dict.pop(key) + + def init_cache(self, bsz: int = 0) -> MutableSequence[KVCacheProtocol]: + bsz = bsz or self.h.max_batch_size + assert bsz <= self.h.max_batch_size + seq_lens = self.h.max_seq_length + dtype = self.bert_proj.bias.dtype + kvclass = self.kv_class + + return nn.ModuleList( + [kvclass(bsz, seq_lens, self.n_head, self.head_dim) for _ in range(self.n_layer)], + ).to(self.device, dtype) # type: ignore + + def embed( + self, + x: list[torch.Tensor], + y: torch.Tensor, + bert_features: list[torch.Tensor], + ): + x_len: list[int] = [i.shape[0] for i in x] + x_len_max = max(x_len) + xy_pos = torch.zeros((len(x), x_len_max + y.shape[1], self.embedding_dim)).to(bert_features[0].dtype) + + bert_features = list(map(lambda x: x.transpose(0, 1), bert_features)) + + y_len = y.shape[1] + y_emb = self.ar_audio_embedding(y) + y_pos = self.ar_audio_position.prefill(y_emb) + + for bs, (x_, len_, bert_feature) in enumerate(zip(x, x_len, bert_features)): + x_emb = self.ar_text_embedding(x_) + bert = self.bert_proj(bert_feature) + x_emb = x_emb + bert + x_pos = self.ar_text_position.prefill(x_emb.unsqueeze(0)) + xy_pos[[bs], :len_] = x_pos + xy_pos[[bs], len_ : len_ + y_len] = y_pos + + return xy_pos + + def compile(self, *args, **kwds): + # Experimental features to reduce compilation times, will be on by default in future + torch._inductor.config.triton.cudagraph_skip_dynamic_graphs = True + torch._inductor.config.coordinate_descent_tuning = True + torch._inductor.config.triton.unique_kernel_names = True + torch._inductor.config.fx_graph_cache = True + torch._inductor.config.triton.cudagraph_trees = True + torch._inductor.config.triton.cudagraph_support_input_mutation = True + self.h.compile(fullgraph=True, mode="reduce-overhead") + + def capture( + self, input_pos: Tensor, x: Tensor, x_dec: Tensor, kv_caches: MutableSequence[KVCacheProtocol], *args, **kwds + ) -> CUDAGraph: + assert torch.cuda.is_available() + s = torch.cuda.Stream() + s.wait_stream(torch.cuda.current_stream()) + + graph = torch.cuda.CUDAGraph() + + with torch.cuda.stream(s): # type: ignore + for _ in range(5): + self.h(input_pos, x, kv_caches, *args, **kwds) + torch.cuda.current_stream().wait_stream(s) + + with torch.cuda.graph(graph): + x_dec.copy_(self.h(input_pos, x, kv_caches, *args, **kwds)) + torch.cuda.synchronize() + + return graph + + @abstractmethod + def pre_forward(self, session: T2SSession) -> tuple[list[Tensor], dict[str, Tensor]]: + return list(), dict() + + @abstractmethod + def post_forward(self, idx: int, session: T2SSession) -> None: + return + + +class CUDAGraphCacheABC(ABC): + def __init__( + self, + decoder: T2SDecoderABC, + enabled: bool = False, + ) -> None: + if torch.cuda.is_available() and enabled: + self.device: torch.device = decoder.device + self.dtype = decoder.bert_proj.bias.dtype + + self.assigned: bool = False + + self.decoder: T2SDecoderABC = decoder + self.kv_cache: MutableSequence[KVCacheProtocol] = decoder.init_cache(decoder.max_batch_size) + self.xy_pos = torch.rand(size=(decoder.max_batch_size, 1, decoder.embedding_dim), device=self.device).to( + self.dtype + ) + self.xy_dec = self.xy_pos.clone() + + self.input_pos = torch.tensor([10] * decoder.max_batch_size, device=self.device).int() + self.graph: torch.cuda.CUDAGraph | None = None + self.stream: torch.cuda.Stream | None + + self.id: int = random.randint(1, 2**32 - 1) + + def assign_graph(self, session: T2SSession): + if self.graph is None: + args, kwds = self.decoder.pre_forward(session) + graph = self.decoder.capture(self.input_pos, self.xy_pos, self.xy_dec, self.kv_cache, *args, **kwds) + self.graph = graph + self.stream = torch.cuda.Stream() # type: ignore + + if self.assigned is False: + self.get_cache_graph(session) + session.id = self.id + self.assigned = True + else: + self.capture_new_graph(session) + + @abstractmethod + def release_graph(self, session: T2SSession): ... + + @abstractmethod + def get_cache_graph(self, session: T2SSession): + pass + + @abstractmethod + def capture_new_graph(self, session: T2SSession): + pass + + +class TorchProfiler: + def __init__(self, debug: bool, log_dir: str = "./profiler") -> None: + self.debug = debug + self.log_dir = log_dir + self.__profiler: torch.profiler.profile + + if self.debug and not os.path.exists(self.log_dir): + os.makedirs(self.log_dir) + + self.tensorboard_handler = tensorboard_trace_handler(self.log_dir) + + def profiler_callback(self, prof: torch.profiler.profile): + print(prof.key_averages().table(sort_by="cuda_time_total", row_limit=30)) + print(prof.key_averages().table(sort_by="cpu_time_total", row_limit=30)) + self.tensorboard_handler(prof) + + @staticmethod + def three_step_schedule(step: int) -> ProfilerAction: + if step == 0: + return ProfilerAction.NONE + elif step == 1: + return ProfilerAction.RECORD + elif step == 2: + return ProfilerAction.RECORD_AND_SAVE + else: + return ProfilerAction.NONE + + def start(self): + if not self.debug: + return + assert self.__profiler is not None + self.__profiler.step() + + def end(self): + if not self.debug: + return + assert self.__profiler is not None + self.__profiler.step() + + def profiler(self): + if self.debug: + activities_list = [torch.profiler.ProfilerActivity.CPU] + if torch.cuda.is_available(): + activities_list.append(torch.profiler.ProfilerActivity.CUDA) + + self.__profiler = torch.profiler.profile( + activities=activities_list, + record_shapes=True, + with_stack=True, + with_modules=True, + profile_memory=True, + schedule=self.three_step_schedule, + on_trace_ready=self.profiler_callback, + ) + return self.__profiler + else: + return nullcontext() + + def record(self, func_name: str): + if self.debug: + return torch.profiler.record_function(func_name) + else: + return nullcontext() diff --git a/GPT_SoVITS/Accelerate/__init__.py b/GPT_SoVITS/Accelerate/__init__.py new file mode 100644 index 00000000..ea07fe5d --- /dev/null +++ b/GPT_SoVITS/Accelerate/__init__.py @@ -0,0 +1,12 @@ +from . import MLX, PyTorch +from .logger import logger, tb +from .PyTorch import T2SEngineTorch, T2SRequest, T2SResult + +backends = PyTorch.backends + MLX.backends + +backends = [ + b.replace("_", "-").title().replace("Mlx", "MLX").replace("Mps", "MPS").replace("Cuda", "CUDA") for b in backends +] + + +__all__ = ["T2SEngineTorch", "T2SRequest", "T2SResult", "backends", "MLX", "PyTorch", "logger", "tb"] diff --git a/GPT_SoVITS/Accelerate/logger.py b/GPT_SoVITS/Accelerate/logger.py new file mode 100644 index 00000000..772544d6 --- /dev/null +++ b/GPT_SoVITS/Accelerate/logger.py @@ -0,0 +1,36 @@ +import sys + +from loguru import logger +from rich.console import Console +from rich.traceback import Traceback, install + +install() + + +def rich_format(record): + level = record["level"].name + color = { + "DEBUG": "green", + "INFO": "cyan", + "WARNING": "yellow", + "ERROR": "red", + "CRITICAL": "magenta", + }.get(level, "black") + return f"[bold {color}][{level}][/bold {color}] {record['message']}" + + +def tb(show_locals: bool = True): + exc_type, exc_value, exc_tb = sys.exc_info() + assert exc_type + assert exc_value + tb = Traceback.from_exception(exc_type, exc_value, exc_tb, show_locals=show_locals) + + return tb + + +console = Console() + +logger.remove() +logger.add(console.print, format=rich_format) + +__all__ = ["logger", "console", "tb"] diff --git a/GPT_SoVITS/inference_cli.py b/GPT_SoVITS/inference_cli.py deleted file mode 100644 index 459a3d36..00000000 --- a/GPT_SoVITS/inference_cli.py +++ /dev/null @@ -1,86 +0,0 @@ -import argparse -import os -import soundfile as sf - -from tools.i18n.i18n import I18nAuto -from GPT_SoVITS.inference_webui import change_gpt_weights, change_sovits_weights, get_tts_wav - -i18n = I18nAuto() - - -def synthesize( - GPT_model_path, - SoVITS_model_path, - ref_audio_path, - ref_text_path, - ref_language, - target_text_path, - target_language, - output_path, -): - # Read reference text - with open(ref_text_path, "r", encoding="utf-8") as file: - ref_text = file.read() - - # Read target text - with open(target_text_path, "r", encoding="utf-8") as file: - target_text = file.read() - - # Change model weights - change_gpt_weights(gpt_path=GPT_model_path) - change_sovits_weights(sovits_path=SoVITS_model_path) - - # Synthesize audio - synthesis_result = get_tts_wav( - ref_wav_path=ref_audio_path, - prompt_text=ref_text, - prompt_language=i18n(ref_language), - text=target_text, - text_language=i18n(target_language), - top_p=1, - temperature=1, - ) - - result_list = list(synthesis_result) - - if result_list: - last_sampling_rate, last_audio_data = result_list[-1] - output_wav_path = os.path.join(output_path, "output.wav") - sf.write(output_wav_path, last_audio_data, last_sampling_rate) - print(f"Audio saved to {output_wav_path}") - - -def main(): - parser = argparse.ArgumentParser(description="GPT-SoVITS Command Line Tool") - parser.add_argument("--gpt_model", required=True, help="Path to the GPT model file") - parser.add_argument("--sovits_model", required=True, help="Path to the SoVITS model file") - parser.add_argument("--ref_audio", required=True, help="Path to the reference audio file") - parser.add_argument("--ref_text", required=True, help="Path to the reference text file") - parser.add_argument( - "--ref_language", required=True, choices=["中文", "英文", "日文"], help="Language of the reference audio" - ) - parser.add_argument("--target_text", required=True, help="Path to the target text file") - parser.add_argument( - "--target_language", - required=True, - choices=["中文", "英文", "日文", "中英混合", "日英混合", "多语种混合"], - help="Language of the target text", - ) - parser.add_argument("--output_path", required=True, help="Path to the output directory") - - args = parser.parse_args() - - synthesize( - args.gpt_model, - args.sovits_model, - args.ref_audio, - args.ref_text, - args.ref_language, - args.target_text, - args.target_language, - args.output_path, - ) - - -if __name__ == "__main__": - main() diff --git a/GPT_SoVITS/inference_gui.py b/GPT_SoVITS/inference_gui.py deleted file mode 100644 index 379f7fa8..00000000 --- a/GPT_SoVITS/inference_gui.py +++ /dev/null @@ -1,316 +0,0 @@ -import os -import sys -from PyQt5.QtCore import QEvent -from PyQt5.QtWidgets import QApplication, QMainWindow, QLabel, QLineEdit, QPushButton, QTextEdit -from PyQt5.QtWidgets import QGridLayout, QVBoxLayout, QWidget, QFileDialog, QStatusBar, QComboBox -import soundfile as sf - -from tools.i18n.i18n import I18nAuto - -i18n = I18nAuto() - -from inference_webui import gpt_path, sovits_path, change_gpt_weights, change_sovits_weights, get_tts_wav - - -class GPTSoVITSGUI(QMainWindow): - GPT_Path = gpt_path - SoVITS_Path = sovits_path - - def __init__(self): - super().__init__() - - self.setWindowTitle("GPT-SoVITS GUI") - self.setGeometry(800, 450, 950, 850) - - self.setStyleSheet(""" - QWidget { - background-color: #a3d3b1; - } - - QTabWidget::pane { - background-color: #a3d3b1; - } - - QTabWidget::tab-bar { - alignment: left; - } - - QTabBar::tab { - background: #8da4bf; - color: #ffffff; - padding: 8px; - } - - QTabBar::tab:selected { - background: #2a3f54; - } - - QLabel { - color: #000000; - } - - QPushButton { - background-color: #4CAF50; - color: white; - padding: 8px; - border: 1px solid #4CAF50; - border-radius: 4px; - } - - QPushButton:hover { - background-color: #45a049; - border: 1px solid #45a049; - box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.1); - } - """) - - license_text = ( - "本软件以MIT协议开源, 作者不对软件具备任何控制力, 使用软件者、传播软件导出的声音者自负全责. " - "如不认可该条款, 则不能使用或引用软件包内任何代码和文件. 详见根目录LICENSE." - ) - license_label = QLabel(license_text) - license_label.setWordWrap(True) - - self.GPT_model_label = QLabel("选择GPT模型:") - self.GPT_model_input = QLineEdit() - self.GPT_model_input.setPlaceholderText("拖拽或选择文件") - self.GPT_model_input.setText(self.GPT_Path) - self.GPT_model_input.setReadOnly(True) - self.GPT_model_button = QPushButton("选择GPT模型文件") - self.GPT_model_button.clicked.connect(self.select_GPT_model) - - self.SoVITS_model_label = QLabel("选择SoVITS模型:") - self.SoVITS_model_input = QLineEdit() - self.SoVITS_model_input.setPlaceholderText("拖拽或选择文件") - self.SoVITS_model_input.setText(self.SoVITS_Path) - self.SoVITS_model_input.setReadOnly(True) - self.SoVITS_model_button = QPushButton("选择SoVITS模型文件") - self.SoVITS_model_button.clicked.connect(self.select_SoVITS_model) - - self.ref_audio_label = QLabel("上传参考音频:") - self.ref_audio_input = QLineEdit() - self.ref_audio_input.setPlaceholderText("拖拽或选择文件") - self.ref_audio_input.setReadOnly(True) - self.ref_audio_button = QPushButton("选择音频文件") - self.ref_audio_button.clicked.connect(self.select_ref_audio) - - self.ref_text_label = QLabel("参考音频文本:") - self.ref_text_input = QLineEdit() - self.ref_text_input.setPlaceholderText("直接输入文字或上传文本") - self.ref_text_button = QPushButton("上传文本") - self.ref_text_button.clicked.connect(self.upload_ref_text) - - self.ref_language_label = QLabel("参考音频语言:") - self.ref_language_combobox = QComboBox() - self.ref_language_combobox.addItems(["中文", "英文", "日文", "中英混合", "日英混合", "多语种混合"]) - self.ref_language_combobox.setCurrentText("多语种混合") - - self.target_text_label = QLabel("合成目标文本:") - self.target_text_input = QLineEdit() - self.target_text_input.setPlaceholderText("直接输入文字或上传文本") - self.target_text_button = QPushButton("上传文本") - self.target_text_button.clicked.connect(self.upload_target_text) - - self.target_language_label = QLabel("合成音频语言:") - self.target_language_combobox = QComboBox() - self.target_language_combobox.addItems(["中文", "英文", "日文", "中英混合", "日英混合", "多语种混合"]) - self.target_language_combobox.setCurrentText("多语种混合") - - self.output_label = QLabel("输出音频路径:") - self.output_input = QLineEdit() - self.output_input.setPlaceholderText("拖拽或选择文件") - self.output_input.setReadOnly(True) - self.output_button = QPushButton("选择文件夹") - self.output_button.clicked.connect(self.select_output_path) - - self.output_text = QTextEdit() - self.output_text.setReadOnly(True) - - self.add_drag_drop_events( - [ - self.GPT_model_input, - self.SoVITS_model_input, - self.ref_audio_input, - self.ref_text_input, - self.target_text_input, - self.output_input, - ] - ) - - self.synthesize_button = QPushButton("合成") - self.synthesize_button.clicked.connect(self.synthesize) - - self.clear_output_button = QPushButton("清空输出") - self.clear_output_button.clicked.connect(self.clear_output) - - self.status_bar = QStatusBar() - - main_layout = QVBoxLayout() - - input_layout = QGridLayout(self) - input_layout.setSpacing(10) - - input_layout.addWidget(license_label, 0, 0, 1, 3) - - input_layout.addWidget(self.GPT_model_label, 1, 0) - input_layout.addWidget(self.GPT_model_input, 2, 0, 1, 2) - input_layout.addWidget(self.GPT_model_button, 2, 2) - - input_layout.addWidget(self.SoVITS_model_label, 3, 0) - input_layout.addWidget(self.SoVITS_model_input, 4, 0, 1, 2) - input_layout.addWidget(self.SoVITS_model_button, 4, 2) - - input_layout.addWidget(self.ref_audio_label, 5, 0) - input_layout.addWidget(self.ref_audio_input, 6, 0, 1, 2) - input_layout.addWidget(self.ref_audio_button, 6, 2) - - input_layout.addWidget(self.ref_language_label, 7, 0) - input_layout.addWidget(self.ref_language_combobox, 8, 0, 1, 1) - input_layout.addWidget(self.ref_text_label, 9, 0) - input_layout.addWidget(self.ref_text_input, 10, 0, 1, 2) - input_layout.addWidget(self.ref_text_button, 10, 2) - - input_layout.addWidget(self.target_language_label, 11, 0) - input_layout.addWidget(self.target_language_combobox, 12, 0, 1, 1) - input_layout.addWidget(self.target_text_label, 13, 0) - input_layout.addWidget(self.target_text_input, 14, 0, 1, 2) - input_layout.addWidget(self.target_text_button, 14, 2) - - input_layout.addWidget(self.output_label, 15, 0) - input_layout.addWidget(self.output_input, 16, 0, 1, 2) - input_layout.addWidget(self.output_button, 16, 2) - - main_layout.addLayout(input_layout) - - output_layout = QVBoxLayout() - output_layout.addWidget(self.output_text) - main_layout.addLayout(output_layout) - - main_layout.addWidget(self.synthesize_button) - - main_layout.addWidget(self.clear_output_button) - - main_layout.addWidget(self.status_bar) - - self.central_widget = QWidget() - self.central_widget.setLayout(main_layout) - self.setCentralWidget(self.central_widget) - - def dragEnterEvent(self, event): - if event.mimeData().hasUrls(): - event.acceptProposedAction() - - def dropEvent(self, event): - if event.mimeData().hasUrls(): - file_paths = [url.toLocalFile() for url in event.mimeData().urls()] - if len(file_paths) == 1: - self.update_ref_audio(file_paths[0]) - else: - self.update_ref_audio(", ".join(file_paths)) - - def add_drag_drop_events(self, widgets): - for widget in widgets: - widget.setAcceptDrops(True) - widget.installEventFilter(self) - - def eventFilter(self, obj, event): - if event.type() in (QEvent.DragEnter, QEvent.Drop): - mime_data = event.mimeData() - if mime_data.hasUrls(): - event.acceptProposedAction() - - return super().eventFilter(obj, event) - - def select_GPT_model(self): - file_path, _ = QFileDialog.getOpenFileName(self, "选择GPT模型文件", "", "GPT Files (*.ckpt)") - if file_path: - self.GPT_model_input.setText(file_path) - - def select_SoVITS_model(self): - file_path, _ = QFileDialog.getOpenFileName(self, "选择SoVITS模型文件", "", "SoVITS Files (*.pth)") - if file_path: - self.SoVITS_model_input.setText(file_path) - - def select_ref_audio(self): - file_path, _ = QFileDialog.getOpenFileName(self, "选择参考音频文件", "", "Audio Files (*.wav *.mp3)") - if file_path: - self.update_ref_audio(file_path) - - def upload_ref_text(self): - file_path, _ = QFileDialog.getOpenFileName(self, "选择文本文件", "", "Text Files (*.txt)") - if file_path: - with open(file_path, "r", encoding="utf-8") as file: - content = file.read() - self.ref_text_input.setText(content) - - def upload_target_text(self): - file_path, _ = QFileDialog.getOpenFileName(self, "选择文本文件", "", "Text Files (*.txt)") - if file_path: - with open(file_path, "r", encoding="utf-8") as file: - content = file.read() - self.target_text_input.setText(content) - - def select_output_path(self): - options = QFileDialog.Options() - options |= QFileDialog.DontUseNativeDialog - options |= QFileDialog.ShowDirsOnly - - folder_dialog = QFileDialog() - folder_dialog.setOptions(options) - folder_dialog.setFileMode(QFileDialog.Directory) - - if folder_dialog.exec_(): - folder_path = folder_dialog.selectedFiles()[0] - self.output_input.setText(folder_path) - - def update_ref_audio(self, file_path): - self.ref_audio_input.setText(file_path) - - def clear_output(self): - self.output_text.clear() - - def synthesize(self): - GPT_model_path = self.GPT_model_input.text() - SoVITS_model_path = self.SoVITS_model_input.text() - ref_audio_path = self.ref_audio_input.text() - language_combobox = self.ref_language_combobox.currentText() - language_combobox = i18n(language_combobox) - ref_text = self.ref_text_input.text() - target_language_combobox = self.target_language_combobox.currentText() - target_language_combobox = i18n(target_language_combobox) - target_text = self.target_text_input.text() - output_path = self.output_input.text() - - if GPT_model_path != self.GPT_Path: - change_gpt_weights(gpt_path=GPT_model_path) - self.GPT_Path = GPT_model_path - if SoVITS_model_path != self.SoVITS_Path: - change_sovits_weights(sovits_path=SoVITS_model_path) - self.SoVITS_Path = SoVITS_model_path - - synthesis_result = get_tts_wav( - ref_wav_path=ref_audio_path, - prompt_text=ref_text, - prompt_language=language_combobox, - text=target_text, - text_language=target_language_combobox, - ) - - result_list = list(synthesis_result) - - if result_list: - last_sampling_rate, last_audio_data = result_list[-1] - output_wav_path = os.path.join(output_path, "output.wav") - sf.write(output_wav_path, last_audio_data, last_sampling_rate) - - result = "Audio saved to " + output_wav_path - - self.status_bar.showMessage("合成完成！输出路径：" + output_wav_path, 5000) - self.output_text.append("处理结果：\n" + result) - - -if __name__ == "__main__": - app = QApplication(sys.argv) - mainWin = GPTSoVITSGUI() - mainWin.show() - sys.exit(app.exec_()) diff --git a/GPT_SoVITS/inference_webui.py b/GPT_SoVITS/inference_webui.py index a361ed58..6f60e5c8 100644 --- a/GPT_SoVITS/inference_webui.py +++ b/GPT_SoVITS/inference_webui.py @@ -1,36 +1,50 @@ -""" -按中英混合识别 -按日英混合识别 -多语种启动切分识别语种 -全部按中文识别 -全部按英文识别 -全部按日文识别 -""" -import psutil -import os - -def set_high_priority(): - """把当前 Python 进程设为 HIGH_PRIORITY_CLASS""" - if os.name != "nt": - return # 仅 Windows 有效 - p = psutil.Process(os.getpid()) - try: - p.nice(psutil.HIGH_PRIORITY_CLASS) - print("已将进程优先级设为 High") - except psutil.AccessDenied: - print("权限不足，无法修改优先级（请用管理员运行）") -set_high_priority() +import argparse +import contextlib import json import logging import os +import random import re -import sys import traceback import warnings +from pathlib import Path +from time import time as ttime +import gradio as gr +import librosa +import numpy as np +import psutil import torch import torchaudio -from text.LangSegmenter import LangSegmenter +from peft import LoraConfig, get_peft_model +from transformers import AutoModelForMaskedLM, AutoTokenizer + +from config import ( + change_choices, + get_dtype, + get_weights_names, + name2gpt_path, + name2sovits_path, + pretrained_sovits_name, +) +from config import ( + infer_device as default_device, +) +from GPT_SoVITS.Accelerate import MLX, PyTorch, T2SRequest, backends +from GPT_SoVITS.feature_extractor import cnhubert +from GPT_SoVITS.module.mel_processing import mel_spectrogram_torch, spectrogram_torch +from GPT_SoVITS.module.models import Generator, SynthesizerTrn, SynthesizerTrnV3 +from GPT_SoVITS.process_ckpt import get_sovits_version_from_path_fast, load_sovits_new +from GPT_SoVITS.sv import SV +from GPT_SoVITS.text import cleaned_text_to_sequence +from GPT_SoVITS.text.cleaner import clean_text +from GPT_SoVITS.text.LangSegmenter import LangSegmenter +from tools.assets import css, js, top_html +from tools.i18n.i18n import I18nAuto, scan_language_list + +with contextlib.suppress(ImportError): + import mlx.core as mx + import mlx.utils as mxutils logging.getLogger("markdown_it").setLevel(logging.ERROR) logging.getLogger("urllib3").setLevel(logging.ERROR) @@ -42,101 +56,116 @@ logging.getLogger("torchaudio._extension").setLevel(logging.ERROR) logging.getLogger("multipart.multipart").setLevel(logging.ERROR) warnings.simplefilter(action="ignore", category=FutureWarning) -version = model_version = os.environ.get("version", "v2") +os.environ["TOKENIZERS_PARALLELISM"] = "false" -from config import change_choices, get_weights_names, name2gpt_path, name2sovits_path + +def set_high_priority(): + if os.name != "nt": + return + p = psutil.Process(os.getpid()) + with contextlib.suppress(psutil.AccessDenied): + p.nice(psutil.HIGH_PRIORITY_CLASS) + print("已将进程优先级设为 High") + + +set_high_priority() + +_LANG_RE = re.compile(r"^[a-z]{2}[_-][A-Z]{2}$") + + +def lang_type(text: str) -> str: + if text == "Auto": + return text + if not _LANG_RE.match(text): + raise argparse.ArgumentTypeError(f"Unspported Format: {text}, Expected ll_CC/ll-CC") + ll, cc = re.split(r"[_-]", text) + language = f"{ll}_{cc}" + if language in scan_language_list(): + return language + else: + return "Auto" + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="inference_webui", + description="python -s inference_webui.py zh_CN -i naive", + ) + p.add_argument( + "language", + nargs="?", + default="Auto", + type=lang_type, + help="Language Code, Such as zh_CN, en-US", + ) + p.add_argument( + "--backends", + "-b", + choices=backends, + default="naive", + help="AR Inference Backend", + required=False, + ) + p.add_argument( + "--device", + "-d", + default=str(default_device), + help="Inference Device", + required=False, + ) + p.add_argument( + "--port", + "-p", + default=9872, + type=int, + help="WebUI Binding Port", + required=False, + ) + p.add_argument( + "--share", + "-s", + default=False, + action="store_true", + help="Gradio Share Link", + required=False, + ) + p.add_argument( + "--cnhubert", + default="GPT_SoVITS/pretrained_models/chinese-hubert-base", + help="CNHuBERT Pretrain", + required=False, + ) + p.add_argument( + "--bert", + default="GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large", + help="BERT Pretrain", + required=False, + ) + + return p + + +args = build_parser().parse_args() SoVITS_names, GPT_names = get_weights_names() -from config import pretrained_sovits_name + +version = model_version = os.environ.get("version", "v2") path_sovits_v3 = pretrained_sovits_name["v3"] path_sovits_v4 = pretrained_sovits_name["v4"] is_exist_s2gv3 = os.path.exists(path_sovits_v3) is_exist_s2gv4 = os.path.exists(path_sovits_v4) -if os.path.exists("./weight.json"): - pass -else: - with open("./weight.json", "w", encoding="utf-8") as file: - json.dump({"GPT": {}, "SoVITS": {}}, file) - -with open("./weight.json", "r", encoding="utf-8") as file: - weight_data = file.read() - weight_data = json.loads(weight_data) - gpt_path = os.environ.get("gpt_path", weight_data.get("GPT", {}).get(version, GPT_names[-1])) - sovits_path = os.environ.get("sovits_path", weight_data.get("SoVITS", {}).get(version, SoVITS_names[0])) - if isinstance(gpt_path, list): - gpt_path = gpt_path[0] - if isinstance(sovits_path, list): - sovits_path = sovits_path[0] - -# print(2333333) -# print(os.environ["gpt_path"]) -# print(gpt_path) -# print(GPT_names) -# print(weight_data) -# print(weight_data.get("GPT", {})) -# print(version)###GPT version里没有s2的v2pro -# print(weight_data.get("GPT", {}).get(version, GPT_names[-1])) - -cnhubert_base_path = os.environ.get("cnhubert_base_path", "GPT_SoVITS/pretrained_models/chinese-hubert-base") -bert_path = os.environ.get("bert_path", "GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large") -infer_ttswebui = os.environ.get("infer_ttswebui", 9872) -infer_ttswebui = int(infer_ttswebui) -is_share = os.environ.get("is_share", "False") -is_share = eval(is_share) -if "_CUDA_VISIBLE_DEVICES" in os.environ: - os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["_CUDA_VISIBLE_DEVICES"] -is_half = eval(os.environ.get("is_half", "True")) and torch.cuda.is_available() -# is_half=False -punctuation = set(["!", "?", "…", ",", ".", "-", " "]) -import gradio as gr -import librosa -import numpy as np -from feature_extractor import cnhubert -from transformers import AutoModelForMaskedLM, AutoTokenizer - -cnhubert.cnhubert_base_path = cnhubert_base_path - -import random - -from GPT_SoVITS.module.models import Generator, SynthesizerTrn, SynthesizerTrnV3 +cnhubert_base_path = str(args.cuhubert) +bert_path = str(args.bert) +infer_ttswebui = int(args.port) +is_share = args.share == "True" -def set_seed(seed): - if seed == -1: - seed = random.randint(0, 1000000) - seed = int(seed) - random.seed(seed) - os.environ["PYTHONHASHSEED"] = str(seed) - np.random.seed(seed) - torch.manual_seed(seed) - torch.cuda.manual_seed(seed) +i18n = I18nAuto(language=args.language) +ar_backend: str = args.backends -# set_seed(42) - -from time import time as ttime - -from AR.models.t2s_lightning_module import Text2SemanticLightningModule -from peft import LoraConfig, get_peft_model -from text import cleaned_text_to_sequence -from text.cleaner import clean_text - -from tools.assets import css, js, top_html -from tools.i18n.i18n import I18nAuto, scan_language_list - -language = os.environ.get("language", "Auto") -language = sys.argv[-1] if sys.argv[-1] in scan_language_list() else language -i18n = I18nAuto(language=language) - -# os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1' # 确保直接启动推理UI时也能够设置。 - -if torch.cuda.is_available(): - device = "cuda" -else: - device = "cpu" - dict_language_v1 = { i18n("中文"): "all_zh", # 全部按中文识别 i18n("英文"): "en", # 全部按英文识别#######不变 @@ -160,21 +189,86 @@ dict_language_v2 = { } dict_language = dict_language_v1 if version == "v1" else dict_language_v2 +punctuation = set(["!", "?", "…", ",", ".", "-", " "]) +splits = {"，", "。", "？", "！", ",", ".", "?", "!", "~", ":", "：", "—", "…"} +v3v4set = {"v3", "v4"} + +infer_device = torch.device(args.device) +device = infer_device if infer_device.type == "cuda" else torch.device("cpu") + +dtype = get_dtype(device.index) +if_half = dtype == torch.float16 + tokenizer = AutoTokenizer.from_pretrained(bert_path) -bert_model = AutoModelForMaskedLM.from_pretrained(bert_path) -if is_half == True: - bert_model = bert_model.half().to(device) +bert_model = AutoModelForMaskedLM.from_pretrained(bert_path).to(infer_device, dtype) + +cnhubert.cnhubert_base_path = cnhubert_base_path +ssl_model = cnhubert.get_model().to(infer_device, dtype) + +spec_min = -12 +spec_max = 2 + + +def norm_spec(x): + return (x - spec_min) / (spec_max - spec_min) * 2 - 1 + + +def denorm_spec(x): + return (x + 1) / 2 * (spec_max - spec_min) + spec_min + + +def mel_fn(x): + return mel_spectrogram_torch( + y=x, + n_fft=1024, + num_mels=100, + sampling_rate=24000, + hop_size=256, + win_size=1024, + fmin=0, + fmax=None, + center=False, + ) + + +def mel_fn_v4(x): + return mel_spectrogram_torch( + y=x, + n_fft=1280, + num_mels=100, + sampling_rate=32000, + hop_size=320, + win_size=1280, + fmin=0, + fmax=None, + center=False, + ) + + +if os.path.exists("./weight.json"): + pass else: - bert_model = bert_model.to(device) + with open("./weight.json", "w", encoding="utf-8") as file: + json.dump({"GPT": {}, "SoVITS": {}}, file) + +with open("./weight.json", "r", encoding="utf-8") as file: + weight_data = file.read() + weight_data = json.loads(weight_data) + gpt_path = os.environ.get("gpt_path", weight_data.get("GPT", {}).get(version, GPT_names[-1])) + sovits_path = os.environ.get("sovits_path", weight_data.get("SoVITS", {}).get(version, SoVITS_names[0])) + if isinstance(gpt_path, list): + gpt_path = gpt_path[0] + if isinstance(sovits_path, list): + sovits_path = sovits_path[0] def get_bert_feature(text, word2ph): - with torch.no_grad(): - inputs = tokenizer(text, return_tensors="pt") - for i in inputs: - inputs[i] = inputs[i].to(device) - res = bert_model(**inputs, output_hidden_states=True) - res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()[1:-1] + inputs = tokenizer(text, return_tensors="pt") + for i in inputs: + inputs[i] = inputs[i].to(infer_device) + res = bert_model(**inputs, output_hidden_states=True) + res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()[1:-1] + assert len(word2ph) == len(text) phone_level_feature = [] for i in range(len(word2ph)): @@ -212,20 +306,6 @@ class DictToAttrRecursive(dict): raise AttributeError(f"Attribute {item} not found") -ssl_model = cnhubert.get_model() -if is_half == True: - ssl_model = ssl_model.half().to(device) -else: - ssl_model = ssl_model.to(device) - - -###todo:put them to process_ckpt and modify my_save func (save sovits weights), gpt save weights use my_save in process_ckpt -# symbol_version-model_version-if_lora_v3 -from process_ckpt import get_sovits_version_from_path_fast, load_sovits_new - -v3v4set = {"v3", "v4"} - - def change_sovits_weights(sovits_path, prompt_language=None, text_language=None): if "！" in sovits_path or "!" in sovits_path: sovits_path = name2sovits_path[sovits_path] @@ -234,10 +314,10 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None) print(sovits_path, version, model_version, if_lora_v3) is_exist = is_exist_s2gv3 if model_version == "v3" else is_exist_s2gv4 path_sovits = path_sovits_v3 if model_version == "v3" else path_sovits_v4 - if if_lora_v3 == True and is_exist == False: - info = path_sovits + "SoVITS %s" % model_version + i18n("底模缺失，无法加载相应 LoRA 权重") + if if_lora_v3 is True and is_exist is False: + info = f"{path_sovits} SoVITS {model_version} {i18n('底模缺失，无法加载相应 LoRA 权重')}" gr.Warning(info) - raise FileExistsError(info) + raise FileNotFoundError(info) dict_language = dict_language_v1 if version == "v1" else dict_language_v2 if prompt_language is not None and text_language is not None: if prompt_language in list(dict_language.keys()): @@ -291,7 +371,7 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None) version = hps.model.version # print("sovits版本:",hps.model.version) if model_version not in v3v4set: - if "Pro" not in model_version: + if "Pro" not in model_version: # type: ignore model_version = version else: hps.model.version = model_version @@ -308,23 +388,19 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None) hps.train.segment_size // hps.data.hop_length, n_speakers=hps.data.n_speakers, **hps.model, - ) + ).eval() if "pretrained" not in sovits_path: try: del vq_model.enc_q - except: + finally: pass - if is_half == True: - vq_model = vq_model.half().to(device) - else: - vq_model = vq_model.to(device) - vq_model.eval() - if if_lora_v3 == False: - print("loading sovits_%s" % model_version, vq_model.load_state_dict(dict_s2["weight"], strict=False)) + + if if_lora_v3 is False: + print(f">> loading sovits_{model_version}", vq_model.load_state_dict(dict_s2["weight"], strict=False)) else: path_sovits = path_sovits_v3 if model_version == "v3" else path_sovits_v4 print( - "loading sovits_%spretrained_G" % model_version, + ">> loading sovits_%spretrained_G" % model_version, vq_model.load_state_dict(load_sovits_new(path_sovits)["weight"], strict=False), ) lora_rank = dict_s2["lora_rank"] @@ -334,27 +410,29 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None) lora_alpha=lora_rank, init_lora_weights=True, ) - vq_model.cfm = get_peft_model(vq_model.cfm, lora_config) - print("loading sovits_%s_lora%s" % (model_version, lora_rank)) + vq_model.cfm = get_peft_model(vq_model.cfm, lora_config) # type: ignore + print(f">> loading sovits_{model_version}_lora{model_version}") vq_model.load_state_dict(dict_s2["weight"], strict=False) - vq_model.cfm = vq_model.cfm.merge_and_unload() + vq_model.cfm = vq_model.cfm.merge_and_unload() # pyright: ignore[reportAttributeAccessIssue, reportCallIssue] # torch.save(vq_model.state_dict(),"merge_win.pth") vq_model.eval() + vq_model = vq_model.to(infer_device, dtype) + yield ( {"__type__": "update", "choices": list(dict_language.keys())}, {"__type__": "update", "choices": list(dict_language.keys())}, - prompt_text_update, - prompt_language_update, - text_update, - text_language_update, + prompt_text_update, # type: ignore + prompt_language_update, # type: ignore + text_update, # type: ignore + text_language_update, # type: ignore { "__type__": "update", - "visible": visible_sample_steps, + "visible": visible_sample_steps, # type: ignore "value": 32 if model_version == "v3" else 8, "choices": [4, 8, 16, 32, 64, 128] if model_version == "v3" else [4, 8, 16, 32], }, - {"__type__": "update", "visible": visible_inp_refs}, + {"__type__": "update", "visible": visible_inp_refs}, # type: ignore {"__type__": "update", "value": False, "interactive": True if model_version not in v3v4set else False}, {"__type__": "update", "visible": True if model_version == "v3" else False}, {"__type__": "update", "value": i18n("合成语音"), "interactive": True}, @@ -367,28 +445,33 @@ def change_sovits_weights(sovits_path, prompt_language=None, text_language=None) f.write(json.dumps(data)) -try: +with contextlib.suppress(UnboundLocalError): next(change_sovits_weights(sovits_path)) -except: - pass def change_gpt_weights(gpt_path): if "！" in gpt_path or "!" in gpt_path: gpt_path = name2gpt_path[gpt_path] - global hz, max_sec, t2s_model, config - hz = 50 - dict_s1 = torch.load(gpt_path, map_location="cpu", weights_only=False) - config = dict_s1["config"] - max_sec = config["data"]["max_sec"] - t2s_model = Text2SemanticLightningModule(config, "****", is_train=False) - t2s_model.load_state_dict(dict_s1["weight"]) - if is_half == True: - t2s_model = t2s_model.half() - t2s_model = t2s_model.to(device) - t2s_model.eval() - # total = sum([param.nelement() for param in t2s_model.parameters()]) - # print("Number of parameter: %.2fM" % (total / 1e6)) + global t2s_engine, config + + if "mlx" in ar_backend.lower(): + t2s_engine = MLX.T2SEngineMLX( + MLX.T2SEngineMLX.load_decoder(Path(gpt_path), backend=ar_backend), + "mx.gpu", + dtype=dtype, + ) + # t2s_engine.decoder_model.compile() + total = sum((p[-1].size for p in mxutils.tree_flatten(t2s_engine.decoder_model.parameters()))) # type: ignore + else: + t2s_engine = PyTorch.T2SEngineTorch( + PyTorch.T2SEngineTorch.load_decoder(Path(gpt_path), backend=ar_backend), + device, + dtype=dtype, + ) + # t2s_engine.decoder_model.compile() + total = sum(p.numel() for p in t2s_engine.decoder_model.parameters()) + print(">> Number of parameter: %.2fM" % (total / 1e6)) + with open("./weight.json") as f: data = f.read() data = json.loads(data) @@ -398,43 +481,36 @@ def change_gpt_weights(gpt_path): change_gpt_weights(gpt_path) -os.environ["HF_ENDPOINT"] = "https://hf-mirror.com" -import torch - -now_dir = os.getcwd() def clean_hifigan_model(): global hifigan_model if hifigan_model: hifigan_model = hifigan_model.cpu() - hifigan_model = None - try: + del hifigan_model + if torch.cuda.is_available(): torch.cuda.empty_cache() - except: - pass + hifigan_model = None def clean_bigvgan_model(): global bigvgan_model if bigvgan_model: bigvgan_model = bigvgan_model.cpu() - bigvgan_model = None - try: + del bigvgan_model + if torch.cuda.is_available(): torch.cuda.empty_cache() - except: - pass + bigvgan_model = None def clean_sv_cn_model(): global sv_cn_model if sv_cn_model: sv_cn_model.embedding_model = sv_cn_model.embedding_model.cpu() - sv_cn_model = None - try: + del sv_cn_model + if torch.cuda.is_available(): torch.cuda.empty_cache() - except: - pass + sv_cn_model = None def init_bigvgan(): @@ -442,18 +518,14 @@ def init_bigvgan(): from BigVGAN import bigvgan bigvgan_model = bigvgan.BigVGAN.from_pretrained( - "%s/GPT_SoVITS/pretrained_models/models--nvidia--bigvgan_v2_24khz_100band_256x" % (now_dir,), + "./GPT_SoVITS/pretrained_models/models--nvidia--bigvgan_v2_24khz_100band_256x", use_cuda_kernel=False, ) # if True, RuntimeError: Ninja is required to load C++ extensions # remove weight norm in the model and set to eval mode bigvgan_model.remove_weight_norm() - bigvgan_model = bigvgan_model.eval() + bigvgan_model = bigvgan_model.to(infer_device, dtype).eval() clean_hifigan_model() clean_sv_cn_model() - if is_half == True: - bigvgan_model = bigvgan_model.half().to(device) - else: - bigvgan_model = bigvgan_model.to(device) def init_hifigan(): @@ -472,25 +544,20 @@ def init_hifigan(): hifigan_model.eval() hifigan_model.remove_weight_norm() state_dict_g = torch.load( - "%s/GPT_SoVITS/pretrained_models/gsv-v4-pretrained/vocoder.pth" % (now_dir,), + "./GPT_SoVITS/pretrained_models/gsv-v4-pretrained/vocoder.pth", map_location="cpu", weights_only=False, ) - print("loading vocoder", hifigan_model.load_state_dict(state_dict_g)) + print(">> loading vocoder", hifigan_model.load_state_dict(state_dict_g)) clean_bigvgan_model() clean_sv_cn_model() - if is_half == True: - hifigan_model = hifigan_model.half().to(device) - else: - hifigan_model = hifigan_model.to(device) - -from sv import SV + hifigan_model = hifigan_model.to(infer_device, dtype) def init_sv_cn(): global hifigan_model, bigvgan_model, sv_cn_model - sv_cn_model = SV(device, is_half) + sv_cn_model = SV(infer_device, is_half) clean_bigvgan_model() clean_hifigan_model() @@ -508,18 +575,13 @@ resample_transform_dict = {} def resample(audio_tensor, sr0, sr1, device): global resample_transform_dict - key = "%s-%s-%s" % (sr0, sr1, str(device)) + key = f"{sr0}-{sr1}-{str(device)}" if key not in resample_transform_dict: resample_transform_dict[key] = torchaudio.transforms.Resample(sr0, sr1).to(device) return resample_transform_dict[key](audio_tensor) def get_spepc(hps, filename, dtype, device, is_v2pro=False): - # audio = load_audio(filename, int(hps.data.sampling_rate)) - - # audio, sampling_rate = librosa.load(filename, sr=int(hps.data.sampling_rate)) - # audio = torch.FloatTensor(audio) - sr1 = int(hps.data.sampling_rate) audio, sr0 = torchaudio.load(filename) if sr0 != sr1: @@ -532,7 +594,7 @@ def get_spepc(hps, filename, dtype, device, is_v2pro=False): if audio.shape[0] == 2: audio = audio.mean(0).unsqueeze(0) - maxx = audio.abs().max() + maxx = float(audio.abs().max()) if maxx > 1: audio /= min(2, maxx) spec = spectrogram_torch( @@ -544,7 +606,7 @@ def get_spepc(hps, filename, dtype, device, is_v2pro=False): center=False, ) spec = spec.to(dtype) - if is_v2pro == True: + if is_v2pro is True: audio = resample(audio, sr1, 16000, device).to(dtype) return spec, audio @@ -556,9 +618,6 @@ def clean_text_inf(text, language, version): return phones, word2ph, norm_text -dtype = torch.float16 if is_half == True else torch.float32 - - def get_bert_inf(phones, word2ph, norm_text, language): language = language.replace("all_", "") if language == "zh": @@ -566,58 +625,38 @@ def get_bert_inf(phones, word2ph, norm_text, language): else: bert = torch.zeros( (1024, len(phones)), - dtype=torch.float16 if is_half == True else torch.float32, + dtype=torch.float16 if is_half is True else torch.float32, ).to(device) return bert -splits = { - "，", - "。", - "？", - "！", - ",", - ".", - "?", - "!", - "~", - ":", - "：", - "—", - "…", -} - - def get_first(text): pattern = "[" + "".join(re.escape(sep) for sep in splits) + "]" text = re.split(pattern, text)[0].strip() return text -from text import chinese - - def get_phones_and_bert(text, language, version, final=False): - text = re.sub(r' {2,}', ' ', text) + text = re.sub(r" {2,}", " ", text) textlist = [] langlist = [] if language == "all_zh": - for tmp in LangSegmenter.getTexts(text,"zh"): + for tmp in LangSegmenter.getTexts(text, "zh"): langlist.append(tmp["lang"]) textlist.append(tmp["text"]) elif language == "all_yue": - for tmp in LangSegmenter.getTexts(text,"zh"): + for tmp in LangSegmenter.getTexts(text, "zh"): if tmp["lang"] == "zh": tmp["lang"] = "yue" langlist.append(tmp["lang"]) textlist.append(tmp["text"]) elif language == "all_ja": - for tmp in LangSegmenter.getTexts(text,"ja"): + for tmp in LangSegmenter.getTexts(text, "ja"): langlist.append(tmp["lang"]) textlist.append(tmp["text"]) elif language == "all_ko": - for tmp in LangSegmenter.getTexts(text,"ko"): + for tmp in LangSegmenter.getTexts(text, "ko"): langlist.append(tmp["lang"]) textlist.append(tmp["text"]) elif language == "en": @@ -667,48 +706,6 @@ def get_phones_and_bert(text, language, version, final=False): return phones, bert.to(dtype), norm_text -from module.mel_processing import mel_spectrogram_torch, spectrogram_torch - -spec_min = -12 -spec_max = 2 - - -def norm_spec(x): - return (x - spec_min) / (spec_max - spec_min) * 2 - 1 - - -def denorm_spec(x): - return (x + 1) / 2 * (spec_max - spec_min) + spec_min - - -mel_fn = lambda x: mel_spectrogram_torch( - x, - **{ - "n_fft": 1024, - "win_size": 1024, - "hop_size": 256, - "num_mels": 100, - "sampling_rate": 24000, - "fmin": 0, - "fmax": None, - "center": False, - }, -) -mel_fn_v4 = lambda x: mel_spectrogram_torch( - x, - **{ - "n_fft": 1280, - "win_size": 1280, - "hop_size": 320, - "num_mels": 100, - "sampling_rate": 32000, - "fmin": 0, - "fmax": None, - "center": False, - }, -) - - def merge_short_text_in_array(texts, threshold): if (len(texts)) < 2: return texts @@ -732,19 +729,17 @@ sr_model = None def audio_sr(audio, sr): global sr_model - if sr_model == None: + if sr_model is None: from tools.audio_sr import AP_BWE try: - sr_model = AP_BWE(device, DictToAttrRecursive) + sr_model = AP_BWE(infer_device, DictToAttrRecursive) except FileNotFoundError: gr.Warning(i18n("你没有下载超分模型的参数，因此不进行超分。如想超分请先参照教程把文件下载好")) return audio.cpu().detach().numpy(), sr return sr_model(audio, sr) -##ref_wav_path+prompt_text+prompt_language+text(单个)+text_language+top_k+top_p+temperature -# cache_tokens={}#暂未实现清理机制 cache = {} @@ -794,20 +789,20 @@ def get_tts_wav( prompt_text = prompt_text.strip("\n") if prompt_text[-1] not in splits: prompt_text += "。" if prompt_language != "en" else "." - print(i18n("实际输入的参考文本:"), prompt_text) + print(">>", i18n("实际输入的参考文本:"), prompt_text) text = text.strip("\n") # if (text[0] not in splits and len(get_first(text)) < 4): text = "。" + text if text_language != "en" else "." + text - print(i18n("实际输入的目标文本:"), text) + print(">>", i18n("实际输入的目标文本:"), text) zero_wav = np.zeros( int(hps.data.sampling_rate * pause_second), - dtype=np.float16 if is_half == True else np.float32, + dtype=np.float16 if is_half is True else np.float32, ) zero_wav_torch = torch.from_numpy(zero_wav) - if is_half == True: - zero_wav_torch = zero_wav_torch.half().to(device) + if is_half is True: + zero_wav_torch = zero_wav_torch.half().to(infer_device) else: - zero_wav_torch = zero_wav_torch.to(device) + zero_wav_torch = zero_wav_torch.to(infer_device) if not ref_free: with torch.no_grad(): wav16k, sr = librosa.load(ref_wav_path, sr=16000) @@ -815,15 +810,17 @@ def get_tts_wav( gr.Warning(i18n("参考音频在3~10秒范围外，请更换！")) raise OSError(i18n("参考音频在3~10秒范围外，请更换！")) wav16k = torch.from_numpy(wav16k) - if is_half == True: - wav16k = wav16k.half().to(device) + if is_half is True: + wav16k = wav16k.half().to(infer_device) else: - wav16k = wav16k.to(device) + wav16k = wav16k.to(infer_device) wav16k = torch.cat([wav16k, zero_wav_torch]) ssl_content = ssl_model.model(wav16k.unsqueeze(0))["last_hidden_state"].transpose(1, 2) # .float() codes = vq_model.extract_latent(ssl_content) prompt_semantic = codes[0, 0] prompt = prompt_semantic.unsqueeze(0).to(device) + else: + prompt = torch.zeros((1, 0)).to(device, torch.int32) t1 = ttime() t.append(t1 - t0) @@ -840,14 +837,19 @@ def get_tts_wav( text = cut5(text) while "\n\n" in text: text = text.replace("\n\n", "\n") - print(i18n("实际输入的目标文本(切句后):"), text) + print(">>", i18n("实际输入的目标文本(切句后):"), text) texts = text.split("\n") texts = process_text(texts) texts = merge_short_text_in_array(texts, 5) audio_opt = [] ###s2v3暂不支持ref_free if not ref_free: - phones1, bert1, norm_text1 = get_phones_and_bert(prompt_text, prompt_language, version) + phones1, bert1, _ = get_phones_and_bert(prompt_text, prompt_language, version) + else: + phones1, bert1 = [], torch.zeros(1024, 0) + + infer_len: list[int] = [] + infer_time: list[float] = [] for i_text, text in enumerate(texts): # 解决输入目标文本的空行导致报错的问题 @@ -855,84 +857,96 @@ def get_tts_wav( continue if text[-1] not in splits: text += "。" if text_language != "en" else "." - print(i18n("实际输入的目标文本(每句):"), text) + print(">>", i18n("实际输入的目标文本(每句):"), text) phones2, bert2, norm_text2 = get_phones_and_bert(text, text_language, version) - print(i18n("前端处理后的文本(每句):"), norm_text2) - if not ref_free: - bert = torch.cat([bert1, bert2], 1) - all_phoneme_ids = torch.LongTensor(phones1 + phones2).to(device).unsqueeze(0) - else: - bert = bert2 - all_phoneme_ids = torch.LongTensor(phones2).to(device).unsqueeze(0) + print(">>", i18n("前端处理后的文本(每句):"), norm_text2) + + bert = torch.cat([bert1, bert2], 1) + all_phoneme_ids = torch.LongTensor(phones1 + phones2).to(device).unsqueeze(0) bert = bert.to(device).unsqueeze(0) all_phoneme_len = torch.tensor([all_phoneme_ids.shape[-1]]).to(device) t2 = ttime() - # cache_key="%s-%s-%s-%s-%s-%s-%s-%s"%(ref_wav_path,prompt_text,prompt_language,text,text_language,top_k,top_p,temperature) - # print(cache.keys(),if_freeze) - if i_text in cache and if_freeze == True: + if i_text in cache and if_freeze is True: pred_semantic = cache[i_text] else: - with torch.no_grad(): - pred_semantic, idx = t2s_model.model.infer_panel( - all_phoneme_ids, - all_phoneme_len, - None if ref_free else prompt, - bert, - # prompt_phone_len=ph_offset, - top_k=top_k, - top_p=top_p, - temperature=temperature, - early_stop_num=hz * max_sec, - ) - pred_semantic = pred_semantic[:, -idx:].unsqueeze(0) - cache[i_text] = pred_semantic + t2s_request = T2SRequest( + [all_phoneme_ids.squeeze(0)], + all_phoneme_len, + prompt, + [bert.squeeze(0)], + valid_length=1, + top_k=top_k, + top_p=top_p, + temperature=temperature, + early_stop_num=1500, + use_cuda_graph=torch.cuda.is_available(), + # debug=True, + ) + t2s_result = t2s_engine.generate(t2s_request) + if t2s_result.exception is not None: + print(t2s_result.traceback) + raise RuntimeError() + pred_semantic_list = t2s_result.result + assert pred_semantic_list, t2s_result.traceback + pred_semantic = pred_semantic_list[0].unsqueeze(0).to(infer_device) + infer_len.append(pred_semantic.shape[-1]) + infer_time.append(t2s_result.infer_speed[-1]) + + cache[i_text] = pred_semantic t3 = ttime() is_v2pro = model_version in {"v2Pro", "v2ProPlus"} - # print(23333,is_v2pro,model_version) ###v3不存在以下逻辑和inp_refs + sv_emb = [] if model_version not in v3v4set: refers = [] - if is_v2pro: - sv_emb = [] - if sv_cn_model == None: - init_sv_cn() + if is_v2pro and sv_cn_model is None: + init_sv_cn() if inp_refs: for path in inp_refs: try: #####这里加上提取sv的逻辑，要么一堆sv一堆refer，要么单个sv单个refer - refer, audio_tensor = get_spepc(hps, path.name, dtype, device, is_v2pro) + refer, audio_tensor = get_spepc(hps, path.name, dtype, infer_device, is_v2pro) refers.append(refer) if is_v2pro: + assert sv_cn_model sv_emb.append(sv_cn_model.compute_embedding3(audio_tensor)) - except: + except Exception as e: + print(e) traceback.print_exc() if len(refers) == 0: - refers, audio_tensor = get_spepc(hps, ref_wav_path, dtype, device, is_v2pro) + refers, audio_tensor = get_spepc(hps, ref_wav_path, dtype, infer_device, is_v2pro) refers = [refers] if is_v2pro: + assert sv_cn_model sv_emb = [sv_cn_model.compute_embedding3(audio_tensor)] if is_v2pro: audio = vq_model.decode( - pred_semantic, torch.LongTensor(phones2).to(device).unsqueeze(0), refers, speed=speed, sv_emb=sv_emb - )[0][0] + pred_semantic, + torch.LongTensor(phones2).to(infer_device).unsqueeze(0), + refers, + speed=speed, + sv_emb=sv_emb, + )[0][0] # type: ignore else: audio = vq_model.decode( - pred_semantic, torch.LongTensor(phones2).to(device).unsqueeze(0), refers, speed=speed - )[0][0] + pred_semantic, + torch.LongTensor(phones2).to(infer_device).unsqueeze(0), + refers, + speed=speed, + )[0][0] # type: ignore else: - refer, audio_tensor = get_spepc(hps, ref_wav_path, dtype, device) - phoneme_ids0 = torch.LongTensor(phones1).to(device).unsqueeze(0) - phoneme_ids1 = torch.LongTensor(phones2).to(device).unsqueeze(0) - fea_ref, ge = vq_model.decode_encp(prompt.unsqueeze(0), phoneme_ids0, refer) + refer, audio_tensor = get_spepc(hps, ref_wav_path, dtype, infer_device) + phoneme_ids0 = torch.LongTensor(phones1).to(infer_device).unsqueeze(0) + phoneme_ids1 = torch.LongTensor(phones2).to(infer_device).unsqueeze(0) + fea_ref, ge = vq_model.decode_encp(prompt.unsqueeze(0), phoneme_ids0, refer) # type: ignore ref_audio, sr = torchaudio.load(ref_wav_path) - ref_audio = ref_audio.to(device).float() + ref_audio = ref_audio.to(infer_device).float() if ref_audio.shape[0] == 2: ref_audio = ref_audio.mean(0).unsqueeze(0) tgt_sr = 24000 if model_version == "v3" else 32000 if sr != tgt_sr: - ref_audio = resample(ref_audio, sr, tgt_sr, device) - # print("ref_audio",ref_audio.abs().mean()) + ref_audio = resample(ref_audio, sr, tgt_sr, infer_device) mel2 = mel_fn(ref_audio) if model_version == "v3" else mel_fn_v4(ref_audio) mel2 = norm_spec(mel2) T_min = min(mel2.shape[2], fea_ref.shape[2]) @@ -946,7 +960,7 @@ def get_tts_wav( T_min = Tref chunk_len = Tchunk - T_min mel2 = mel2.to(dtype) - fea_todo, ge = vq_model.decode_encp(pred_semantic, phoneme_ids1, refer, ge, speed) + fea_todo, ge = vq_model.decode_encp(pred_semantic, phoneme_ids1, refer, ge, speed) # type: ignore cfm_resss = [] idx = 0 while 1: @@ -955,7 +969,7 @@ def get_tts_wav( break idx += chunk_len fea = torch.cat([fea_ref, fea_todo_chunk], 2).transpose(2, 1) - cfm_res = vq_model.cfm.inference( + cfm_res = vq_model.cfm.inference( # type: ignore fea, torch.LongTensor([fea.size(1)]).to(fea.device), mel2, sample_steps, inference_cfg_rate=0 ) cfm_res = cfm_res[:, :, mel2.shape[2] :] @@ -965,14 +979,14 @@ def get_tts_wav( cfm_res = torch.cat(cfm_resss, 2) cfm_res = denorm_spec(cfm_res) if model_version == "v3": - if bigvgan_model == None: + if bigvgan_model is None: init_bigvgan() else: # v4 - if hifigan_model == None: + if hifigan_model is None: init_hifigan() vocoder_model = bigvgan_model if model_version == "v3" else hifigan_model with torch.inference_mode(): - wav_gen = vocoder_model(cfm_res) + wav_gen = vocoder_model(cfm_res) # type: ignore audio = wav_gen[0][0] # .cpu().detach().numpy() max_audio = torch.abs(audio).max() # 简单防止16bit爆音 if max_audio > 1: @@ -982,7 +996,7 @@ def get_tts_wav( t4 = ttime() t.extend([t2 - t1, t3 - t2, t4 - t3]) t1 = ttime() - print("%.3f\t%.3f\t%.3f\t%.3f" % (t[0], sum(t[1::3]), sum(t[2::3]), sum(t[3::3]))) + audio_opt = torch.cat(audio_opt, 0) # np.concatenate if model_version in {"v1", "v2", "v2Pro", "v2ProPlus"}: opt_sr = 32000 @@ -990,14 +1004,30 @@ def get_tts_wav( opt_sr = 24000 else: opt_sr = 48000 # v4 - if if_sr == True and opt_sr == 24000: - print(i18n("音频超分中")) + if if_sr is True and opt_sr == 24000: + print(">>", i18n("音频超分中")) audio_opt, opt_sr = audio_sr(audio_opt.unsqueeze(0), opt_sr) max_audio = np.abs(audio_opt).max() if max_audio > 1: audio_opt /= max_audio else: audio_opt = audio_opt.cpu().detach().numpy() + + t0 = t[0] + t1 = sum(t[1::3]) + t2 = sum(t[2::3]) + t3 = sum(t[3::3]) + + infer_speed_avg = sum(infer_len) / sum(infer_time) + rtf_value = sum(t) / (audio_opt.__len__() / opt_sr) + + print(f">> Time Stamps: {t0:.3f}\t{t1:.3f}\t{t2:.3f}\t{t3:.3f}") + print(f">> Infer Speed: {infer_speed_avg:.2f} Token/s") + print(f">> RTF: {rtf_value:.2f}") + + gr.Info(f"{infer_speed_avg:.2f} Token/s", title="Infer Speed") + gr.Info(f"{rtf_value:.2f}", title="RTF") + yield opt_sr, (audio_opt * 32767).astype(np.int16) @@ -1023,7 +1053,7 @@ def split(todo_text): def cut1(inp): inp = inp.strip("\n") inps = split(inp) - split_idx = list(range(0, len(inps), 4)) + split_idx: list[int | None] = list(range(0, len(inps) + 1, 4)) split_idx[-1] = None if len(split_idx) > 1: opts = [] @@ -1052,7 +1082,6 @@ def cut2(inp): tmp_str = "" if tmp_str != "": opts.append(tmp_str) - # print(opts) if len(opts) > 1 and len(opts[-1]) < 50: ##如果最后一个太短了，和前一个合一起 opts[-2] = opts[-2] + opts[-1] opts = opts[:-1] @@ -1062,7 +1091,7 @@ def cut2(inp): def cut3(inp): inp = inp.strip("\n") - opts = ["%s" % item for item in inp.strip("。").split("。")] + opts = inp.strip("。").split("。") opts = [item for item in opts if not set(item).issubset(punctuation)] return "\n".join(opts) @@ -1141,7 +1170,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css ) with gr.Group(): gr.Markdown(html_center(i18n("模型切换"), "h3")) - with gr.Row(): + with gr.Row(equal_height=True): GPT_dropdown = gr.Dropdown( label=i18n("GPT模型列表"), choices=sorted(GPT_names, key=custom_sort_key), @@ -1159,8 +1188,15 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css refresh_button = gr.Button(i18n("刷新模型路径"), variant="primary", scale=14) refresh_button.click(fn=change_choices, inputs=[], outputs=[SoVITS_dropdown, GPT_dropdown]) gr.Markdown(html_center(i18n("*请上传并填写参考信息"), "h3")) - with gr.Row(): - inp_ref = gr.Audio(label=i18n("请上传3~10秒内参考音频，超过会报错！"), type="filepath", scale=13) + with gr.Row(equal_height=True): + inp_ref = gr.Audio( + label=i18n("请上传3~10秒内参考音频，超过会报错！"), + type="filepath", + sources="upload", + scale=13, + editable=False, + waveform_options={"show_recording_waveform": False}, + ) with gr.Column(scale=13): ref_text_free = gr.Checkbox( label=i18n("开启无参考文本模式。不填参考文本亦相当于开启。") @@ -1223,7 +1259,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css visible=False if model_version != "v3" else True, ) gr.Markdown(html_center(i18n("*请填写需要合成的目标文本和语种模式"), "h3")) - with gr.Row(): + with gr.Row(equal_height=True): with gr.Column(scale=13): text = gr.Textbox(label=i18n("需要合成的文本"), value="", lines=26, max_lines=26) with gr.Column(scale=7): @@ -1255,7 +1291,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css show_label=True, scale=1, ) - with gr.Row(): + with gr.Row(equal_height=True): speed = gr.Slider( minimum=0.6, maximum=1.65, step=0.05, label=i18n("语速"), value=1, interactive=True, scale=1 ) @@ -1278,13 +1314,14 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css temperature = gr.Slider( minimum=0, maximum=1, step=0.05, label=i18n("temperature"), value=1, interactive=True, scale=1 ) - # with gr.Column(): - # gr.Markdown(value=i18n("手工调整音素。当音素框不为空时使用手工音素输入推理，无视目标文本框。")) - # phoneme=gr.Textbox(label=i18n("音素框"), value="") - # get_phoneme_button = gr.Button(i18n("目标文本转音素"), variant="primary") - with gr.Row(): + with gr.Row(equal_height=True): inference_button = gr.Button(value=i18n("合成语音"), variant="primary", size="lg", scale=25) - output = gr.Audio(label=i18n("输出的语音"), scale=14) + output = gr.Audio( + label=i18n("输出的语音"), + scale=14, + waveform_options={"show_recording_waveform": False}, + editable=False, + ) inference_button.click( get_tts_wav, @@ -1327,24 +1364,9 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css ) GPT_dropdown.change(change_gpt_weights, [GPT_dropdown], []) - # gr.Markdown(value=i18n("文本切分工具。太长的文本合成出来效果不一定好，所以太长建议先切。合成会根据文本的换行分开合成再拼起来。")) - # with gr.Row(): - # text_inp = gr.Textbox(label=i18n("需要合成的切分前文本"), value="") - # button1 = gr.Button(i18n("凑四句一切"), variant="primary") - # button2 = gr.Button(i18n("凑50字一切"), variant="primary") - # button3 = gr.Button(i18n("按中文句号。切"), variant="primary") - # button4 = gr.Button(i18n("按英文句号.切"), variant="primary") - # button5 = gr.Button(i18n("按标点符号切"), variant="primary") - # text_opt = gr.Textbox(label=i18n("切分后文本"), value="") - # button1.click(cut1, [text_inp], [text_opt]) - # button2.click(cut2, [text_inp], [text_opt]) - # button3.click(cut3, [text_inp], [text_opt]) - # button4.click(cut4, [text_inp], [text_opt]) - # button5.click(cut5, [text_inp], [text_opt]) - # gr.Markdown(html_center(i18n("后续将支持转音素、手工修改音素、语音合成分步执行。"))) if __name__ == "__main__": - app.queue().launch( # concurrency_count=511, max_size=1022 + app.queue(api_open=False, default_concurrency_limit=511).launch( # concurrency_count=511, max_size=1022 server_name="0.0.0.0", inbrowser=True, share=is_share, diff --git a/GPT_SoVITS/inference_webui_fast.py b/GPT_SoVITS/inference_webui_fast.py index 51a120f1..92315981 100644 --- a/GPT_SoVITS/inference_webui_fast.py +++ b/GPT_SoVITS/inference_webui_fast.py @@ -315,7 +315,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css with gr.Column(): # with gr.Group(): gr.Markdown(value=i18n("模型切换")) - with gr.Row(): + with gr.Row(equal_height=True): GPT_dropdown = gr.Dropdown( label=i18n("GPT模型列表"), choices=sorted(GPT_names, key=custom_sort_key), @@ -331,18 +331,22 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css refresh_button = gr.Button(i18n("刷新模型路径"), variant="primary") refresh_button.click(fn=change_choices, inputs=[], outputs=[SoVITS_dropdown, GPT_dropdown]) - with gr.Row(): + with gr.Row(equal_height=True): with gr.Column(): gr.Markdown(value=i18n("*请上传并填写参考信息")) - with gr.Row(): - inp_ref = gr.Audio(label=i18n("主参考音频(请上传3~10秒内参考音频，超过会报错！)"), type="filepath") + with gr.Row(equal_height=True): + inp_ref = gr.Audio( + label=i18n("主参考音频(请上传3~10秒内参考音频，超过会报错！)"), + type="filepath", + waveform_options={"show_recording_waveform": False}, + ) inp_refs = gr.File( label=i18n("辅参考音频(可选多个，或不选)"), file_count="multiple", visible=True if model_version != "v3" else False, ) prompt_text = gr.Textbox(label=i18n("主参考音频的文本"), value="", lines=2) - with gr.Row(): + with gr.Row(equal_height=True): prompt_language = gr.Dropdown( label=i18n("主参考音频的语种"), choices=list(dict_language.keys()), value=i18n("中文") ) @@ -368,26 +372,26 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css with gr.Group(): gr.Markdown(value=i18n("推理设置")) - with gr.Row(): + with gr.Row(equal_height=True): with gr.Column(): - with gr.Row(): + with gr.Row(equal_height=True): batch_size = gr.Slider( minimum=1, maximum=200, step=1, label=i18n("batch_size"), value=20, interactive=True ) sample_steps = gr.Radio( label=i18n("采样步数(仅对V3/4生效)"), value=32, choices=[4, 8, 16, 32, 64, 128], visible=True ) - with gr.Row(): + with gr.Row(equal_height=True): fragment_interval = gr.Slider( minimum=0.01, maximum=1, step=0.01, label=i18n("分段间隔(秒)"), value=0.3, interactive=True ) speed_factor = gr.Slider( minimum=0.6, maximum=1.65, step=0.05, label="语速", value=1.0, interactive=True ) - with gr.Row(): + with gr.Row(equal_height=True): top_k = gr.Slider(minimum=1, maximum=100, step=1, label=i18n("top_k"), value=5, interactive=True) top_p = gr.Slider(minimum=0, maximum=1, step=0.05, label=i18n("top_p"), value=1, interactive=True) - with gr.Row(): + with gr.Row(equal_height=True): temperature = gr.Slider( minimum=0, maximum=1, step=0.05, label=i18n("temperature"), value=1, interactive=True ) @@ -396,7 +400,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css ) with gr.Column(): - with gr.Row(): + with gr.Row(equal_height=True): how_to_cut = gr.Dropdown( label=i18n("怎么切"), choices=[ @@ -415,7 +419,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css label=i18n("音频超采样(仅对V3生效))"), value=False, interactive=True, show_label=True ) - with gr.Row(): + with gr.Row(equal_height=True): parallel_infer = gr.Checkbox(label=i18n("并行推理"), value=True, interactive=True, show_label=True) split_bucket = gr.Checkbox( label=i18n("数据分桶(并行推理时会降低一点计算量)"), @@ -424,12 +428,15 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css show_label=True, ) - with gr.Row(): + with gr.Row(equal_height=True): seed = gr.Number(label=i18n("随机种子"), value=-1) keep_random = gr.Checkbox(label=i18n("保持随机"), value=True, interactive=True, show_label=True) - output = gr.Audio(label=i18n("输出的语音")) - with gr.Row(): + output = gr.Audio( + label=i18n("输出的语音"), + waveform_options={"show_recording_waveform": False}, + ) + with gr.Row(equal_height=True): inference_button = gr.Button(i18n("合成语音"), variant="primary") stop_infer = gr.Button(i18n("终止合成"), variant="primary") @@ -485,7 +492,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css "文本切分工具。太长的文本合成出来效果不一定好，所以太长建议先切。合成会根据文本的换行分开合成再拼起来。" ) ) - with gr.Row(): + with gr.Row(equal_height=True): text_inp = gr.Textbox(label=i18n("需要合成的切分前文本"), value="", lines=4) with gr.Column(): _how_to_cut = gr.Radio( diff --git a/GPT_SoVITS/prepare_datasets/1-get-text.py b/GPT_SoVITS/prepare_datasets/1-get-text.py index 8d83e79a..7e2be0e3 100644 --- a/GPT_SoVITS/prepare_datasets/1-get-text.py +++ b/GPT_SoVITS/prepare_datasets/1-get-text.py @@ -1,113 +1,24 @@ -# -*- coding: utf-8 -*- - +import argparse import os - -inp_text = os.environ.get("inp_text") -inp_wav_dir = os.environ.get("inp_wav_dir") -exp_name = os.environ.get("exp_name") -i_part = os.environ.get("i_part") -all_parts = os.environ.get("all_parts") -if "_CUDA_VISIBLE_DEVICES" in os.environ: - os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["_CUDA_VISIBLE_DEVICES"] -opt_dir = os.environ.get("opt_dir") -bert_pretrained_dir = os.environ.get("bert_pretrained_dir") -import torch - -is_half = eval(os.environ.get("is_half", "True")) and torch.cuda.is_available() -version = os.environ.get("version", None) -import traceback import os.path -from text.cleaner import clean_text +import traceback +from multiprocessing import Process, Queue, set_start_method + +import torch +from rich.progress import track from transformers import AutoModelForMaskedLM, AutoTokenizer + +from GPT_SoVITS.Accelerate import logger, tb +from GPT_SoVITS.text.cleaner import clean_text from tools.my_utils import clean_path -# inp_text=sys.argv[1] -# inp_wav_dir=sys.argv[2] -# exp_name=sys.argv[3] -# i_part=sys.argv[4] -# all_parts=sys.argv[5] -# os.environ["CUDA_VISIBLE_DEVICES"]=sys.argv[6]#i_gpu -# opt_dir="/data/docker/liujing04/gpt-vits/fine_tune_dataset/%s"%exp_name -# bert_pretrained_dir="/data/docker/liujing04/bert-vits2/Bert-VITS2-master20231106/bert/chinese-roberta-wwm-ext-large" +torch.set_grad_enabled(False) -from time import time as ttime -import shutil +set_start_method("spawn", force=True) -def my_save(fea, path): #####fix issue: torch.save doesn't support chinese path - dir = os.path.dirname(path) - name = os.path.basename(path) - # tmp_path="%s/%s%s.pth"%(dir,ttime(),i_part) - tmp_path = "%s%s.pth" % (ttime(), i_part) - torch.save(fea, tmp_path) - shutil.move(tmp_path, "%s/%s" % (dir, name)) - - -txt_path = "%s/2-name2text-%s.txt" % (opt_dir, i_part) -if os.path.exists(txt_path) == False: - bert_dir = "%s/3-bert" % (opt_dir) - os.makedirs(opt_dir, exist_ok=True) - os.makedirs(bert_dir, exist_ok=True) - if torch.cuda.is_available(): - device = "cuda:0" - # elif torch.backends.mps.is_available(): - # device = "mps" - else: - device = "cpu" - if os.path.exists(bert_pretrained_dir): - ... - else: - raise FileNotFoundError(bert_pretrained_dir) - tokenizer = AutoTokenizer.from_pretrained(bert_pretrained_dir) - bert_model = AutoModelForMaskedLM.from_pretrained(bert_pretrained_dir) - if is_half == True: - bert_model = bert_model.half().to(device) - else: - bert_model = bert_model.to(device) - - def get_bert_feature(text, word2ph): - with torch.no_grad(): - inputs = tokenizer(text, return_tensors="pt") - for i in inputs: - inputs[i] = inputs[i].to(device) - res = bert_model(**inputs, output_hidden_states=True) - res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()[1:-1] - - assert len(word2ph) == len(text) - phone_level_feature = [] - for i in range(len(word2ph)): - repeat_feature = res[i].repeat(word2ph[i], 1) - phone_level_feature.append(repeat_feature) - - phone_level_feature = torch.cat(phone_level_feature, dim=0) - - return phone_level_feature.T - - def process(data, res): - for name, text, lan in data: - try: - name = clean_path(name) - name = os.path.basename(name) - print(name) - phones, word2ph, norm_text = clean_text(text.replace("%", "-").replace("￥", ","), lan, version) - path_bert = "%s/%s.pt" % (bert_dir, name) - if os.path.exists(path_bert) == False and lan == "zh": - bert_feature = get_bert_feature(norm_text, word2ph) - assert bert_feature.shape[-1] == len(phones) - # torch.save(bert_feature, path_bert) - my_save(bert_feature, path_bert) - phones = " ".join(phones) - # res.append([name,phones]) - res.append([name, phones, word2ph, norm_text]) - except: - print(name, text, traceback.format_exc()) - - todo = [] - res = [] - with open(inp_text, "r", encoding="utf8") as f: - lines = f.read().strip("\n").split("\n") - - language_v1_to_language_v2 = { +def lang_map(lang: str) -> str: + m = { "ZH": "zh", "zh": "zh", "JP": "ja", @@ -124,20 +35,179 @@ if os.path.exists(txt_path) == False: "YUE": "yue", "Yue": "yue", } - for line in lines[int(i_part) :: int(all_parts)]: - try: - wav_name, spk_name, language, text = line.split("|") - # todo.append([name,text,"zh"]) - if language in language_v1_to_language_v2.keys(): - todo.append([wav_name, text, language_v1_to_language_v2.get(language, language)]) - else: - print(f"\033[33m[Waring] The {language = } of {wav_name} is not supported for training.\033[0m") - except: - print(line, traceback.format_exc()) + return m.get(lang, "") - process(todo, res) - opt = [] - for name, phones, word2ph, norm_text in res: - opt.append("%s\t%s\t%s\t%s" % (name, phones, word2ph, norm_text)) - with open(txt_path, "w", encoding="utf8") as f: - f.write("\n".join(opt) + "\n") + +def parse_inp_text_line(line: str) -> tuple[str, str, str]: + wav_name, _, language, text = line.split("|", 3) + return wav_name, language, text + + +def build_device_strings(device_type: str, device_ids: list[int], procs_per_device: int) -> list[str]: + devices = [] + for device_id in device_ids: + dstr = f"{device_type}:{device_id}" + devices.extend([dstr] * procs_per_device) + return devices + + +def worker_run( + wid: int, + device_str: str, + tasks_q: Queue[tuple[int, str, str, str]], + results_q: Queue[tuple[int, tuple[str, str, list[int] | None, str]]], + bert_pretrained_dir: str, + opt_dir: str, + fp16: bool, + version: str, +): + device = torch.device(device_str) + + if device.type == "cuda": + assert torch.cuda.is_available() + torch.cuda.set_device(device.index) + elif device.type == "mps": + assert torch.backends.mps.is_available() + + bert_dir = os.path.join(opt_dir, "3-bert") + os.makedirs(bert_dir, exist_ok=True) + + if not os.path.exists(bert_pretrained_dir): + raise FileNotFoundError(bert_pretrained_dir) + + tokenizer = AutoTokenizer.from_pretrained(bert_pretrained_dir) + bert_model = AutoModelForMaskedLM.from_pretrained(bert_pretrained_dir) + + if fp16: + bert_model = bert_model.half().to(device) + else: + bert_model = bert_model.to(device) + + def get_bert_feature(text: str, word2ph: list[int]) -> torch.Tensor: + inputs = tokenizer(text, return_tensors="pt") + for k in inputs: + inputs[k] = inputs[k].to(device) + out = bert_model(**inputs, output_hidden_states=True) + layer = out.hidden_states[-3][0].cpu()[1:-1] # [seq-2, hid] + assert len(word2ph) == len(text) + phone_level_feature = [] + for i in range(len(word2ph)): + phone_level_feature.append(layer[i].repeat(word2ph[i], 1)) + feats = torch.cat(phone_level_feature, dim=0) # [phones, hid] + return feats.T # [hid, phones] + + while True: + item = tasks_q.get() + if item is None: + break + + idx, wav_name, language, text = item + try: + name = clean_path(os.path.basename(wav_name)) + mapped_lang = lang_map(language) + if not mapped_lang: + logger.warning(f"[W{wid}] Unsupported language: {language} of {wav_name}") + results_q.put((idx, ("", "", [], ""))) + continue + + phones, word2ph, norm_text = clean_text( + text.replace("%", "-").replace("￥", ","), + mapped_lang, + version, + ) + + if mapped_lang == "zh": + path_bert = os.path.join(bert_dir, f"{name}.pt") + if not os.path.exists(path_bert): + assert word2ph + bert_feature = get_bert_feature(norm_text, word2ph) + assert bert_feature.shape[-1] == len(phones) + torch.save(bert_feature, path_bert) + + phones_str = " ".join(phones) + results_q.put((idx, (name, phones_str, word2ph, norm_text))) + except Exception: + logger.error(f"[W{wid}] Failed: {wav_name} | {text}\n{tb()}") + results_q.put((idx, ("", "", [], ""))) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--inp", type=str, required=True, help="list File：wav|spk|lang|text") + parser.add_argument("--opt", type=str, required=True) + parser.add_argument("--bert", type=str, required=True) + parser.add_argument("--version", type=str, default=None) + parser.add_argument("--device", type=str, default="cpu", choices=["cpu", "cuda", "mps"]) + parser.add_argument("--device-id", type=str, default="0", help="CUDA_VISIBLE_DEVICE") + parser.add_argument("--nproc", type=int, default=1) + parser.add_argument("--fp16", action="store_true") + args = parser.parse_args() + + device_ids = [int(x) for x in args.devices.split(",") if x.strip() != ""] + if args.device in {"cpu", "mps"} and device_ids != [0]: + raise ValueError(f"Invalid Device ID {device_ids}") + if args.nproc < 1: + raise ValueError(f"Invalid Num Process {args.nproc}") + + os.makedirs(args.opt, exist_ok=True) + merged_path = os.path.join(args.opt, "2-name2text.txt") + + with open(args.inp, "r", encoding="utf8") as f: + lines = [ln for ln in f.read().splitlines() if ln.strip()] + + tasks_all: list[tuple[int, str, str, str]] = [] + for idx, line in enumerate(lines): + try: + wav_name, language, text = parse_inp_text_line(line) + tasks_all.append((idx, wav_name, language, text)) + except Exception: + logger.error(f"Skip line {idx}: {line}\n{traceback.format_exc()}") + + n_tasks = len(tasks_all) + if n_tasks == 0: + logger.warning("Empty list") + with open(merged_path, "w", encoding="utf8") as fout: + pass + return + + device_strs = build_device_strings(args.device, device_ids, args.nproc) + total_workers = len(device_strs) + + tasks_q: Queue[tuple[int, str, str, str] | None] = Queue(maxsize=total_workers * 2) + results_q: Queue = Queue() + + for task in tasks_all: + tasks_q.put(task) + for _ in range(total_workers): + tasks_q.put(None) + + procs: list[Process] = [] + for wid, dstr in enumerate(device_strs): + p = Process( + target=worker_run, + args=(wid, dstr, tasks_q, results_q, args.bert, args.opt, bool(args.fp16), args.version), + daemon=False, + ) + p.start() + procs.append(p) + + ordered: list[tuple[str, str, list[int], str]] = [("", "", [], "")] * n_tasks + for _ in track(range(n_tasks)): + idx, tup = results_q.get() # (idx, (name, phones_str, word2ph, norm_text)) + ordered[idx] = tup + + for p in procs: + p.join() + + with open(merged_path, "w", encoding="utf8") as fout: + for name, phones_str, word2ph, norm_text in ordered: + if name == "": + pass + else: + fout.write(f"{name}\t{phones_str}\t{word2ph}\t{norm_text}\n") + + logger.info(f"Done: {merged_path}") + + +if __name__ == "__main__": + main() diff --git a/GPT_SoVITS/prepare_datasets/2-get-hubert-wav32k copy.py b/GPT_SoVITS/prepare_datasets/2-get-hubert-wav32k copy.py new file mode 100644 index 00000000..3a84c014 --- /dev/null +++ b/GPT_SoVITS/prepare_datasets/2-get-hubert-wav32k copy.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- + +import sys +import os + +inp_text = os.environ.get("inp_text") +inp_wav_dir = os.environ.get("inp_wav_dir") +exp_name = os.environ.get("exp_name") +i_part = os.environ.get("i_part") +all_parts = os.environ.get("all_parts") +if "_CUDA_VISIBLE_DEVICES" in os.environ: + os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["_CUDA_VISIBLE_DEVICES"] +from feature_extractor import cnhubert + +opt_dir = os.environ.get("opt_dir") +cnhubert.cnhubert_base_path = os.environ.get("cnhubert_base_dir") +import torch + +is_half = eval(os.environ.get("is_half", "True")) and torch.cuda.is_available() + +import traceback +import numpy as np +from scipy.io import wavfile +import librosa + +now_dir = os.getcwd() +sys.path.append(now_dir) +from tools.my_utils import load_audio, clean_path + +# from config import cnhubert_base_path +# cnhubert.cnhubert_base_path=cnhubert_base_path +# inp_text=sys.argv[1] +# inp_wav_dir=sys.argv[2] +# exp_name=sys.argv[3] +# i_part=sys.argv[4] +# all_parts=sys.argv[5] +# os.environ["CUDA_VISIBLE_DEVICES"]=sys.argv[6] +# cnhubert.cnhubert_base_path=sys.argv[7] +# opt_dir="/data/docker/liujing04/gpt-vits/fine_tune_dataset/%s"%exp_name + +from time import time as ttime +import shutil + + +def my_save(fea, path): #####fix issue: torch.save doesn't support chinese path + dir = os.path.dirname(path) + name = os.path.basename(path) + # tmp_path="%s/%s%s.pth"%(dir,ttime(),i_part) + tmp_path = "%s%s.pth" % (ttime(), i_part) + torch.save(fea, tmp_path) + shutil.move(tmp_path, "%s/%s" % (dir, name)) + + +hubert_dir = "%s/4-cnhubert" % (opt_dir) +wav32dir = "%s/5-wav32k" % (opt_dir) +os.makedirs(opt_dir, exist_ok=True) +os.makedirs(hubert_dir, exist_ok=True) +os.makedirs(wav32dir, exist_ok=True) + +maxx = 0.95 +alpha = 0.5 +if torch.cuda.is_available(): + device = "cuda:0" +# elif torch.backends.mps.is_available(): +# device = "mps" +else: + device = "cpu" +model = cnhubert.get_model() +# is_half=False +if is_half == True: + model = model.half().to(device) +else: + model = model.to(device) + +nan_fails = [] + + +def name2go(wav_name, wav_path): + hubert_path = "%s/%s.pt" % (hubert_dir, wav_name) + if os.path.exists(hubert_path): + return + tmp_audio = load_audio(wav_path, 32000) + tmp_max = np.abs(tmp_audio).max() + if tmp_max > 2.2: + print("%s-filtered,%s" % (wav_name, tmp_max)) + return + tmp_audio32 = (tmp_audio / tmp_max * (maxx * alpha * 32768)) + ((1 - alpha) * 32768) * tmp_audio + tmp_audio32b = (tmp_audio / tmp_max * (maxx * alpha * 1145.14)) + ((1 - alpha) * 1145.14) * tmp_audio + tmp_audio = librosa.resample(tmp_audio32b, orig_sr=32000, target_sr=16000) # 不是重采样问题 + tensor_wav16 = torch.from_numpy(tmp_audio) + if is_half == True: + tensor_wav16 = tensor_wav16.half().to(device) + else: + tensor_wav16 = tensor_wav16.to(device) + ssl = model.model(tensor_wav16.unsqueeze(0))["last_hidden_state"].transpose(1, 2).cpu() # torch.Size([1, 768, 215]) + if np.isnan(ssl.detach().numpy()).sum() != 0: + nan_fails.append((wav_name, wav_path)) + print("nan filtered:%s" % wav_name) + return + wavfile.write( + "%s/%s" % (wav32dir, wav_name), + 32000, + tmp_audio32.astype("int16"), + ) + my_save(ssl, hubert_path) + + +with open(inp_text, "r", encoding="utf8") as f: + lines = f.read().strip("\n").split("\n") + +for line in lines[int(i_part) :: int(all_parts)]: + try: + # wav_name,text=line.split("\t") + wav_name, spk_name, language, text = line.split("|") + wav_name = clean_path(wav_name) + if inp_wav_dir != "" and inp_wav_dir != None: + wav_name = os.path.basename(wav_name) + wav_path = "%s/%s" % (inp_wav_dir, wav_name) + + else: + wav_path = wav_name + wav_name = os.path.basename(wav_name) + name2go(wav_name, wav_path) + except: + print(line, traceback.format_exc()) + +if len(nan_fails) > 0 and is_half == True: + is_half = False + model = model.float() + for wav in nan_fails: + try: + name2go(wav[0], wav[1]) + except: + print(wav_name, traceback.format_exc()) diff --git a/GPT_SoVITS/s1_train.py b/GPT_SoVITS/s1_train.py index 1176f0bc..8067a03b 100644 --- a/GPT_SoVITS/s1_train.py +++ b/GPT_SoVITS/s1_train.py @@ -1,29 +1,31 @@ # modified from https://github.com/feng-yufei/shared_debugging_code/blob/main/train_t2s.py -import os - -if "_CUDA_VISIBLE_DEVICES" in os.environ: - os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["_CUDA_VISIBLE_DEVICES"] import argparse import logging +import os import platform +from collections import OrderedDict from pathlib import Path import torch -from AR.data.data_module import Text2SemanticDataModule -from AR.models.t2s_lightning_module import Text2SemanticLightningModule -from AR.utils.io import load_yaml_config from pytorch_lightning import Trainer, seed_everything from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning.loggers import TensorBoardLogger # WandbLogger -from pytorch_lightning.strategies import DDPStrategy +from pytorch_lightning.strategies import DDPStrategy, SingleDeviceStrategy + +from GPT_SoVITS.AR.data.data_module import Text2SemanticDataModule +from GPT_SoVITS.AR.models.t2s_lightning_module import Text2SemanticLightningModule +from GPT_SoVITS.AR.utils import get_newest_ckpt +from GPT_SoVITS.AR.utils.io import load_yaml_config +from GPT_SoVITS.process_ckpt import my_save logging.getLogger("numba").setLevel(logging.WARNING) logging.getLogger("matplotlib").setLevel(logging.WARNING) torch.set_float32_matmul_precision("high") -from collections import OrderedDict -from AR.utils import get_newest_ckpt -from process_ckpt import my_save + +os.environ["MASTER_ADDR"] = "localhost" +if platform.system() == "Windows": + os.environ["USE_LIBUV"] = "0" class my_model_ckpt(ModelCheckpoint): @@ -49,35 +51,30 @@ class my_model_ckpt(ModelCheckpoint): monitor_candidates = self._monitor_candidates(trainer) if self._every_n_epochs >= 1 and (trainer.current_epoch + 1) % self._every_n_epochs == 0: if ( - self.if_save_latest == True + self.if_save_latest is True ): ####如果设置只保存最后一个ckpt，在保存下一个ckpt后要清理掉之前的所有ckpt to_clean = list(os.listdir(self.dirpath)) self._save_topk_checkpoint(trainer, monitor_candidates) - if self.if_save_latest == True: + if self.if_save_latest is True: for name in to_clean: try: - os.remove("%s/%s" % (self.dirpath, name)) - except: + os.remove(f"{self.dirpath}/{name}") + except Exception as _: pass - if self.if_save_every_weights == True: + if self.if_save_every_weights is True: to_save_od = OrderedDict() to_save_od["weight"] = OrderedDict() dictt = trainer.strategy._lightning_module.state_dict() for key in dictt: to_save_od["weight"][key] = dictt[key].half() to_save_od["config"] = self.config - to_save_od["info"] = "GPT-e%s" % (trainer.current_epoch + 1) + to_save_od["info"] = f"GPT-e{trainer.current_epoch + 1}" # torch.save( # print(os.environ) if os.environ.get("LOCAL_RANK", "0") == "0": my_save( to_save_od, - "%s/%s-e%s.ckpt" - % ( - self.half_weights_save_dir, - self.exp_name, - trainer.current_epoch + 1, - ), + f"{self.half_weights_save_dir}/{self.exp_name}-e{trainer.current_epoch + 1}.ckpt", ) self._save_last_checkpoint(trainer, monitor_candidates) @@ -91,6 +88,14 @@ def main(args): ckpt_dir = output_dir / "ckpt" ckpt_dir.mkdir(parents=True, exist_ok=True) + if torch.cuda.is_available(): + if torch.cuda.device_count() > 1: + strategy = DDPStrategy(process_group_backend="nccl" if platform.system() != "Windows" else "gloo") + else: + strategy = SingleDeviceStrategy("cuda") + else: + strategy = SingleDeviceStrategy("cpu") + seed_everything(config["train"]["seed"], workers=True) ckpt_callback: ModelCheckpoint = my_model_ckpt( config=config, @@ -106,8 +111,7 @@ def main(args): dirpath=ckpt_dir, ) logger = TensorBoardLogger(name=output_dir.stem, save_dir=output_dir) - os.environ["MASTER_ADDR"] = "localhost" - os.environ["USE_LIBUV"] = "0" + trainer: Trainer = Trainer( max_epochs=config["train"]["epochs"], accelerator="gpu" if torch.cuda.is_available() else "cpu", @@ -117,9 +121,7 @@ def main(args): devices=-1 if torch.cuda.is_available() else 1, benchmark=False, fast_dev_run=False, - strategy=DDPStrategy(process_group_backend="nccl" if platform.system() != "Windows" else "gloo") - if torch.cuda.is_available() - else "auto", + strategy=strategy, precision=config["train"]["precision"], logger=logger, num_sanity_val_steps=0, diff --git a/GPT_SoVITS/s2_train.py b/GPT_SoVITS/s2_train.py index 4b9f6488..04bd1f5c 100644 --- a/GPT_SoVITS/s2_train.py +++ b/GPT_SoVITS/s2_train.py @@ -6,7 +6,6 @@ import os import utils hps = utils.get_hparams(stage=2) -os.environ["CUDA_VISIBLE_DEVICES"] = hps.train.gpu_numbers.replace("-", ",") import logging import torch diff --git a/GPT_SoVITS/s2_train_v3.py b/GPT_SoVITS/s2_train_v3.py index aa8dae7f..0bf6e0d0 100644 --- a/GPT_SoVITS/s2_train_v3.py +++ b/GPT_SoVITS/s2_train_v3.py @@ -6,7 +6,6 @@ import os import utils hps = utils.get_hparams(stage=2) -os.environ["CUDA_VISIBLE_DEVICES"] = hps.train.gpu_numbers.replace("-", ",") import logging import torch diff --git a/GPT_SoVITS/s2_train_v3_lora.py b/GPT_SoVITS/s2_train_v3_lora.py index ba9e4ed4..351c7320 100644 --- a/GPT_SoVITS/s2_train_v3_lora.py +++ b/GPT_SoVITS/s2_train_v3_lora.py @@ -6,7 +6,6 @@ import os import utils hps = utils.get_hparams(stage=2) -os.environ["CUDA_VISIBLE_DEVICES"] = hps.train.gpu_numbers.replace("-", ",") import logging import torch @@ -28,8 +27,8 @@ from module import commons from module.data_utils import ( DistributedBucketSampler, TextAudioSpeakerCollateV3, - TextAudioSpeakerLoaderV3, TextAudioSpeakerCollateV4, + TextAudioSpeakerLoaderV3, TextAudioSpeakerLoaderV4, ) from module.models import ( diff --git a/GPT_SoVITS/text/LangSegmenter/langsegmenter.py b/GPT_SoVITS/text/LangSegmenter/langsegmenter.py index 99b3a422..17e70689 100644 --- a/GPT_SoVITS/text/LangSegmenter/langsegmenter.py +++ b/GPT_SoVITS/text/LangSegmenter/langsegmenter.py @@ -1,40 +1,41 @@ import logging import re +from pathlib import Path -# jieba静音 +import fast_langdetect import jieba +from split_lang import LangSplitter + jieba.setLogLevel(logging.CRITICAL) -# 更改fast_langdetect大模型位置 -from pathlib import Path -import fast_langdetect -fast_langdetect.infer._default_detector = fast_langdetect.infer.LangDetector(fast_langdetect.infer.LangDetectConfig(cache_dir=Path(__file__).parent.parent.parent / "pretrained_models" / "fast_langdetect")) - - -from split_lang import LangSplitter +fast_langdetect.infer._default_detector = fast_langdetect.infer.LangDetector( + fast_langdetect.infer.LangDetectConfig( + cache_dir=str(Path(__file__).parent.parent.parent / "pretrained_models" / "fast_langdetect") + ) +) def full_en(text): - pattern = r'^(?=.*[A-Za-z])[A-Za-z0-9\s\u0020-\u007E\u2000-\u206F\u3000-\u303F\uFF00-\uFFEF]+$' + pattern = r"^(?=.*[A-Za-z])[A-Za-z0-9\s\u0020-\u007E\u2000-\u206F\u3000-\u303F\uFF00-\uFFEF]+$" return bool(re.match(pattern, text)) def full_cjk(text): # 来自wiki cjk_ranges = [ - (0x4E00, 0x9FFF), # CJK Unified Ideographs - (0x3400, 0x4DB5), # CJK Extension A - (0x20000, 0x2A6DD), # CJK Extension B - (0x2A700, 0x2B73F), # CJK Extension C - (0x2B740, 0x2B81F), # CJK Extension D - (0x2B820, 0x2CEAF), # CJK Extension E - (0x2CEB0, 0x2EBEF), # CJK Extension F - (0x30000, 0x3134A), # CJK Extension G - (0x31350, 0x323AF), # CJK Extension H - (0x2EBF0, 0x2EE5D), # CJK Extension H + (0x4E00, 0x9FFF), # CJK Unified Ideographs + (0x3400, 0x4DB5), # CJK Extension A + (0x20000, 0x2A6DD), # CJK Extension B + (0x2A700, 0x2B73F), # CJK Extension C + (0x2B740, 0x2B81F), # CJK Extension D + (0x2B820, 0x2CEAF), # CJK Extension E + (0x2CEB0, 0x2EBEF), # CJK Extension F + (0x30000, 0x3134A), # CJK Extension G + (0x31350, 0x323AF), # CJK Extension H + (0x2EBF0, 0x2EE5D), # CJK Extension H ] - pattern = r'[0-9、-〜。！？.!?… /]+$' + pattern = r"[0-9、-〜。！？.!?… /]+$" cjk_text = "" for char in text: @@ -45,7 +46,7 @@ def full_cjk(text): return cjk_text -def split_jako(tag_lang,item): +def split_jako(tag_lang, item): if tag_lang == "ja": pattern = r"([\u3041-\u3096\u3099\u309A\u30A1-\u30FA\u30FC]+(?:[0-9、-〜。！？.!?… ]+[\u3041-\u3096\u3099\u309A\u30A1-\u30FA\u30FC]*)*)" else: @@ -53,41 +54,42 @@ def split_jako(tag_lang,item): lang_list: list[dict] = [] tag = 0 - for match in re.finditer(pattern, item['text']): + for match in re.finditer(pattern, item["text"]): if match.start() > tag: - lang_list.append({'lang':item['lang'],'text':item['text'][tag:match.start()]}) + lang_list.append({"lang": item["lang"], "text": item["text"][tag : match.start()]}) tag = match.end() - lang_list.append({'lang':tag_lang,'text':item['text'][match.start():match.end()]}) + lang_list.append({"lang": tag_lang, "text": item["text"][match.start() : match.end()]}) - if tag < len(item['text']): - lang_list.append({'lang':item['lang'],'text':item['text'][tag:len(item['text'])]}) + if tag < len(item["text"]): + lang_list.append({"lang": item["lang"], "text": item["text"][tag : len(item["text"])]}) return lang_list def merge_lang(lang_list, item): - if lang_list and item['lang'] == lang_list[-1]['lang']: - lang_list[-1]['text'] += item['text'] + if lang_list and item["lang"] == lang_list[-1]["lang"]: + lang_list[-1]["text"] += item["text"] else: lang_list.append(item) return lang_list -class LangSegmenter(): +class LangSegmenter: # 默认过滤器, 基于gsv目前四种语言 DEFAULT_LANG_MAP = { "zh": "zh", "yue": "zh", # 粤语 "wuu": "zh", # 吴语 "zh-cn": "zh", - "zh-tw": "x", # 繁体设置为x + "zh-tw": "x", # 繁体设置为x "ko": "ko", "ja": "ja", "en": "en", } - def getTexts(text,default_lang = ""): + @staticmethod + def getTexts(text, default_lang=""): lang_splitter = LangSplitter(lang_map=LangSegmenter.DEFAULT_LANG_MAP) lang_splitter.merge_across_digit = False substr = lang_splitter.split_by_lang(text=text) @@ -97,31 +99,31 @@ class LangSegmenter(): have_num = False for _, item in enumerate(substr): - dict_item = {'lang':item.lang,'text':item.text} + dict_item = {"lang": item.lang, "text": item.text} - if dict_item['lang'] == 'digit': + if dict_item["lang"] == "digit": if default_lang != "": - dict_item['lang'] = default_lang + dict_item["lang"] = default_lang else: have_num = True - lang_list = merge_lang(lang_list,dict_item) + lang_list = merge_lang(lang_list, dict_item) continue # 处理短英文被识别为其他语言的问题 - if full_en(dict_item['text']): - dict_item['lang'] = 'en' - lang_list = merge_lang(lang_list,dict_item) + if full_en(dict_item["text"]): + dict_item["lang"] = "en" + lang_list = merge_lang(lang_list, dict_item) continue if default_lang != "": - dict_item['lang'] = default_lang - lang_list = merge_lang(lang_list,dict_item) + dict_item["lang"] = default_lang + lang_list = merge_lang(lang_list, dict_item) continue else: # 处理非日语夹日文的问题(不包含CJK) ja_list: list[dict] = [] - if dict_item['lang'] != 'ja': - ja_list = split_jako('ja',dict_item) + if dict_item["lang"] != "ja": + ja_list = split_jako("ja", dict_item) if not ja_list: ja_list.append(dict_item) @@ -130,8 +132,8 @@ class LangSegmenter(): ko_list: list[dict] = [] temp_list: list[dict] = [] for _, ko_item in enumerate(ja_list): - if ko_item["lang"] != 'ko': - ko_list = split_jako('ko',ko_item) + if ko_item["lang"] != "ko": + ko_list = split_jako("ko", ko_item) if ko_list: temp_list.extend(ko_list) @@ -141,77 +143,76 @@ class LangSegmenter(): # 未存在非日韩文夹日韩文 if len(temp_list) == 1: # 未知语言检查是否为CJK - if dict_item['lang'] == 'x': - cjk_text = full_cjk(dict_item['text']) + if dict_item["lang"] == "x": + cjk_text = full_cjk(dict_item["text"]) if cjk_text: - dict_item = {'lang':'zh','text':cjk_text} - lang_list = merge_lang(lang_list,dict_item) + dict_item = {"lang": "zh", "text": cjk_text} + lang_list = merge_lang(lang_list, dict_item) else: - lang_list = merge_lang(lang_list,dict_item) + lang_list = merge_lang(lang_list, dict_item) continue else: - lang_list = merge_lang(lang_list,dict_item) + lang_list = merge_lang(lang_list, dict_item) continue # 存在非日韩文夹日韩文 for _, temp_item in enumerate(temp_list): # 未知语言检查是否为CJK - if temp_item['lang'] == 'x': - cjk_text = full_cjk(temp_item['text']) + if temp_item["lang"] == "x": + cjk_text = full_cjk(temp_item["text"]) if cjk_text: - lang_list = merge_lang(lang_list,{'lang':'zh','text':cjk_text}) + lang_list = merge_lang(lang_list, {"lang": "zh", "text": cjk_text}) else: - lang_list = merge_lang(lang_list,temp_item) + lang_list = merge_lang(lang_list, temp_item) else: - lang_list = merge_lang(lang_list,temp_item) + lang_list = merge_lang(lang_list, temp_item) # 有数字 if have_num: temp_list = lang_list lang_list = [] for i, temp_item in enumerate(temp_list): - if temp_item['lang'] == 'digit': + if temp_item["lang"] == "digit": if default_lang: - temp_item['lang'] = default_lang + temp_item["lang"] = default_lang elif lang_list and i == len(temp_list) - 1: - temp_item['lang'] = lang_list[-1]['lang'] + temp_item["lang"] = lang_list[-1]["lang"] elif not lang_list and i < len(temp_list) - 1: - temp_item['lang'] = temp_list[1]['lang'] + temp_item["lang"] = temp_list[1]["lang"] elif lang_list and i < len(temp_list) - 1: - if lang_list[-1]['lang'] == temp_list[i + 1]['lang']: - temp_item['lang'] = lang_list[-1]['lang'] - elif lang_list[-1]['text'][-1] in [",",".","!","?","，","。","！","？"]: - temp_item['lang'] = temp_list[i + 1]['lang'] - elif temp_list[i + 1]['text'][0] in [",",".","!","?","，","。","！","？"]: - temp_item['lang'] = lang_list[-1]['lang'] - elif temp_item['text'][-1] in ["。","."]: - temp_item['lang'] = lang_list[-1]['lang'] - elif len(lang_list[-1]['text']) >= len(temp_list[i + 1]['text']): - temp_item['lang'] = lang_list[-1]['lang'] + if lang_list[-1]["lang"] == temp_list[i + 1]["lang"]: + temp_item["lang"] = lang_list[-1]["lang"] + elif lang_list[-1]["text"][-1] in [",", ".", "!", "?", "，", "。", "！", "？"]: + temp_item["lang"] = temp_list[i + 1]["lang"] + elif temp_list[i + 1]["text"][0] in [",", ".", "!", "?", "，", "。", "！", "？"]: + temp_item["lang"] = lang_list[-1]["lang"] + elif temp_item["text"][-1] in ["。", "."]: + temp_item["lang"] = lang_list[-1]["lang"] + elif len(lang_list[-1]["text"]) >= len(temp_list[i + 1]["text"]): + temp_item["lang"] = lang_list[-1]["lang"] else: - temp_item['lang'] = temp_list[i + 1]['lang'] + temp_item["lang"] = temp_list[i + 1]["lang"] else: - temp_item['lang'] = 'zh' - - lang_list = merge_lang(lang_list,temp_item) + temp_item["lang"] = "zh" + lang_list = merge_lang(lang_list, temp_item) # 筛X temp_list = lang_list lang_list = [] for _, temp_item in enumerate(temp_list): - if temp_item['lang'] == 'x': + if temp_item["lang"] == "x": if lang_list: - temp_item['lang'] = lang_list[-1]['lang'] + temp_item["lang"] = lang_list[-1]["lang"] elif len(temp_list) > 1: - temp_item['lang'] = temp_list[1]['lang'] + temp_item["lang"] = temp_list[1]["lang"] else: - temp_item['lang'] = 'zh' + temp_item["lang"] = "zh" - lang_list = merge_lang(lang_list,temp_item) + lang_list = merge_lang(lang_list, temp_item) return lang_list - + if __name__ == "__main__": text = "MyGO?,你也喜欢まいご吗？" @@ -221,5 +222,5 @@ if __name__ == "__main__": print(LangSegmenter.getTexts(text)) text = "当时ThinkPad T60刚刚发布，一同推出的还有一款名为Advanced Dock的扩展坞配件。这款扩展坞通过连接T60底部的插槽，扩展出包括PCIe在内的一大堆接口，并且自带电源，让T60可以安装桌面显卡来提升性能。" - print(LangSegmenter.getTexts(text,"zh")) - print(LangSegmenter.getTexts(text)) \ No newline at end of file + print(LangSegmenter.getTexts(text, "zh")) + print(LangSegmenter.getTexts(text)) diff --git a/GPT_SoVITS/text/cleaner.py b/GPT_SoVITS/text/cleaner.py index 7ba8f376..a13c39e9 100644 --- a/GPT_SoVITS/text/cleaner.py +++ b/GPT_SoVITS/text/cleaner.py @@ -1,12 +1,13 @@ -from text import cleaned_text_to_sequence import os + +from text import cleaned_text_to_sequence + # if os.environ.get("version","v1")=="v1": # from text import chinese # from text.symbols import symbols # else: # from text import chinese2 as chinese # from text.symbols2 import symbols - from text import symbols as symbols_v1 from text import symbols2 as symbols_v2 @@ -18,7 +19,7 @@ special = [ ] -def clean_text(text, language, version=None): +def clean_text(text, language, version=None) -> tuple[list[str], list[int] | None, str]: if version is None: version = os.environ.get("version", "v2") if version == "v1": diff --git a/README.md b/README.md index 86d50ea2..8dd4ea7f 100644 --- a/README.md +++ b/README.md @@ -248,13 +248,13 @@ if you want to switch to V1,then double-click`go-webui-v1.bat` or use `go-webui- #### Others ```bash -python webui.py +PYTHONPATH=. python webui.py ``` if you want to switch to V1,then ```bash -python webui.py v1 +PYTHONPATH=. python webui.py v1 ``` Or maunally switch version in WebUI @@ -285,7 +285,7 @@ python GPT_SoVITS/inference_webui.py OR ```bash -python webui.py +PYTHONPATH=. python webui.py ``` then open the inference webui at `1-GPT-SoVITS-TTS/1C-inference` diff --git a/config.py b/config.py index fdc11c0a..2185fd4a 100644 --- a/config.py +++ b/config.py @@ -145,9 +145,20 @@ webui_port_subfix = 9871 api_port = 9880 +def get_dtype(idx: int): + if not torch.cuda.is_available(): + return torch.float32 + capability = torch.cuda.get_device_capability(idx) + major, minor = capability + sm_version = major + minor / 10.0 + if sm_version > 6.1: + return torch.float16 + return torch.float32 + + # Thanks to the contribution of @Karasukaigan and @XXXXRT666 def get_device_dtype_sm(idx: int) -> tuple[torch.device, torch.dtype, float, float]: - cpu = torch.device("cpu") + cpu = torch.device("cpu:0") cuda = torch.device(f"cuda:{idx}") if not torch.cuda.is_available(): return cpu, torch.float32, 0.0, 0.0 @@ -161,7 +172,7 @@ def get_device_dtype_sm(idx: int) -> tuple[torch.device, torch.dtype, float, flo is_16_series = bool(re.search(r"16\d{2}", name)) and sm_version == 7.5 if mem_gb < 4 or sm_version < 5.3: return cpu, torch.float32, 0.0, 0.0 - if sm_version == 6.1 or is_16_series == True: + if sm_version == 6.1 or is_16_series is True: return cuda, torch.float32, sm_version, mem_gb if sm_version > 6.1: return cuda, torch.float16, sm_version, mem_gb @@ -190,8 +201,11 @@ if not GPU_INFOS: IS_GPU = False GPU_INFOS.append(CPU_INFO) GPU_INDEX.add(0) +if torch.mps.is_available(): + infer_device = torch.device("mps:0") +else: + infer_device = max(tmp, key=lambda x: (x[2], x[3]))[0] -infer_device = max(tmp, key=lambda x: (x[2], x[3]))[0] is_half = any(dtype == torch.float16 for _, dtype, _, _ in tmp) @@ -216,3 +230,22 @@ class Config: self.webui_port_subfix = webui_port_subfix self.api_port = api_port + + +def get_implement(device: torch.device): + if torch.cuda.is_available(): + idx = device.index + capability = torch.cuda.get_device_capability(idx) + major, minor = capability + sm_version = major + minor / 10.0 + if sm_version >= 7.5: + return "flash_attn" + else: + if sys.platform == "linux": + return "sage_attn" + else: + return "naive" + elif torch.mps.is_available(): + return "mlx" + else: + return "naive" diff --git a/docs/cn/README.md b/docs/cn/README.md index 793734d8..a297cdf8 100644 --- a/docs/cn/README.md +++ b/docs/cn/README.md @@ -236,13 +236,13 @@ D:\GPT-SoVITS\xxx/xxx.wav|xxx|zh|我爱玩原神. #### 其他 ```bash -python webui.py +PYTHONPATH=. python webui.py ``` 若想使用 V1,则 ```bash -python webui.py v1 +PYTHONPATH=. python webui.py v1 ``` 或者在 webUI 内动态切换 @@ -273,7 +273,7 @@ python GPT_SoVITS/inference_webui.py 或者 ```bash -python webui.py +PYTHONPATH=. python webui.py ``` 然后在 `1-GPT-SoVITS-TTS/1C-推理` 中打开推理 webUI diff --git a/docs/ja/README.md b/docs/ja/README.md index 1ee9abb4..b4761263 100644 --- a/docs/ja/README.md +++ b/docs/ja/README.md @@ -222,13 +222,13 @@ V1 に切り替えたい場合は、`go-webui-v1.bat`をダブルクリックす #### その他 ```bash -python webui.py <言語(オプション)> +PYTHONPATH=. python webui.py <言語(オプション)> ``` V1 に切り替えたい場合は ```bash -python webui.py v1 <言語(オプション)> +PYTHONPATH=. python webui.py v1 <言語(オプション)> ``` または WebUI で手動でバージョンを切り替えてください. @@ -259,7 +259,7 @@ python GPT_SoVITS/inference_webui.py <言語(オプション)> または ```bash -python webui.py +PYTHONPATH=. python webui.py ``` その後、`1-GPT-SoVITS-TTS/1C-inference`で推論 webui を開きます. diff --git a/docs/ko/README.md b/docs/ko/README.md index 9ff32f83..e69927ed 100644 --- a/docs/ko/README.md +++ b/docs/ko/README.md @@ -228,13 +228,13 @@ V1으로 전환하려면, `go-webui-v1.bat`을 더블 클릭하거나 `go-webui- #### 기타 ```bash -python webui.py <언어(옵션)> +PYTHONPATH=. python webui.py <언어(옵션)> ``` V1으로 전환하려면, ```bash -python webui.py v1 <언어(옵션)> +PYTHONPATH=. python webui.py v1 <언어(옵션)> ``` 또는 WebUI에서 수동으로 버전을 전환하십시오. @@ -265,7 +265,7 @@ python GPT_SoVITS/inference_webui.py <언어(옵션)> 또는 ```bash -python webui.py +PYTHONPATH=. python webui.py ``` 그런 다음 `1-GPT-SoVITS-TTS/1C-inference`에서 추론 webui를 엽니다. diff --git a/docs/tr/README.md b/docs/tr/README.md index b80c764b..e93b3746 100644 --- a/docs/tr/README.md +++ b/docs/tr/README.md @@ -229,13 +229,13 @@ V1'e geçmek istiyorsanız, `go-webui-v1.bat` dosyasına çift tıklayın veya ` #### Diğerleri ```bash -python webui.py +PYTHONPATH=. python webui.py ``` V1'e geçmek istiyorsanız, ```bash -python webui.py v1 +PYTHONPATH=. python webui.py v1 ``` veya WebUI'de manuel olarak sürüm değiştirin. @@ -266,7 +266,7 @@ python GPT_SoVITS/inference_webui.py VEYA ```bash -python webui.py +PYTHONPATH=. python webui.py ``` ardından çıkarım webui'sini `1-GPT-SoVITS-TTS/1C-inference` adresinde açın. diff --git a/go-webui.bat b/go-webui.bat index c1c81089..6918cc5f 100644 --- a/go-webui.bat +++ b/go-webui.bat @@ -1,6 +1,7 @@ set "SCRIPT_DIR=%~dp0" set "SCRIPT_DIR=%SCRIPT_DIR:~0,-1%" cd /d "%SCRIPT_DIR%" -set "PATH=%SCRIPT_DIR%\runtime;%PATH%" +set "PATH=%SCRIPT_DIR%\runtime" +set "PYTHONPATH=%SCRIPT_DIR%" runtime\python.exe -I webui.py zh_CN pause diff --git a/go-webui.ps1 b/go-webui.ps1 index 09103425..42d336e3 100644 --- a/go-webui.ps1 +++ b/go-webui.ps1 @@ -2,6 +2,7 @@ $ErrorActionPreference = "SilentlyContinue" chcp 65001 Set-Location $PSScriptRoot $runtimePath = Join-Path $PSScriptRoot "runtime" -$env:PATH = "$runtimePath;$env:PATH" +$env:PATH = "$runtimePath" +$env:PYTHONPATH = "$runtimePath" & "$runtimePath\python.exe" -I "$PSScriptRoot\webui.py" zh_CN pause diff --git a/gpt-sovits_kaggle.ipynb b/gpt-sovits_kaggle.ipynb deleted file mode 100644 index 764c23c4..00000000 --- a/gpt-sovits_kaggle.ipynb +++ /dev/null @@ -1,243 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "9fd922fb", - "metadata": {}, - "source": [ - "# Deprecated" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "45857cb2", - "metadata": { - "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", - "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5", - "execution": { - "iopub.execute_input": "2024-02-18T14:43:46.735480Z", - "iopub.status.busy": "2024-02-18T14:43:46.735183Z", - "iopub.status.idle": "2024-02-18T14:48:10.724175Z", - "shell.execute_reply": "2024-02-18T14:48:10.723059Z" - }, - "papermill": { - "duration": 263.994935, - "end_time": "2024-02-18T14:48:10.726613", - "exception": false, - "start_time": "2024-02-18T14:43:46.731678", - "status": "completed" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "!git clone https://github.com/RVC-Boss/GPT-SoVITS.git\n", - "%cd GPT-SoVITS\n", - "!apt-get update && apt-get install -y --no-install-recommends tzdata ffmpeg libsox-dev parallel aria2 git git-lfs && git lfs install\n", - "!pip install -r requirements.txt\n", - "!pip install -r extra-req.txt --no-deps" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b9d346b4", - "metadata": { - "execution": { - "iopub.execute_input": "2024-02-18T14:48:10.815802Z", - "iopub.status.busy": "2024-02-18T14:48:10.814899Z", - "iopub.status.idle": "2024-02-18T14:50:31.253276Z", - "shell.execute_reply": "2024-02-18T14:50:31.252024Z" - }, - "papermill": { - "duration": 140.484893, - "end_time": "2024-02-18T14:50:31.255720", - "exception": false, - "start_time": "2024-02-18T14:48:10.770827", - "status": "completed" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# @title Download pretrained models 下载预训练模型\n", - "!mkdir -p /kaggle/working/GPT-SoVITS/GPT_SoVITS/pretrained_models\n", - "!mkdir -p /kaggle/working/GPT-SoVITS/tools/asr/models\n", - "!mkdir -p /kaggle/working/GPT-SoVITS/tools/uvr5\n", - "%cd /kaggle/working/GPT-SoVITS/GPT_SoVITS/pretrained_models\n", - "!git clone https://huggingface.co/lj1995/GPT-SoVITS\n", - "%cd /kaggle/working/GPT-SoVITS/tools/asr/models\n", - "!git clone https://www.modelscope.cn/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch.git\n", - "!git clone https://www.modelscope.cn/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch.git\n", - "!git clone https://www.modelscope.cn/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch.git\n", - "# # @title UVR5 pretrains 安装uvr5模型\n", - "%cd /kaggle/working/GPT-SoVITS/tools/uvr5\n", - "!git clone https://huggingface.co/Delik/uvr5_weights\n", - "!git config core.sparseCheckout true\n", - "!mv /kaggle/working/GPT-SoVITS/GPT_SoVITS/pretrained_models/GPT-SoVITS/* /kaggle/working/GPT-SoVITS/GPT_SoVITS/pretrained_models/" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ea94d245", - "metadata": { - "execution": { - "iopub.execute_input": "2024-02-18T14:29:01.071549Z", - "iopub.status.busy": "2024-02-18T14:29:01.070592Z", - "iopub.status.idle": "2024-02-18T14:40:45.318368Z", - "shell.execute_reply": "2024-02-18T14:40:45.317130Z", - "shell.execute_reply.started": "2024-02-18T14:29:01.071512Z" - }, - "papermill": { - "duration": null, - "end_time": null, - "exception": false, - "start_time": "2024-02-18T14:50:31.309013", - "status": "running" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# @title launch WebUI 启动WebUI\n", - "%cd /kaggle/working/GPT-SoVITS/\n", - "!npm install -g localtunnel\n", - "import subprocess\n", - "import threading\n", - "import time\n", - "import socket\n", - "import urllib.request\n", - "\n", - "\n", - "def iframe_thread(port):\n", - " while True:\n", - " time.sleep(0.5)\n", - " sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n", - " result = sock.connect_ex((\"127.0.0.1\", port))\n", - " if result == 0:\n", - " break\n", - " sock.close()\n", - "\n", - " from colorama import Fore, Style\n", - " print(\n", - " Fore.GREEN + \"\\nIP: \",\n", - " Fore.RED,\n", - " urllib.request.urlopen(\"https://ipv4.icanhazip.com\").read().decode(\"utf8\").strip(\"\\n\"),\n", - " \"\\n\",\n", - " Style.RESET_ALL,\n", - " )\n", - " p = subprocess.Popen([\"lt\", \"--port\", \"{}\".format(port)], stdout=subprocess.PIPE)\n", - " for line in p.stdout:\n", - " print(line.decode(), end=\"\")\n", - "\n", - "\n", - "threading.Thread(target=iframe_thread, daemon=True, args=(9874,)).start()\n", - "\n", - "!python webui.py" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dda88a6d", - "metadata": { - "execution": { - "iopub.execute_input": "2024-02-18T14:40:56.880608Z", - "iopub.status.busy": "2024-02-18T14:40:56.879879Z" - }, - "papermill": { - "duration": null, - "end_time": null, - "exception": null, - "start_time": null, - "status": "pending" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# 开启推理页面\n", - "%cd /kaggle/working/GPT-SoVITS/\n", - "!npm install -g localtunnel\n", - "import threading\n", - "\n", - "\n", - "def iframe_thread(port):\n", - " while True:\n", - " time.sleep(0.5)\n", - " sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n", - " result = sock.connect_ex((\"127.0.0.1\", port))\n", - " if result == 0:\n", - " break\n", - " sock.close()\n", - "\n", - " from colorama import Fore, Style\n", - " print(\n", - " Fore.GREEN + \"\\nIP: \",\n", - " Fore.RED,\n", - " urllib.request.urlopen(\"https://ipv4.icanhazip.com\").read().decode(\"utf8\").strip(\"\\n\"),\n", - " \"\\n\",\n", - " Style.RESET_ALL,\n", - " )\n", - " p = subprocess.Popen([\"lt\", \"--port\", \"{}\".format(port)], stdout=subprocess.PIPE)\n", - " for line in p.stdout:\n", - " print(line.decode(), end=\"\")\n", - "\n", - "\n", - "threading.Thread(target=iframe_thread, daemon=True, args=(9872,)).start()\n", - "\n", - "!python ./GPT_SoVITS/inference_webui.py" - ] - } - ], - "metadata": { - "kaggle": { - "accelerator": "nvidiaTeslaT4", - "dataSources": [ - { - "datasetId": 4459328, - "sourceId": 7649639, - "sourceType": "datasetVersion" - } - ], - "dockerImageVersionId": 30646, - "isGpuEnabled": true, - "isInternetEnabled": true, - "language": "python", - "sourceType": "notebook" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - }, - "papermill": { - "default_parameters": {}, - "duration": null, - "end_time": null, - "environment_variables": {}, - "exception": null, - "input_path": "__notebook__.ipynb", - "output_path": "__notebook__.ipynb", - "parameters": {}, - "start_time": "2024-02-18T14:43:44.011910", - "version": "2.5.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/install.ps1 b/install.ps1 index 9c33ace8..ab761ba8 100644 --- a/install.ps1 +++ b/install.ps1 @@ -40,6 +40,10 @@ function Write-Info($msg) { Write-Host "[INFO]:" -ForegroundColor Green -NoNewline Write-Host " $msg" } +function Write-Warning($msg) { + Write-Host "[Warning]:" -ForegroundColor Yellow -NoNewline + Write-Host " $msg" +} function Write-Success($msg) { Write-Host "[SUCCESS]:" -ForegroundColor Blue -NoNewline Write-Host " $msg" @@ -137,7 +141,7 @@ chcp 65001 Set-Location $PSScriptRoot Write-Info "Installing FFmpeg & CMake..." -Invoke-Conda ffmpeg cmake +Invoke-Conda ffmpeg cmake vc14_runtime Write-Success "FFmpeg & CMake Installed" $PretrainedURL = "" @@ -208,12 +212,30 @@ if ($DownloadUVR5) { switch ($Device) { "CU128" { + $cudaLine = nvidia-smi | Select-String "CUDA Version" + $version = ($cudaLine -split "CUDA Version:")[1].Trim() + Write-Info "Maximum CUDA Version Supported By Current Driver: $version" + if ([version](nvidia-smi | Select-String "CUDA Version" | ForEach-Object { ($_ -split "CUDA Version:")[1].Trim() }) -ge [version]"12.8") { + Write-Warning "CUDA 12.8 Is Not Supported By Current Driver" + } Write-Info "Installing PyTorch For CUDA 12.8..." Invoke-Pip torch torchaudio --index-url "https://download.pytorch.org/whl/cu128" + Invoke-Conda cuda-nvcc=12.8 + Invoke-Pip psutil ninja packaging wheel "setuptools>=42" + Invoke-Pip flash-attn -i https://xxxxrt666.github.io/PIP-Index/ --no-build-isolation } "CU126" { + $cudaLine = nvidia-smi | Select-String "CUDA Version" + $version = ($cudaLine -split "CUDA Version:")[1].Trim() + Write-Info "Maximum CUDA Version Supported By Current Driver: $version" + if ([version](nvidia-smi | Select-String "CUDA Version" | ForEach-Object { ($_ -split "CUDA Version:")[1].Trim() }) -ge [version]"12.8") { + Write-Warning "CUDA 12.6 Is Not Supported By Current Driver" + } Write-Info "Installing PyTorch For CUDA 12.6..." Invoke-Pip torch torchaudio --index-url "https://download.pytorch.org/whl/cu126" + Invoke-Conda cuda-nvcc=12.6 + Invoke-Pip psutil ninja packaging wheel "setuptools>=42" + Invoke-Pip flash-attn -i https://xxxxrt666.github.io/PIP-Index/ --no-build-isolation } "CPU" { Write-Info "Installing PyTorch For CPU..." diff --git a/install.sh b/install.sh index 7d80ec28..c0cf3543 100644 --- a/install.sh +++ b/install.sh @@ -127,7 +127,7 @@ while [[ $# -gt 0 ]]; do USE_ROCM=true ;; MPS) - USE_CPU=true + USE_MPS=true ;; CPU) USE_CPU=true @@ -157,7 +157,7 @@ while [[ $# -gt 0 ]]; do esac done -if ! $USE_CUDA && ! $USE_ROCM && ! $USE_CPU; then +if ! $USE_CUDA && ! $USE_ROCM && ! $USE_MPS && ! $USE_CPU; then echo -e "${ERROR}Error: Device is REQUIRED" echo "" print_help @@ -322,13 +322,29 @@ if [ "$USE_ROCM" = true ] && [ "$WORKFLOW" = false ]; then fi if [ "$USE_CUDA" = true ] && [ "$WORKFLOW" = false ]; then + CUDAVERSION=$(nvidia-smi | grep "CUDA Version" | sed -E 's/.*CUDA Version: ([0-9]+\.[0-9]+).*/\1/') + echo -e "${INFO}Maximum CUDA Version Supported By Current Driver: $CUDAVERSION" if [ "$CUDA" = 128 ]; then + if awk "BEGIN {exit !($CUDAVERSION < 12.8)}"; then + echo -r "${WARNING}CUDA 12.8 Is Not Supported By Current Driver" + fi echo -e "${INFO}Installing PyTorch For CUDA 12.8..." run_pip_quiet torch torchaudio --index-url "https://download.pytorch.org/whl/cu128" + run_conda_quiet cuda-nvcc=12.8 elif [ "$CUDA" = 126 ]; then + if awk "BEGIN {exit !($CUDAVERSION < 12.6)}"; then + echo -r "${WARNING}CUDA 12.6 Is Not Supported By Current Driver" + fi echo -e "${INFO}Installing PyTorch For CUDA 12.6..." run_pip_quiet torch torchaudio --index-url "https://download.pytorch.org/whl/cu126" + run_conda_quiet cuda-nvcc=12.6 fi + run_pip_quiet psutil ninja packaging wheel "setuptools>=42" + run_pip_quiet flash-attn -i https://xxxxrt666.github.io/PIP-Index/ --no-build-isolation +elif [ "$USE_MPS" = true ] && [ "$WORKFLOW" = false ]; then + echo -e "${INFO}Installing PyTorch For MPS..." + run_pip_quiet torch torchaudio --index-url "https://download.pytorch.org/whl/cpu" + run_pip_quiet mlx elif [ "$USE_ROCM" = true ] && [ "$WORKFLOW" = false ]; then echo -e "${INFO}Installing PyTorch For ROCm 6.2..." run_pip_quiet torch torchaudio --index-url "https://download.pytorch.org/whl/rocm6.2" diff --git a/requirements.txt b/requirements.txt index 90e4957d..eb799f35 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,45 +1,36 @@ --no-binary=opencc -numpy<2.0 -scipy -tensorboard -librosa==0.10.2 -numba -pytorch-lightning>=2.4 -gradio<5 + +cn2an ffmpeg-python +g2pk2 +g2p_en +jieba_fast +kernels +ko_pron +modelscope +opencc +peft +pypinyin +split-lang +transformers +tensorboard +ToJyutping +wordsegment +x_transformers + onnxruntime; platform_machine == "aarch64" or platform_machine == "arm64" onnxruntime-gpu; platform_machine == "x86_64" or platform_machine == "AMD64" -tqdm -funasr==1.0.27 -cn2an -pypinyin -pyopenjtalk>=0.4.1 -g2p_en -torchaudio -modelscope==1.10.0 -sentencepiece -transformers>=4.43,<=4.50 -peft -chardet -PyYAML -psutil -jieba_fast -jieba -split-lang -fast_langdetect>=0.3.1 -wordsegment -rotary_embedding_torch -ToJyutping -g2pk2 -ko_pron -opencc python_mecab_ko; sys_platform != 'win32' -fastapi[standard]>=0.115.2 -x_transformers -torchmetrics<=1.5 -pydantic<=2.10.6 -ctranslate2>=4.0,<5 -huggingface_hub>=0.13 -tokenizers>=0.13,<1 + av>=11 -tqdm +ctranslate2>=4.0,<5 +fastapi[standard]>=0.115.2 +fast_langdetect>=0.3.1 +funasr==1.0.27 +gradio==5.25.0 +librosa==0.10.2 +numpy<2.0 +pydantic<=2.10.6 +pyopenjtalk>=0.4.1 +pytorch-lightning>=2.4 +torchmetrics<=1.5 diff --git a/tools/i18n/locale/en_US.json b/tools/i18n/locale/en_US.json index 24d24de4..380c756d 100644 --- a/tools/i18n/locale/en_US.json +++ b/tools/i18n/locale/en_US.json @@ -1,11 +1,11 @@ { "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；": "(1)MDX-Net(onnx_dereverb): Best choice for dual-channel reverberation, cannot remove single-channel reverberation;", "(234)DeEcho:去除延迟效果。Aggressive比Normal去除得更彻底，DeReverb额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。": "(234)DeEcho: Removes delay effects. Aggressive mode removes more thoroughly than Normal mode. DeReverb additionally removes reverberation, can remove mono reverberation, but does not clean heavily high-frequency plate reverberation.", - "*实验/模型名": "*Experiment/model name", - "*文本标注文件": "*Text labelling file", - "*训练集音频文件目录": "*Audio dataset folder", - "*请上传并填写参考信息": "*Please upload and fill reference information", - "*请填写需要合成的目标文本和语种模式": "*Please fill in the target text and language mode for synthesis", + "实验/模型名": "Experiment/model name", + "文本标注文件": "Text labelling file", + "训练集音频文件目录": "Audio dataset folder", + "请上传并填写参考信息": "Please upload and fill reference information", + "请填写需要合成的目标文本和语种模式": "Please fill in the target text and language mode for synthesis", ".限制范围越小判别效果越好。": "Less Multilingual is better", "1-GPT-SoVITS-TTS": "1-GPT-SOVITS-TTS", "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；": "1. The DeEcho-DeReverb model's processing time is nearly twice that of the other two DeEcho models.", @@ -222,5 +222,6 @@ "预训练SoVITS-D模型路径": "Pretrained SoVITS-D Model Path", "预训练SoVITS-G模型路径": "Pretrained SoVITS-G Model Path", "预训练中文BERT模型路径": "Pretrained Chinese BERT Model Path", - "预训练模型路径": "Pretrained Model Path" + "预训练模型路径": "Pretrained Model Path", + "推理后端": "Inference Backend" } diff --git a/tools/i18n/locale/es_ES.json b/tools/i18n/locale/es_ES.json index 18c8af99..33b0ff87 100644 --- a/tools/i18n/locale/es_ES.json +++ b/tools/i18n/locale/es_ES.json @@ -1,11 +1,11 @@ { "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；": "(1)MDX-Net (onnx_dereverb): reverberación estéreo, la mejor opción; no puede eliminar reverberación mono", "(234)DeEcho:去除延迟效果。Aggressive比Normal去除得更彻底，DeReverb额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。": "(234)DeEcho: Eliminar el efecto de retardo. Aggressive elimina más que Normal, DeReverb elimina reverberación adicional, puede eliminar reverberación mono, pero no limpia bien la reverberación de placa de alta frecuencia", - "*实验/模型名": "*Nombre del experimento/modelo", - "*文本标注文件": "*Archivo de etiquetado de texto", - "*训练集音频文件目录": "*Directorio de archivos de audio de entrenamiento", - "*请上传并填写参考信息": "*Por favor, suba y complete la información de referencia", - "*请填写需要合成的目标文本和语种模式": "*Por favor, complete el texto objetivo a sintetizar y el modo de idioma", + "实验/模型名": "Nombre del experimento/modelo", + "文本标注文件": "Archivo de etiquetado de texto", + "训练集音频文件目录": "Directorio de archivos de audio de entrenamiento", + "请上传并填写参考信息": "Por favor, suba y complete la información de referencia", + "请填写需要合成的目标文本和语种模式": "Por favor, complete el texto objetivo a sintetizar y el modo de idioma", ".限制范围越小判别效果越好。": ".Cuanto más pequeño sea el rango, mejor será el efecto de discriminación.", "1-GPT-SoVITS-TTS": "1-GPT-SoVITS-TTS", "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；": "1. El modelo DeEcho-DeReverb tarda casi el doble que los otros dos modelos DeEcho", diff --git a/tools/i18n/locale/fr_FR.json b/tools/i18n/locale/fr_FR.json index 1d55a56b..18303fa7 100644 --- a/tools/i18n/locale/fr_FR.json +++ b/tools/i18n/locale/fr_FR.json @@ -1,11 +1,11 @@ { "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；": "(1) MDX-Net (onnx_dereverb) : C'est le meilleur choix pour la réverbération à deux canaux, mais il ne peut pas éliminer la réverbération à un seul canal;", "(234)DeEcho:去除延迟效果。Aggressive比Normal去除得更彻底，DeReverb额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。": "(234)DeEcho : Supprime les effets de délai. Aggressive est plus exhaustif que Normal dans la suppression, DeReverb élimine également la réverbération, peut supprimer la réverbération monocanal, mais n'élimine pas complètement la réverbération de plaque à haute fréquence.", - "*实验/模型名": "*Nom de l'expérience/modèle", - "*文本标注文件": "*Fichier d'annotation de texte", - "*训练集音频文件目录": "*Répertoire des fichiers audio d'entraînement", - "*请上传并填写参考信息": "*Veuillez télécharger et remplir les informations de référence", - "*请填写需要合成的目标文本和语种模式": "*Veuillez saisir le texte cible à synthétiser et le mode de langue.", + "实验/模型名": "Nom de l'expérience/modèle", + "文本标注文件": "Fichier d'annotation de texte", + "训练集音频文件目录": "Répertoire des fichiers audio d'entraînement", + "请上传并填写参考信息": "Veuillez télécharger et remplir les informations de référence", + "请填写需要合成的目标文本和语种模式": "Veuillez saisir le texte cible à synthétiser et le mode de langue.", ".限制范围越小判别效果越好。": "Moins il y a de langues, mieux c'est", "1-GPT-SoVITS-TTS": "1-GPT-SoVITS-TTS", "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；": "1. Le temps de traitement du modèle DeEcho-DeReverb est presque le double de celui des deux autres modèles DeEcho;", diff --git a/tools/i18n/locale/it_IT.json b/tools/i18n/locale/it_IT.json index 1c236475..bd136ea9 100644 --- a/tools/i18n/locale/it_IT.json +++ b/tools/i18n/locale/it_IT.json @@ -1,11 +1,11 @@ { "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；": "(1)MDX-Net (onnx_dereverb): È la scelta migliore per la riverberazione a due canali, ma non può rimuovere la riverberazione a canale singolo;", "(234)DeEcho:去除延迟效果。Aggressive比Normal去除得更彻底，DeReverb额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。": "(234)DeEcho: Rimuove gli effetti di ritardo. Aggressive è più completo di Normal nella rimozione, DeReverb rimuove ulteriormente la riverberazione, può rimuovere la riverberazione a canale singolo, ma non rimuove completamente la riverberazione a piastra ad alta frequenza.", - "*实验/模型名": "*Nome dell'esperimento/modello", - "*文本标注文件": "*File di annotazione del testo", - "*训练集音频文件目录": "*Directory dei file audio del set di addestramento", - "*请上传并填写参考信息": "*Carica e compila le informazioni di riferimento", - "*请填写需要合成的目标文本和语种模式": "*Si prega di inserire il testo di destinazione da sintetizzare e la modalità lingua", + "实验/模型名": "Nome dell'esperimento/modello", + "文本标注文件": "File di annotazione del testo", + "训练集音频文件目录": "Directory dei file audio del set di addestramento", + "请上传并填写参考信息": "Carica e compila le informazioni di riferimento", + "请填写需要合成的目标文本和语种模式": "Si prega di inserire il testo di destinazione da sintetizzare e la modalità lingua", ".限制范围越小判别效果越好。": "Meno multilingue è meglio", "1-GPT-SoVITS-TTS": "1-GPT-SoVITS-TTS", "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；": "1. Il tempo di elaborazione del modello DeEcho-DeReverb è quasi il doppio di quello degli altri due modelli DeEcho;", diff --git a/tools/i18n/locale/ja_JP.json b/tools/i18n/locale/ja_JP.json index 4abe5606..639de09d 100644 --- a/tools/i18n/locale/ja_JP.json +++ b/tools/i18n/locale/ja_JP.json @@ -1,11 +1,11 @@ { "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；": "(1)MDX-Net(onnx_dereverb):二重チャンネルのリバーブに最適な選択ですが、単一チャンネルのリバーブは除去できません；", "(234)DeEcho:去除延迟效果。Aggressive比Normal去除得更彻底，DeReverb额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。": "(234)DeEcho:遅延効果を除去します。AggressiveはNormalよりも徹底的に除去し、DeReverbは追加でリバーブを除去し、モノラルリバーブを除去できますが、高周波数のプレートリバーブは完全には除去できません。", - "*实验/模型名": "*実験/モデル名", - "*文本标注文件": "*テキスト注釈ファイル", - "*训练集音频文件目录": "*トレーニングデータのオーディオファイルディレクトリ", - "*请上传并填写参考信息": "*参照情報をアップロードして記入してください", - "*请填写需要合成的目标文本和语种模式": "*合成対象テキストと言語モードを入力してください", + "实验/模型名": "実験/モデル名", + "文本标注文件": "テキスト注釈ファイル", + "训练集音频文件目录": "トレーニングデータのオーディオファイルディレクトリ", + "请上传并填写参考信息": "参照情報をアップロードして記入してください", + "请填写需要合成的目标文本和语种模式": "合成対象テキストと言語モードを入力してください", ".限制范围越小判别效果越好。": "多言語対応を減らした方が良い", "1-GPT-SoVITS-TTS": "1-GPT-SoVITS-TTS", "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；": "1、DeEcho-DeReverbモデルの処理時間は、他の2つのDeEchoモデルのほぼ2倍です；", diff --git a/tools/i18n/locale/ko_KR.json b/tools/i18n/locale/ko_KR.json index f2a78336..560cea1f 100644 --- a/tools/i18n/locale/ko_KR.json +++ b/tools/i18n/locale/ko_KR.json @@ -1,11 +1,11 @@ { "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；": "(1)MDX-Net (onnx_dereverb): 듀얼 채널 리버브에는 가장 적합하지만, 싱글 채널 리버브는 제거할 수 없습니다", "(234)DeEcho:去除延迟效果。Aggressive比Normal去除得更彻底，DeReverb额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。": "(234)DeEcho:지연 효과를 제거합니다. Aggressive는 Normal보다 더 철저하게 제거하며, DeReverb는 추가로 리버브를 제거하여 단일 채널 리버브를 제거할 수 있지만 고주파 리버브는 완전히 제거하지 못합니다.", - "*实验/模型名": "*실험/모델 이름", - "*文本标注文件": "*텍스트 주석 파일", - "*训练集音频文件目录": "*훈련 세트 오디오 파일 디렉터리", - "*请上传并填写参考信息": "*참고 정보를 업로드하고 입력하십시오", - "*请填写需要合成的目标文本和语种模式": "*합성할 목표 텍스트와 언어 모드를 입력하세요", + "实验/模型名": "실험/모델 이름", + "文本标注文件": "텍스트 주석 파일", + "训练集音频文件目录": "훈련 세트 오디오 파일 디렉터리", + "请上传并填写参考信息": "참고 정보를 업로드하고 입력하십시오", + "请填写需要合成的目标文本和语种模式": "합성할 목표 텍스트와 언어 모드를 입력하세요", ".限制范围越小判别效果越好。": "다언어 지원을 줄이는 것이 더 좋습니다", "1-GPT-SoVITS-TTS": "1-GPT-SoVITS-TTS", "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；": "1. DeEcho-DeReverb 모델의 처리 시간은 다른 두 DeEcho 모델의 거의 두 배입니다;", diff --git a/tools/i18n/locale/pt_BR.json b/tools/i18n/locale/pt_BR.json index 987ec468..931f0f47 100644 --- a/tools/i18n/locale/pt_BR.json +++ b/tools/i18n/locale/pt_BR.json @@ -1,11 +1,11 @@ { "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；": "(1)MDX-Net (onnx_dereverb): É a melhor opção para reverberação de dois canais, mas não pode remover a reverberação de um único canal;", "(234)DeEcho:去除延迟效果。Aggressive比Normal去除得更彻底，DeReverb额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。": "(234)DeEcho:Remove os efeitos de atraso. Aggressive é mais completo que Normal na remoção, DeReverb remove adicionalmente a reverberação, pode remover a reverberação de um canal único, mas não remove completamente a reverberação de placa de alta frequência.", - "*实验/模型名": "*Nome do experimento/modelo", - "*文本标注文件": "*Arquivo de marcação de texto", - "*训练集音频文件目录": "*Diretório de arquivos de áudio do conjunto de treinamento", - "*请上传并填写参考信息": "Por favor, faça o upload e preencha as informações de referência", - "*请填写需要合成的目标文本和语种模式": "*Por favor, insira o texto alvo a ser sintetizado e o modo de idioma.", + "实验/模型名": "Nome do experimento/modelo", + "文本标注文件": "Arquivo de marcação de texto", + "训练集音频文件目录": "Diretório de arquivos de áudio do conjunto de treinamento", + "请上传并填写参考信息": "Por favor, faça o upload e preencha as informações de referência", + "请填写需要合成的目标文本和语种模式": "*Por favor, insira o texto alvo a ser sintetizado e o modo de idioma.", ".限制范围越小判别效果越好。": "Menos multilinguismo é melhor", "1-GPT-SoVITS-TTS": "1-GPT-SOVITS-TTS", "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；": "1. O tempo de processamento do modelo DeEcho-DeReverb é quase o dobro dos outros dois modelos DeEcho;", diff --git a/tools/i18n/locale/ru_RU.json b/tools/i18n/locale/ru_RU.json index 5d016987..6e5077a8 100644 --- a/tools/i18n/locale/ru_RU.json +++ b/tools/i18n/locale/ru_RU.json @@ -1,11 +1,11 @@ { "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；": "(1)MDX-Net(onnx_dereverb):Это лучший выбор для реверберации с двумя каналами, но он не может устранить реверберацию с одним каналом;", "(234)DeEcho:去除延迟效果。Aggressive比Normal去除得更彻底，DeReverb额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。": "(234)DeEcho:Устраняет эффект задержки. Aggressive устраняет более тщательно, чем Normal, DeReverb дополнительно устраняет реверберацию, может устранить реверберацию с одного канала, но не полностью устраняет высокочастотную реверберацию.", - "*实验/模型名": "*Название эксперимента/модели", - "*文本标注文件": "*Файл текстовой аннотации", - "*训练集音频文件目录": "*Директория аудиофайлов обучающего набора", - "*请上传并填写参考信息": "*Пожалуйста, загрузите и заполните референтные данные", - "*请填写需要合成的目标文本和语种模式": "*Пожалуйста, введите целевой текст для синтеза и режим языка", + "实验/模型名": "Название эксперимента/модели", + "文本标注文件": "Файл текстовой аннотации", + "训练集音频文件目录": "Директория аудиофайлов обучающего набора", + "请上传并填写参考信息": "Пожалуйста, загрузите и заполните референтные данные", + "请填写需要合成的目标文本和语种模式": "Пожалуйста, введите целевой текст для синтеза и режим языка", ".限制范围越小判别效果越好。": "Чем меньше языков, тем лучше", "1-GPT-SoVITS-TTS": "1-GPT-SoVITS-TTS", "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；": "1. Время обработки модели DeEcho-DeReverb почти вдвое больше, чем у двух других моделей DeEcho;", diff --git a/tools/i18n/locale/tr_TR.json b/tools/i18n/locale/tr_TR.json index 41c5684d..8332247f 100644 --- a/tools/i18n/locale/tr_TR.json +++ b/tools/i18n/locale/tr_TR.json @@ -1,11 +1,11 @@ { "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；": "(1)MDX-Net(onnx_dereverb):İki kanallı yankılar için en iyi seçimdir, ancak tek kanallı yankıları ortadan kaldıramaz;", "(234)DeEcho:去除延迟效果。Aggressive比Normal去除得更彻底，DeReverb额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。": "(234)DeEcho:Gecikme etkilerini giderir. Aggressive, Normal'dan daha kapsamlı bir şekilde giderir, DeReverb ek olarak yankıyı giderir, tek kanallı yankıyı giderebilir, ancak yüksek frekanslı plaka yankısını tamamen gideremez.", - "*实验/模型名": "*Deney/model adı", - "*文本标注文件": "*Metin etiketleme dosyası", - "*训练集音频文件目录": "*Eğitim seti ses dosyası dizini", - "*请上传并填写参考信息": "*Lütfen referans bilgilerini yükleyin ve doldurun", - "*请填写需要合成的目标文本和语种模式": "*Lütfen sentezlenecek hedef metni ve dil modunu giriniz.", + "实验/模型名": "Deney/model adı", + "文本标注文件": "Metin etiketleme dosyası", + "训练集音频文件目录": "Eğitim seti ses dosyası dizini", + "请上传并填写参考信息": "Lütfen referans bilgilerini yükleyin ve doldurun", + "请填写需要合成的目标文本和语种模式": "Lütfen sentezlenecek hedef metni ve dil modunu giriniz.", ".限制范围越小判别效果越好。": "Daha az çok dilli olmak daha iyidir", "1-GPT-SoVITS-TTS": "1-GPT-SoVITS-TTS", "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；": "1. DeEcho-DeReverb modelinin işleme süresi, diğer iki DeEcho modelinin neredeyse iki katıdır;", diff --git a/tools/i18n/locale/zh_CN.json b/tools/i18n/locale/zh_CN.json index 8ede7647..57549d84 100644 --- a/tools/i18n/locale/zh_CN.json +++ b/tools/i18n/locale/zh_CN.json @@ -1,11 +1,11 @@ { "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；": "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；", "(234)DeEcho:去除延迟效果。Aggressive比Normal去除得更彻底，DeReverb额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。": "(234)DeEcho:去除延迟效果。Aggressive 比 Normal 去除得更彻底，DeReverb 额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。", - "*实验/模型名": "*实验/模型名", - "*文本标注文件": "*文本标注文件", - "*训练集音频文件目录": "*训练集音频文件目录", - "*请上传并填写参考信息": "*请上传并填写参考信息", - "*请填写需要合成的目标文本和语种模式": "*请填写需要合成的目标文本和语种模式", + "实验/模型名": "实验/模型名", + "文本标注文件": "文本标注文件", + "训练集音频文件目录": "训练集音频文件目录", + "请上传并填写参考信息": "请上传并填写参考信息", + "请填写需要合成的目标文本和语种模式": "请填写需要合成的目标文本和语种模式", ".限制范围越小判别效果越好。": ".限制范围越小判别效果越好。", "1-GPT-SoVITS-TTS": "1-GPT-SoVITS-TTS", "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；": "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；", @@ -222,5 +222,6 @@ "预训练SoVITS-D模型路径": "预训练SoVITS-D模型路径", "预训练SoVITS-G模型路径": "预训练SoVITS-G模型路径", "预训练中文BERT模型路径": "预训练中文BERT模型路径", - "预训练模型路径": "预训练模型路径" + "预训练模型路径": "预训练模型路径", + "推理后端": "推理后端" } diff --git a/tools/i18n/locale/zh_HK.json b/tools/i18n/locale/zh_HK.json index eb9fc810..e4ee5a8c 100644 --- a/tools/i18n/locale/zh_HK.json +++ b/tools/i18n/locale/zh_HK.json @@ -1,11 +1,11 @@ { "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；": "(1)MDX-Net(onnx_dereverb):對於雙通道混響是最佳選擇，但不能去除單通道混響；", "(234)DeEcho:去除延迟效果。Aggressive比Normal去除得更彻底，DeReverb额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。": "(234)DeEcho: 去除延遲效果。Aggressive 比 Normal 去除得更徹底，DeReverb 額外去除混響，可去除單聲道混響，但對高頻重的板式混響去不乾淨。", - "*实验/模型名": "*實驗/模型名", - "*文本标注文件": "*文本標注文件", - "*训练集音频文件目录": "*訓練集音頻文件目錄", - "*请上传并填写参考信息": "*請上傳並填寫參考信息", - "*请填写需要合成的目标文本和语种模式": "請填寫需要合成的目標文本和語言模式", + "实验/模型名": "實驗/模型名", + "文本标注文件": "文本標注文件", + "训练集音频文件目录": "訓練集音頻文件目錄", + "请上传并填写参考信息": "請上傳並填寫參考信息", + "请填写需要合成的目标文本和语种模式": "請填寫需要合成的目標文本和語言模式", ".限制范围越小判别效果越好。": ".限制范围越小判别效果越好。", "1-GPT-SoVITS-TTS": "1-GPT-SoVITS-TTS", "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；": "1、DeEcho-DeReverb 模型的處理時間是另外兩個 DeEcho 模型的接近兩倍；", diff --git a/tools/i18n/locale/zh_SG.json b/tools/i18n/locale/zh_SG.json index d2ca6ae9..a9ca466e 100644 --- a/tools/i18n/locale/zh_SG.json +++ b/tools/i18n/locale/zh_SG.json @@ -1,11 +1,11 @@ { "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；": "(1)MDX-Net(onnx_dereverb)：對於雙通道混響是最好的選擇，不能去除單通道混響；", "(234)DeEcho:去除延迟效果。Aggressive比Normal去除得更彻底，DeReverb额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。": "(234)DeEcho：去除延遲效果。Aggressive 比 Normal 去除得更徹底，DeReverb 額外去除混響，可去除單聲道混響，但是對高頻重的板式混響去不乾淨。", - "*实验/模型名": "*實驗/模型名", - "*文本标注文件": "*文本標註文件", - "*训练集音频文件目录": "*訓練集音頻文件目錄", - "*请上传并填写参考信息": "*請上傳並填寫參考信息", - "*请填写需要合成的目标文本和语种模式": "請填寫需要合成的目標文本和語言模式", + "实验/模型名": "實驗/模型名", + "文本标注文件": "文本標註文件", + "训练集音频文件目录": "訓練集音頻文件目錄", + "请上传并填写参考信息": "請上傳並填寫參考信息", + "请填写需要合成的目标文本和语种模式": "請填寫需要合成的目標文本和語言模式", ".限制范围越小判别效果越好。": ".限制范围越小判别效果越好。", "1-GPT-SoVITS-TTS": "1-GPT-SoVITS-TTS", "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；": "1、DeEcho-DeReverb 模型的耗時是另外兩個 DeEcho 模型的接近兩倍；", diff --git a/tools/i18n/locale/zh_TW.json b/tools/i18n/locale/zh_TW.json index e78e6df3..be9d34c1 100644 --- a/tools/i18n/locale/zh_TW.json +++ b/tools/i18n/locale/zh_TW.json @@ -1,11 +1,11 @@ { "(1)MDX-Net(onnx_dereverb):对于双通道混响是最好的选择，不能去除单通道混响；": "(1)MDX-Net(onnx_dereverb)：對於雙通道混響是最好的選擇，不能去除單通道混響；", "(234)DeEcho:去除延迟效果。Aggressive比Normal去除得更彻底，DeReverb额外去除混响，可去除单声道混响，但是对高频重的板式混响去不干净。": "(234)DeEcho：去除延遲效果。Aggressive 比 Normal 去除得更徹底，DeReverb 額外去除混響，可去除單聲道混響，但是對高頻重的板式混響去不乾淨。", - "*实验/模型名": "*實驗/模型名", - "*文本标注文件": "*文本標注文件", - "*训练集音频文件目录": "*訓練集音頻文件目錄", - "*请上传并填写参考信息": "*請上傳並填寫參考資訊", - "*请填写需要合成的目标文本和语种模式": "請填寫需要合成的目標文本和語言模式", + "实验/模型名": "實驗/模型名", + "文本标注文件": "文本標注文件", + "训练集音频文件目录": "訓練集音頻文件目錄", + "请上传并填写参考信息": "請上傳並填寫參考資訊", + "请填写需要合成的目标文本和语种模式": "請填寫需要合成的目標文本和語言模式", ".限制范围越小判别效果越好。": ".限制范围越小判别效果越好。", "1-GPT-SoVITS-TTS": "1-GPT-SoVITS-TTS", "1、DeEcho-DeReverb模型的耗时是另外2个DeEcho模型的接近2倍；": "1、DeEcho-DeReverb 模型的耗時是另外兩個 DeEcho 模型的接近兩倍；", diff --git a/tools/my_utils.py b/tools/my_utils.py index 04f1a98a..485eecb4 100644 --- a/tools/my_utils.py +++ b/tools/my_utils.py @@ -37,13 +37,11 @@ def load_audio(file, sr): return np.frombuffer(out, np.float32).flatten() -def clean_path(path_str: str): +def clean_path(path_str: str) -> str: if path_str.endswith(("\\", "/")): return clean_path(path_str[0:-1]) path_str = path_str.replace("/", os.sep).replace("\\", os.sep) - return path_str.strip( - " '\n\"\u202a" - ) # path_str.strip(" ").strip('\'').strip("\n").strip('"').strip(" ").strip("\u202a") + return path_str.strip(" '\n\"\u202a") def check_for_existance(file_list: list = None, is_train=False, is_dataset_processing=False): diff --git a/tools/subfix_webui.py b/tools/subfix_webui.py index 51a7dfad..6a815019 100644 --- a/tools/subfix_webui.py +++ b/tools/subfix_webui.py @@ -1,4 +1,5 @@ import sys + from tools.i18n.i18n import I18nAuto, scan_language_list language = sys.argv[-1] if sys.argv[-1] in scan_language_list() else "Auto" @@ -314,7 +315,7 @@ if __name__ == "__main__": "Submit Text: 将当前页所有文本框内容手工保存到内存和文件(翻页前后或者退出标注页面前如果没点这个按钮，你再翻回来就回滚了，白忙活。)" ) ) - with gr.Row(): + with gr.Row(equal_height=True): btn_change_index = gr.Button("Change Index") btn_submit_change = gr.Button("Submit Text") btn_merge_audio = gr.Button("Merge Audio") @@ -322,7 +323,7 @@ if __name__ == "__main__": btn_previous_index = gr.Button("Previous Index") btn_next_index = gr.Button("Next Index") - with gr.Row(): + with gr.Row(equal_height=True): index_slider = gr.Slider(minimum=0, maximum=g_max_json_index, value=g_index, step=1, label="Index", scale=3) splitpoint_slider = gr.Slider( minimum=0, maximum=120.0, value=0, step=0.1, label="Audio Split Point(s)", scale=3 @@ -331,18 +332,23 @@ if __name__ == "__main__": btn_save_json = gr.Button("Save File", visible=True, scale=1) btn_invert_selection = gr.Button("Invert Selection", scale=1) - with gr.Row(): + with gr.Row(equal_height=True): with gr.Column(): for _ in range(0, g_batch): - with gr.Row(): + with gr.Row(equal_height=True): text = gr.Textbox(label="Text", visible=True, scale=5) - audio_output = gr.Audio(label="Output Audio", visible=True, scale=5) + audio_output = gr.Audio( + label="Output Audio", + visible=True, + scale=5, + waveform_options={"show_recording_waveform": False}, + ) audio_check = gr.Checkbox(label="Yes", show_label=True, info="Choose Audio", scale=1) g_text_list.append(text) g_audio_list.append(audio_output) g_checkbox_list.append(audio_check) - with gr.Row(): + with gr.Row(equal_height=True): batchsize_slider = gr.Slider( minimum=1, maximum=g_batch, value=g_batch, step=1, label="Batch Size", scale=3, interactive=False ) diff --git a/tools/uvr5/webui.py b/tools/uvr5/webui.py index f5f8d3f6..98a720a9 100644 --- a/tools/uvr5/webui.py +++ b/tools/uvr5/webui.py @@ -168,7 +168,7 @@ with gr.Blocks(title="UVR5 WebUI", analytics_enabled=False) as app: "h4", ) ) - with gr.Row(): + with gr.Row(equal_height=True): with gr.Column(): model_choose = gr.Dropdown(label=i18n("模型"), choices=uvr5_names) dir_wav_input = gr.Textbox( @@ -197,9 +197,9 @@ with gr.Blocks(title="UVR5 WebUI", analytics_enabled=False) as app: interactive=True, ) with gr.Column(): - with gr.Row(): + with gr.Row(equal_height=True): but2 = gr.Button(i18n("转换"), variant="primary") - with gr.Row(): + with gr.Row(equal_height=True): vc_output4 = gr.Textbox(label=i18n("输出信息"), lines=3) but2.click( uvr, diff --git a/webui.py b/webui.py index 9a6aae5f..5977c2b5 100644 --- a/webui.py +++ b/webui.py @@ -1,22 +1,97 @@ -import os -import sys - -os.environ["version"] = version = "v2Pro" -now_dir = os.getcwd() -sys.path.insert(0, now_dir) -import warnings - -warnings.filterwarnings("ignore") +import argparse +import contextlib import json +import os import platform +import re import shutil import signal +import site +import subprocess +import traceback +import warnings +from multiprocessing import cpu_count +from subprocess import Popen +from turtle import up +import gradio as gr import psutil import torch import yaml +from config import ( + GPU_INDEX, + GPU_INFOS, + IS_GPU, + GPT_weight_root, + GPT_weight_version2root, + SoVITS_weight_root, + SoVITS_weight_version2root, + change_choices, + exp_root, + get_weights_names, + infer_device, + is_half, + is_share, + memset, + pretrained_gpt_name, + pretrained_sovits_name, + python_exec, + webui_port_infer_tts, + webui_port_main, + webui_port_subfix, + webui_port_uvr5, +) +from GPT_SoVITS.Accelerate import backends, logger, tb +from tools import my_utils +from tools.asr.config import asr_dict +from tools.assets import css, js, top_html +from tools.i18n.i18n import I18nAuto, scan_language_list +from tools.my_utils import check_details, check_for_existance + +os.environ["PYTHONPATH"] = now_dir = os.getcwd() + +backends_gradio = [(b.replace("-", " "), b) for b in backends] + +_LANG_RE = re.compile(r"^[a-z]{2}[_-][A-Z]{2}$") + + +def lang_type(text: str) -> str: + if text == "Auto": + return text + if not _LANG_RE.match(text): + raise argparse.ArgumentTypeError(f"Unspported Format: {text}, Expected ll_CC/ll-CC") + ll, cc = re.split(r"[_-]", text) + language = f"{ll}_{cc}" + if language in scan_language_list(): + return language + else: + return "en_US" + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="train_webui", + description="python -s webui.py zh_CN", + ) + p.add_argument( + "language", + nargs="?", + default="Auto", + type=lang_type, + help="Language Code, Such as zh_CN, en-US", + ) + return p + + +args = build_parser().parse_args() + +os.environ["version"] = version = "v2Pro" os.environ["TORCH_DISTRIBUTED_DEBUG"] = "INFO" + +warnings.filterwarnings("ignore") + + torch.manual_seed(233333) tmp = os.path.join(now_dir, "TEMP") os.makedirs(tmp, exist_ok=True) @@ -25,15 +100,13 @@ if os.path.exists(tmp): for name in os.listdir(tmp): if name == "jieba.cache": continue - path = "%s/%s" % (tmp, name) + path = f"{tmp}/{name}" % (tmp, name) delete = os.remove if os.path.isfile(path) else shutil.rmtree try: delete(path) except Exception as e: - print(str(e)) + print(e) pass -import site -import traceback site_packages_roots = [] for path in site.getsitepackages(): @@ -41,7 +114,6 @@ for path in site.getsitepackages(): site_packages_roots.append(path) if site_packages_roots == []: site_packages_roots = ["%s/runtime/Lib/site-packages" % now_dir] -# os.environ["OPENBLAS_NUM_THREADS"] = "4" os.environ["no_proxy"] = "localhost, 127.0.0.1, ::1" os.environ["all_proxy"] = "" for site_packages_root in site_packages_roots: @@ -49,48 +121,16 @@ for site_packages_root in site_packages_roots: try: with open("%s/users.pth" % (site_packages_root), "w") as f: f.write( - # "%s\n%s/runtime\n%s/tools\n%s/tools/asr\n%s/GPT_SoVITS\n%s/tools/uvr5" "%s\n%s/GPT_SoVITS/BigVGAN\n%s/tools\n%s/tools/asr\n%s/GPT_SoVITS\n%s/tools/uvr5" % (now_dir, now_dir, now_dir, now_dir, now_dir, now_dir) ) break except PermissionError: - traceback.print_exc() -import shutil -import subprocess -from subprocess import Popen + logger.error(tb()) -from tools.assets import css, js, top_html -from tools.i18n.i18n import I18nAuto, scan_language_list -language = sys.argv[-1] if sys.argv[-1] in scan_language_list() else "Auto" -os.environ["language"] = language +language = str(args.language) i18n = I18nAuto(language=language) -from multiprocessing import cpu_count - -from config import ( - GPU_INDEX, - GPU_INFOS, - IS_GPU, - exp_root, - infer_device, - is_half, - is_share, - memset, - python_exec, - webui_port_infer_tts, - webui_port_main, - webui_port_subfix, - webui_port_uvr5, -) -from tools import my_utils -from tools.my_utils import check_details, check_for_existance - -os.environ["HF_ENDPOINT"] = "https://hf-mirror.com" -os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" - -# os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1' # 当遇到mps不支持的步骤时使用cpu -import gradio as gr n_cpu = cpu_count() @@ -139,32 +179,9 @@ def set_default(): set_default() -gpus = "-".join(map(str, GPU_INDEX)) default_gpu_numbers = infer_device.index -def fix_gpu_number(input): # 将越界的number强制改到界内 - try: - if int(input) not in set_gpu_numbers: - return default_gpu_numbers - except: - return input - return input - - -def fix_gpu_numbers(inputs): - output = [] - try: - for input in inputs.split(","): - output.append(str(fix_gpu_number(input))) - return ",".join(output) - except: - return inputs - - -from config import pretrained_gpt_name, pretrained_sovits_name - - def check_pretrained_is_exist(version): pretrained_model_list = ( pretrained_sovits_name[version], @@ -175,7 +192,7 @@ def check_pretrained_is_exist(version): ) _ = "" for i in pretrained_model_list: - if "s2Dv3" not in i and "s2Dv4" not in i and os.path.exists(i) == False: + if "s2Dv3" not in i and "s2Dv4" not in i and os.path.exists(i) is False: _ += f"\n {i}" if _: print("warning: ", i18n("以下模型不存在:") + _) @@ -183,62 +200,43 @@ def check_pretrained_is_exist(version): check_pretrained_is_exist(version) for key in pretrained_sovits_name.keys(): - if os.path.exists(pretrained_sovits_name[key]) == False: + if os.path.exists(pretrained_sovits_name[key]) is False: pretrained_sovits_name[key] = "" for key in pretrained_gpt_name.keys(): - if os.path.exists(pretrained_gpt_name[key]) == False: + if os.path.exists(pretrained_gpt_name[key]) is False: pretrained_gpt_name[key] = "" -from config import ( - GPT_weight_root, - GPT_weight_version2root, - SoVITS_weight_root, - SoVITS_weight_version2root, - change_choices, - get_weights_names, -) for root in SoVITS_weight_root + GPT_weight_root: os.makedirs(root, exist_ok=True) SoVITS_names, GPT_names = get_weights_names() -p_label = None -p_uvr5 = None -p_asr = None -p_denoise = None -p_tts_inference = None +p_label: Popen | None = None +p_uvr5: Popen | None = None +p_asr: Popen | None = None +p_denoise: Popen | None = None +p_tts_inference: Popen | None = None -def kill_proc_tree(pid, including_parent=True): +def kill_process(pid: int, process_name=""): try: - parent = psutil.Process(pid) + p = psutil.Process(pid) except psutil.NoSuchProcess: - # Process already terminated return - children = parent.children(recursive=True) - for child in children: + for c in p.children(recursive=False): try: - os.kill(child.pid, signal.SIGTERM) # or signal.SIGKILL - except OSError: - pass - if including_parent: - try: - os.kill(parent.pid, signal.SIGTERM) # or signal.SIGKILL - except OSError: + c.kill() + c.wait(timeout=5) + except (psutil.NoSuchProcess, psutil.TimeoutExpired): pass + try: + p.kill() + p.wait(timeout=5) + except (psutil.NoSuchProcess, psutil.TimeoutExpired): + pass -system = platform.system() - - -def kill_process(pid, process_name=""): - if system == "Windows": - cmd = "taskkill /t /f /pid %s" % pid - # os.system(cmd) - subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - else: - kill_proc_tree(pid) print(process_name + i18n("进程已终止")) @@ -281,8 +279,8 @@ def change_label(path_list): ) yield ( process_info(process_name_subfix, "opened"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) print(cmd) p_label = Popen(cmd, shell=True) @@ -291,8 +289,8 @@ def change_label(path_list): p_label = None yield ( process_info(process_name_subfix, "closed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) @@ -311,8 +309,8 @@ def change_uvr5(): ) yield ( process_info(process_name_uvr5, "opened"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) print(cmd) p_uvr5 = Popen(cmd, shell=True) @@ -321,51 +319,66 @@ def change_uvr5(): p_uvr5 = None yield ( process_info(process_name_uvr5, "closed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) process_name_tts = i18n("TTS推理WebUI") -def change_tts_inference(bert_path, cnhubert_base_path, gpu_number, gpt_path, sovits_path, batched_infer_enabled): +def change_tts_inference( + bert_path: str, + cnhubert_base_path: str, + gpu_number: int, + gpt_path: str, + sovits_path: str, + batched_infer_enabled: bool, + backends_dropdown: str, +): global p_tts_inference + env = os.environ.copy() + env["CUDA_VISIBLE_DEVICES"] = str(gpu_number) + cmd: list[str] = [python_exec, "-s"] if batched_infer_enabled: - cmd = '"%s" -s GPT_SoVITS/inference_webui_fast.py "%s"' % (python_exec, language) + cmd.extend(["GPT_SoVITS/inference_webui_fast.py", language]) else: - cmd = '"%s" -s GPT_SoVITS/inference_webui.py "%s"' % (python_exec, language) - # #####v3暂不支持加速推理 - # if version=="v3": - # cmd = '"%s" GPT_SoVITS/inference_webui.py "%s"'%(python_exec, language) + # fmt: off + cmd.extend( + [ + "GPT_SoVITS/inference_webui.py", language, + "-b", backends_dropdown, + "-d", "{infer_device.type}:{gpu_number}", + "-p", str(webui_port_infer_tts), + ] + ) # fmt: on + + if is_share: + cmd.append("-s") + if p_tts_inference is None: - os.environ["gpt_path"] = gpt_path - os.environ["sovits_path"] = sovits_path - os.environ["cnhubert_base_path"] = cnhubert_base_path - os.environ["bert_path"] = bert_path - os.environ["_CUDA_VISIBLE_DEVICES"] = fix_gpu_number(gpu_number) - os.environ["is_half"] = str(is_half) - os.environ["infer_ttswebui"] = str(webui_port_infer_tts) - os.environ["is_share"] = str(is_share) + env["gpt_path"] = gpt_path + env["sovits_path"] = sovits_path + env["is_half"] = str(is_half) + env["infer_ttswebui"] = str(webui_port_infer_tts) + env["is_share"] = str(is_share) yield ( process_info(process_name_tts, "opened"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) print(cmd) - p_tts_inference = Popen(cmd, shell=True) + p_tts_inference = Popen(cmd, env=env) else: kill_process(p_tts_inference.pid, process_name_tts) p_tts_inference = None yield ( process_info(process_name_tts, "closed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) -from tools.asr.config import asr_dict - process_name_asr = i18n("语音识别") @@ -386,11 +399,11 @@ def open_asr(asr_inp_dir, asr_opt_dir, asr_model, asr_model_size, asr_lang, asr_ output_file_path = os.path.abspath(f"{output_folder}/{output_file_name}.list") yield ( process_info(process_name_asr, "opened"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, - {"__type__": "update"}, - {"__type__": "update"}, - {"__type__": "update"}, + gr.update(visible=False), + gr.update(visible=True), + gr.skip(), + gr.skip(), + gr.skip(), ) print(cmd) p_asr = Popen(cmd, shell=True) @@ -398,20 +411,20 @@ def open_asr(asr_inp_dir, asr_opt_dir, asr_model, asr_model_size, asr_lang, asr_ p_asr = None yield ( process_info(process_name_asr, "finish"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, - {"__type__": "update", "value": output_file_path}, - {"__type__": "update", "value": output_file_path}, - {"__type__": "update", "value": asr_inp_dir}, + gr.update(visible=True), + gr.update(visible=False), + gr.update(value=output_file_path), + gr.update(value=output_file_path), + gr.update(value=asr_inp_dir), ) else: yield ( process_info(process_name_asr, "occupy"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, - {"__type__": "update"}, - {"__type__": "update"}, - {"__type__": "update"}, + gr.update(visible=False), + gr.update(visible=True), + gr.skip(), + gr.skip(), + gr.skip(), ) @@ -422,64 +435,8 @@ def close_asr(): p_asr = None return ( process_info(process_name_asr, "closed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, - ) - - -process_name_denoise = i18n("语音降噪") - - -def open_denoise(denoise_inp_dir, denoise_opt_dir): - global p_denoise - if p_denoise == None: - denoise_inp_dir = my_utils.clean_path(denoise_inp_dir) - denoise_opt_dir = my_utils.clean_path(denoise_opt_dir) - check_for_existance([denoise_inp_dir]) - cmd = '"%s" -s tools/cmd-denoise.py -i "%s" -o "%s" -p %s' % ( - python_exec, - denoise_inp_dir, - denoise_opt_dir, - "float16" if is_half == True else "float32", - ) - - yield ( - process_info(process_name_denoise, "opened"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, - {"__type__": "update"}, - {"__type__": "update"}, - ) - print(cmd) - p_denoise = Popen(cmd, shell=True) - p_denoise.wait() - p_denoise = None - yield ( - process_info(process_name_denoise, "finish"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, - {"__type__": "update", "value": denoise_opt_dir}, - {"__type__": "update", "value": denoise_opt_dir}, - ) - else: - yield ( - process_info(process_name_denoise, "occupy"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, - {"__type__": "update"}, - {"__type__": "update"}, - ) - - -def close_denoise(): - global p_denoise - if p_denoise is not None: - kill_process(p_denoise.pid, process_name_denoise) - p_denoise = None - return ( - process_info(process_name_denoise, "closed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) @@ -503,7 +460,7 @@ def open1Ba( lora_rank, ): global p_train_SoVITS - if p_train_SoVITS == None: + if p_train_SoVITS is None: exp_name = exp_name.rstrip(" ") config_file = ( "GPT_SoVITS/configs/s2.json" @@ -511,13 +468,13 @@ def open1Ba( else f"GPT_SoVITS/configs/s2{version}.json" ) with open(config_file) as f: - data = f.read() - data = json.loads(data) + config = f.read() + data: dict = json.loads(config) s2_dir = "%s/%s" % (exp_root, exp_name) os.makedirs("%s/logs_s2_%s" % (s2_dir, version), exist_ok=True) if check_for_existance([s2_dir], is_train=True): check_details([s2_dir], is_train=True) - if is_half == False: + if is_half is False: data["train"]["fp16_run"] = False batch_size = max(1, batch_size // 2) data["train"]["batch_size"] = batch_size @@ -528,7 +485,6 @@ def open1Ba( data["train"]["if_save_latest"] = if_save_latest data["train"]["if_save_every_weights"] = if_save_every_weights data["train"]["save_every_epoch"] = save_every_epoch - data["train"]["gpu_numbers"] = gpu_numbers1Ba data["train"]["grad_ckpt"] = if_grad_ckpt data["train"]["lora_rank"] = lora_rank data["model"]["version"] = version @@ -539,16 +495,17 @@ def open1Ba( tmp_config_path = "%s/tmp_s2.json" % tmp with open(tmp_config_path, "w") as f: f.write(json.dumps(data)) + os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_numbers1Ba).strip("[]").replace(" ", "") if version in ["v1", "v2", "v2Pro", "v2ProPlus"]: cmd = '"%s" -s GPT_SoVITS/s2_train.py --config "%s"' % (python_exec, tmp_config_path) else: cmd = '"%s" -s GPT_SoVITS/s2_train_v3_lora.py --config "%s"' % (python_exec, tmp_config_path) yield ( process_info(process_name_sovits, "opened"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, - {"__type__": "update"}, - {"__type__": "update"}, + gr.update(visible=False), + gr.update(visible=True), + gr.skip(), + gr.skip(), ) print(cmd) p_train_SoVITS = Popen(cmd, shell=True) @@ -557,18 +514,18 @@ def open1Ba( SoVITS_dropdown_update, GPT_dropdown_update = change_choices() yield ( process_info(process_name_sovits, "finish"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), SoVITS_dropdown_update, GPT_dropdown_update, ) else: yield ( process_info(process_name_sovits, "occupy"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, - {"__type__": "update"}, - {"__type__": "update"}, + gr.update(visible=False), + gr.update(visible=True), + gr.skip(), + gr.skip(), ) @@ -579,8 +536,8 @@ def close1Ba(): p_train_SoVITS = None return ( process_info(process_name_sovits, "closed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) @@ -600,18 +557,18 @@ def open1Bb( pretrained_s1, ): global p_train_GPT - if p_train_GPT == None: + if p_train_GPT is None: exp_name = exp_name.rstrip(" ") with open( "GPT_SoVITS/configs/s1longer.yaml" if version == "v1" else "GPT_SoVITS/configs/s1longer-v2.yaml" ) as f: - data = f.read() - data = yaml.load(data, Loader=yaml.FullLoader) + config = f.read() + data: dict = yaml.load(config, Loader=yaml.FullLoader) s1_dir = "%s/%s" % (exp_root, exp_name) os.makedirs("%s/logs_s1" % (s1_dir), exist_ok=True) if check_for_existance([s1_dir], is_train=True): check_details([s1_dir], is_train=True) - if is_half == False: + if is_half is False: data["train"]["precision"] = "32" batch_size = max(1, batch_size // 2) data["train"]["batch_size"] = batch_size @@ -628,7 +585,7 @@ def open1Bb( data["output_dir"] = "%s/logs_s1_%s" % (s1_dir, version) # data["version"]=version - os.environ["_CUDA_VISIBLE_DEVICES"] = fix_gpu_numbers(gpu_numbers.replace("-", ",")) + os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_numbers).strip("[]").replace(" ", "") os.environ["hz"] = "25hz" tmp_config_path = "%s/tmp_s1.yaml" % tmp with open(tmp_config_path, "w") as f: @@ -637,10 +594,10 @@ def open1Bb( cmd = '"%s" -s GPT_SoVITS/s1_train.py --config_file "%s" ' % (python_exec, tmp_config_path) yield ( process_info(process_name_gpt, "opened"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, - {"__type__": "update"}, - {"__type__": "update"}, + gr.update(visible=False), + gr.update(visible=True), + gr.skip(), + gr.skip(), ) print(cmd) p_train_GPT = Popen(cmd, shell=True) @@ -649,18 +606,18 @@ def open1Bb( SoVITS_dropdown_update, GPT_dropdown_update = change_choices() yield ( process_info(process_name_gpt, "finish"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), SoVITS_dropdown_update, GPT_dropdown_update, ) else: yield ( process_info(process_name_gpt, "occupy"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, - {"__type__": "update"}, - {"__type__": "update"}, + gr.update(visible=False), + gr.update(visible=True), + gr.skip(), + gr.skip(), ) @@ -671,8 +628,8 @@ def close1Bb(): p_train_GPT = None return ( process_info(process_name_gpt, "closed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) @@ -685,14 +642,13 @@ def open_slice(inp, opt_root, threshold, min_length, min_interval, hop_size, max inp = my_utils.clean_path(inp) opt_root = my_utils.clean_path(opt_root) check_for_existance([inp]) - if os.path.exists(inp) == False: + if os.path.exists(inp) is False: yield ( i18n("输入路径不存在"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, - {"__type__": "update"}, - {"__type__": "update"}, - {"__type__": "update"}, + gr.update(visible=True), + gr.update(visible=False), + gr.skip(), + gr.skip(), ) return if os.path.isfile(inp): @@ -702,11 +658,10 @@ def open_slice(inp, opt_root, threshold, min_length, min_interval, hop_size, max else: yield ( i18n("输入路径存在但不可用"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, - {"__type__": "update"}, - {"__type__": "update"}, - {"__type__": "update"}, + gr.update(visible=True), + gr.update(visible=False), + gr.skip(), + gr.skip(), ) return if ps_slice == []: @@ -730,31 +685,28 @@ def open_slice(inp, opt_root, threshold, min_length, min_interval, hop_size, max ps_slice.append(p) yield ( process_info(process_name_slice, "opened"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, - {"__type__": "update"}, - {"__type__": "update"}, - {"__type__": "update"}, + gr.update(visible=False), + gr.update(visible=True), + gr.skip(), + gr.skip(), ) for p in ps_slice: p.wait() ps_slice = [] yield ( process_info(process_name_slice, "finish"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, - {"__type__": "update", "value": opt_root}, - {"__type__": "update", "value": opt_root}, - {"__type__": "update", "value": opt_root}, + gr.update(visible=True), + gr.update(visible=False), + gr.update(value=opt_root), + gr.update(value=opt_root), ) else: yield ( process_info(process_name_slice, "occupy"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, - {"__type__": "update"}, - {"__type__": "update"}, - {"__type__": "update"}, + gr.update(visible=False), + gr.update(visible=True), + gr.skip(), + gr.skip(), ) @@ -764,86 +716,81 @@ def close_slice(): for p_slice in ps_slice: try: kill_process(p_slice.pid, process_name_slice) - except: + except Exception as _: traceback.print_exc() ps_slice = [] return ( process_info(process_name_slice, "closed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) -ps1a = [] +ps1a = None process_name_1a = i18n("文本分词与特征提取") -def open1a(inp_text, inp_wav_dir, exp_name, gpu_numbers, bert_pretrained_dir): +def open1a( + inp_text: str, + inp_wav_dir: str, + exp_name: str, + gpu_numbers: list[int], + bert_pretrained_dir: str, + version: str, + nproc: int = 1, +): global ps1a inp_text = my_utils.clean_path(inp_text) inp_wav_dir = my_utils.clean_path(inp_wav_dir) if check_for_existance([inp_text, inp_wav_dir], is_dataset_processing=True): check_details([inp_text, inp_wav_dir], is_dataset_processing=True) exp_name = exp_name.rstrip(" ") - if ps1a == []: - opt_dir = "%s/%s" % (exp_root, exp_name) - config = { - "inp_text": inp_text, - "inp_wav_dir": inp_wav_dir, - "exp_name": exp_name, - "opt_dir": opt_dir, - "bert_pretrained_dir": bert_pretrained_dir, - } - gpu_names = gpu_numbers.split("-") - all_parts = len(gpu_names) - for i_part in range(all_parts): - config.update( - { - "i_part": str(i_part), - "all_parts": str(all_parts), - "_CUDA_VISIBLE_DEVICES": fix_gpu_number(gpu_names[i_part]), - "is_half": str(is_half), - } - ) - os.environ.update(config) - cmd = '"%s" -s GPT_SoVITS/prepare_datasets/1-get-text.py' % python_exec - print(cmd) - p = Popen(cmd, shell=True) - ps1a.append(p) + if ps1a is None: + opt_dir = f"{exp_root}/{exp_name}" + + # fmt: off + cmd = [ + python_exec, "-s", "GPT_SoVITS/prepare_datasets/1-get-text.py", + "--inp", inp_text, + "--opt", opt_dir, + "--bert", bert_pretrained_dir, + "--version", version, + "--device", infer_device.type, + "--device-id", str(gpu_numbers).strip("[]").replace(" ",""), + "--nproc", str(nproc), + ] # fmt: on + + if is_half: + cmd.append("--fp16") + + p = Popen(cmd) + yield ( process_info(process_name_1a, "running"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True) ) - for p in ps1a: - p.wait() - opt = [] - for i_part in range(all_parts): - txt_path = "%s/2-name2text-%s.txt" % (opt_dir, i_part) - with open(txt_path, "r", encoding="utf8") as f: - opt += f.read().strip("\n").split("\n") - os.remove(txt_path) - path_text = "%s/2-name2text.txt" % opt_dir - with open(path_text, "w", encoding="utf8") as f: - f.write("\n".join(opt) + "\n") - ps1a = [] - if len("".join(opt)) > 0: + + code = p.wait() + ps1a = None + + if code == 0: yield ( process_info(process_name_1a, "finish"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) else: yield ( process_info(process_name_1a, "failed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) else: yield ( process_info(process_name_1a, "occupy"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) @@ -853,13 +800,13 @@ def close1a(): for p1a in ps1a: try: kill_process(p1a.pid, process_name_1a) - except: + except Exception as _: traceback.print_exc() ps1a = [] return ( process_info(process_name_1a, "closed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) @@ -892,7 +839,6 @@ def open1b(version, inp_text, inp_wav_dir, exp_name, gpu_numbers, ssl_pretrained { "i_part": str(i_part), "all_parts": str(all_parts), - "_CUDA_VISIBLE_DEVICES": fix_gpu_number(gpu_names[i_part]), } ) os.environ.update(config) @@ -902,8 +848,8 @@ def open1b(version, inp_text, inp_wav_dir, exp_name, gpu_numbers, ssl_pretrained ps1b.append(p) yield ( process_info(process_name_1b, "running"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) for p in ps1b: p.wait() @@ -914,7 +860,6 @@ def open1b(version, inp_text, inp_wav_dir, exp_name, gpu_numbers, ssl_pretrained { "i_part": str(i_part), "all_parts": str(all_parts), - "_CUDA_VISIBLE_DEVICES": fix_gpu_number(gpu_names[i_part]), } ) os.environ.update(config) @@ -927,14 +872,14 @@ def open1b(version, inp_text, inp_wav_dir, exp_name, gpu_numbers, ssl_pretrained ps1b = [] yield ( process_info(process_name_1b, "finish"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) else: yield ( process_info(process_name_1b, "occupy"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) @@ -944,13 +889,13 @@ def close1b(): for p1b in ps1b: try: kill_process(p1b.pid, process_name_1b) - except: + except Exception as _: traceback.print_exc() ps1b = [] return ( process_info(process_name_1b, "closed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) @@ -986,7 +931,6 @@ def open1c(version, inp_text, inp_wav_dir, exp_name, gpu_numbers, pretrained_s2G { "i_part": str(i_part), "all_parts": str(all_parts), - "_CUDA_VISIBLE_DEVICES": fix_gpu_number(gpu_names[i_part]), } ) os.environ.update(config) @@ -996,8 +940,8 @@ def open1c(version, inp_text, inp_wav_dir, exp_name, gpu_numbers, pretrained_s2G ps1c.append(p) yield ( process_info(process_name_1c, "running"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) for p in ps1c: p.wait() @@ -1013,14 +957,14 @@ def open1c(version, inp_text, inp_wav_dir, exp_name, gpu_numbers, pretrained_s2G ps1c = [] yield ( process_info(process_name_1c, "finish"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) else: yield ( process_info(process_name_1c, "occupy"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) @@ -1030,13 +974,13 @@ def close1c(): for p1c in ps1c: try: kill_process(p1c.pid, process_name_1c) - except: + except Exception as _: traceback.print_exc() ps1c = [] return ( process_info(process_name_1c, "closed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) @@ -1067,8 +1011,8 @@ def open1abc( try: #############################1a path_text = "%s/2-name2text.txt" % opt_dir - if os.path.exists(path_text) == False or ( - os.path.exists(path_text) == True + if os.path.exists(path_text) is False or ( + os.path.exists(path_text) is True and len(open(path_text, "r", encoding="utf8").read().strip("\n").split("\n")) < 2 ): config = { @@ -1086,7 +1030,6 @@ def open1abc( { "i_part": str(i_part), "all_parts": str(all_parts), - "_CUDA_VISIBLE_DEVICES": fix_gpu_number(gpu_names[i_part]), } ) os.environ.update(config) @@ -1096,8 +1039,8 @@ def open1abc( ps1abc.append(p) yield ( i18n("进度") + ": 1A-Doing", - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) for p in ps1abc: p.wait() @@ -1113,8 +1056,8 @@ def open1abc( assert len("".join(opt)) > 0, process_info(process_name_1a, "failed") yield ( i18n("进度") + ": 1A-Done", - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) ps1abc = [] #############################1b @@ -1133,7 +1076,6 @@ def open1abc( { "i_part": str(i_part), "all_parts": str(all_parts), - "_CUDA_VISIBLE_DEVICES": fix_gpu_number(gpu_names[i_part]), } ) os.environ.update(config) @@ -1143,8 +1085,8 @@ def open1abc( ps1abc.append(p) yield ( i18n("进度") + ": 1A-Done, 1B-Doing", - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) for p in ps1abc: p.wait() @@ -1155,7 +1097,6 @@ def open1abc( { "i_part": str(i_part), "all_parts": str(all_parts), - "_CUDA_VISIBLE_DEVICES": fix_gpu_number(gpu_names[i_part]), } ) os.environ.update(config) @@ -1168,13 +1109,13 @@ def open1abc( ps1abc = [] yield ( i18n("进度") + ": 1A-Done, 1B-Done", - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) #############################1c path_semantic = "%s/6-name2semantic.tsv" % opt_dir - if os.path.exists(path_semantic) == False or ( - os.path.exists(path_semantic) == True and os.path.getsize(path_semantic) < 31 + if os.path.exists(path_semantic) is False or ( + os.path.exists(path_semantic) is True and os.path.getsize(path_semantic) < 31 ): config_file = ( "GPT_SoVITS/configs/s2.json" @@ -1195,7 +1136,6 @@ def open1abc( { "i_part": str(i_part), "all_parts": str(all_parts), - "_CUDA_VISIBLE_DEVICES": fix_gpu_number(gpu_names[i_part]), } ) os.environ.update(config) @@ -1205,8 +1145,8 @@ def open1abc( ps1abc.append(p) yield ( i18n("进度") + ": 1A-Done, 1B-Done, 1C-Doing", - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) for p in ps1abc: p.wait() @@ -1221,28 +1161,28 @@ def open1abc( f.write("\n".join(opt) + "\n") yield ( i18n("进度") + ": 1A-Done, 1B-Done, 1C-Done", - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) ps1abc = [] yield ( process_info(process_name_1abc, "finish"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) - except: + except Exception as _: traceback.print_exc() close1abc() yield ( process_info(process_name_1abc, "failed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) else: yield ( process_info(process_name_1abc, "occupy"), - {"__type__": "update", "visible": False}, - {"__type__": "update", "visible": True}, + gr.update(visible=False), + gr.update(visible=True), ) @@ -1252,13 +1192,13 @@ def close1abc(): for p1abc in ps1abc: try: kill_process(p1abc.pid, process_name_1abc) - except: + except Exception as _: traceback.print_exc() ps1abc = [] return ( process_info(process_name_1abc, "closed"), - {"__type__": "update", "visible": True}, - {"__type__": "update", "visible": False}, + gr.update(visible=True), + gr.update(visible=False), ) @@ -1272,23 +1212,20 @@ def switch_version(version_): gr.Warning(i18n("未下载模型") + ": " + version.upper()) set_default() return ( - {"__type__": "update", "value": pretrained_sovits_name[version]}, - {"__type__": "update", "value": pretrained_sovits_name[version].replace("s2G", "s2D")}, - {"__type__": "update", "value": pretrained_gpt_name[version]}, - {"__type__": "update", "value": pretrained_gpt_name[version]}, - {"__type__": "update", "value": pretrained_sovits_name[version]}, - {"__type__": "update", "value": default_batch_size, "maximum": default_max_batch_size}, - {"__type__": "update", "value": default_sovits_epoch, "maximum": max_sovits_epoch}, - {"__type__": "update", "value": default_sovits_save_every_epoch, "maximum": max_sovits_save_every_epoch}, - {"__type__": "update", "visible": True if version not in v3v4set else False}, - { - "__type__": "update", - "value": False if not if_force_ckpt else True, - "interactive": True if not if_force_ckpt else False, - }, - {"__type__": "update", "interactive": True, "value": False}, - {"__type__": "update", "visible": True if version in v3v4set else False}, - ) # {'__type__': 'update', "interactive": False if version in v3v4set else True, "value": False}, \ ####batch infer + gr.update(value=pretrained_sovits_name[version]), + gr.update(value=pretrained_sovits_name[version].replace("s2G", "s2D")), + gr.update(value=pretrained_gpt_name[version]), + gr.update(value=pretrained_gpt_name[version]), + gr.update(value=pretrained_sovits_name[version]), + gr.update(value=default_batch_size, maximum=default_max_batch_size), + gr.update(value=default_sovits_epoch, maximum=max_sovits_epoch), + gr.update(value=default_sovits_save_every_epoch, maximum=max_sovits_save_every_epoch), + gr.update(visible=True if version not in v3v4set else False), + gr.update(value=False if not if_force_ckpt else True, interactive=True if not if_force_ckpt else False), + gr.update(value=False, interactive=True), + gr.update(value=True, interactive=False), + gr.update(visible=True if version in v3v4set else False), + ) if os.path.exists("GPT_SoVITS/text/G2PWModel"): @@ -1300,9 +1237,19 @@ else: def sync(text): - return {"__type__": "update", "value": text} + return gr.update(value=text) +def changeBackend(flag: bool): + if flag: + return gr.update(choices=["Torch Varlen"], value="Torch Varlen") + else: + return gr.update(choices=backends_gradio, value=backends_gradio[-1][-1]) + + +GPU_INDEX.add(0) +GPU_INDEX_LIST = list(GPU_INDEX) +GPU_INDEX_LIST.sort() with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css) as app: gr.HTML( top_html.format( @@ -1315,9 +1262,9 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css with gr.Tabs(): with gr.TabItem("0-" + i18n("前置数据集获取工具")): # 提前随机切片防止uvr5爆内存->uvr5->slicer->asr->打标 with gr.Accordion(label="0a-" + i18n("UVR5人声伴奏分离&去混响去延迟工具")): - with gr.Row(): + with gr.Row(equal_height=True): with gr.Column(scale=3): - with gr.Row(): + with gr.Row(equal_height=True): uvr5_info = gr.Textbox(label=process_info(process_name_uvr5, "info")) open_uvr5 = gr.Button( value=process_info(process_name_uvr5, "open"), variant="primary", visible=True @@ -1327,14 +1274,19 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css ) with gr.Accordion(label="0b-" + i18n("语音切分工具")): - with gr.Row(): + with gr.Row(equal_height=True): with gr.Column(scale=3): - with gr.Row(): - slice_inp_path = gr.Textbox(label=i18n("音频自动切分输入路径，可文件可文件夹"), value="") + with gr.Row(equal_height=True): + slice_inp_path = gr.Textbox( + label=i18n("音频自动切分输入路径，可文件可文件夹"), + placeholder="D:/InputAudioFolder" + if platform.system() == "Windows" + else "~/InputAudioFolder", + ) slice_opt_root = gr.Textbox( label=i18n("切分后的子音频的输出根目录"), value="output/slicer_opt" ) - with gr.Row(): + with gr.Row(equal_height=True): threshold = gr.Textbox( label=i18n("threshold:音量小于这个值视作静音的备选切割点"), value="-34" ) @@ -1348,7 +1300,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css value="10", ) max_sil_kept = gr.Textbox(label=i18n("max_sil_kept:切完后静音最多留多长"), value="500") - with gr.Row(): + with gr.Row(equal_height=True): _max = gr.Slider( minimum=0, maximum=1, @@ -1365,7 +1317,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css value=0.25, interactive=True, ) - with gr.Row(): + with gr.Row(equal_height=True): n_process = gr.Slider( minimum=1, maximum=n_cpu, @@ -1382,32 +1334,19 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css value=process_info(process_name_slice, "close"), variant="primary", visible=False ) - # gr.Markdown(value="0bb-" + i18n("语音降噪工具")+i18n("(不稳定，先别用，可能劣化模型效果！)")) - with gr.Row(visible=False): - with gr.Column(scale=3): - with gr.Row(): - denoise_input_dir = gr.Textbox(label=i18n("输入文件夹路径"), value="") - denoise_output_dir = gr.Textbox(label=i18n("输出文件夹路径"), value="output/denoise_opt") - with gr.Row(): - denoise_info = gr.Textbox(label=process_info(process_name_denoise, "info")) - open_denoise_button = gr.Button( - value=process_info(process_name_denoise, "open"), variant="primary", visible=True - ) - close_denoise_button = gr.Button( - value=process_info(process_name_denoise, "close"), variant="primary", visible=False - ) - with gr.Accordion(label="0c-" + i18n("语音识别工具")): - with gr.Row(): + with gr.Row(equal_height=True): with gr.Column(scale=3): - with gr.Row(): + with gr.Row(equal_height=True): asr_inp_dir = gr.Textbox( - label=i18n("输入文件夹路径"), value="D:\\GPT-SoVITS\\raw\\xxx", interactive=True + label=i18n("输入文件夹路径"), + value="output/silcer_opt", + interactive=True, ) asr_opt_dir = gr.Textbox( label=i18n("输出文件夹路径"), value="output/asr_opt", interactive=True ) - with gr.Row(): + with gr.Row(equal_height=True): asr_model = gr.Dropdown( label=i18n("ASR 模型"), choices=list(asr_dict.keys()), @@ -1423,7 +1362,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css asr_precision = gr.Dropdown( label=i18n("数据类型精度"), choices=["float32"], interactive=True, value="float32" ) - with gr.Row(): + with gr.Row(equal_height=True): asr_info = gr.Textbox(label=process_info(process_name_asr, "info")) open_asr_button = gr.Button( value=process_info(process_name_asr, "open"), variant="primary", visible=True @@ -1433,10 +1372,10 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css ) def change_lang_choices(key): # 根据选择的模型修改可选的语言 - return {"__type__": "update", "choices": asr_dict[key]["lang"], "value": asr_dict[key]["lang"][0]} + return gr.update(value=asr_dict[key]["lang"][0], choices=asr_dict[key]["lang"]) def change_size_choices(key): # 根据选择的模型修改可选的模型尺寸 - return {"__type__": "update", "choices": asr_dict[key]["size"], "value": asr_dict[key]["size"][-1]} + return gr.update(value=asr_dict[key]["size"][-1], choices=asr_dict[key]["size"]) def change_precision_choices(key): # 根据选择的模型修改可选的语言 if key == "Faster Whisper (多语种)": @@ -1448,19 +1387,19 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css precision = "float32" else: precision = "float32" - return {"__type__": "update", "choices": asr_dict[key]["precision"], "value": precision} + return gr.update(value=precision, choices=asr_dict[key]["precision"]) asr_model.change(change_lang_choices, [asr_model], [asr_lang]) asr_model.change(change_size_choices, [asr_model], [asr_size]) asr_model.change(change_precision_choices, [asr_model], [asr_precision]) with gr.Accordion(label="0d-" + i18n("语音文本校对标注工具")): - with gr.Row(): + with gr.Row(equal_height=True): with gr.Column(scale=3): - with gr.Row(): + with gr.Row(equal_height=True): path_list = gr.Textbox( label=i18n("标注文件路径 (含文件后缀 *.list)"), - value="D:\\RVC1006\\GPT-SoVITS\\raw\\xxx.list", + value="output/asr_opt/slicer_opt.list", interactive=True, ) label_info = gr.Textbox(label=process_info(process_name_subfix, "info")) @@ -1478,14 +1417,15 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css with gr.TabItem(i18n("1-GPT-SoVITS-TTS")): with gr.Accordion(i18n("微调模型信息")): - with gr.Row(): - with gr.Row(equal_height=True): + with gr.Row(equal_height=True): + with gr.Column(): exp_name = gr.Textbox( - label=i18n("*实验/模型名"), + label=i18n("实验/模型名"), value="xxx", interactive=True, scale=3, ) + with gr.Column(): gpu_info_box = gr.Textbox( label=i18n("显卡信息"), value=gpu_info, @@ -1493,14 +1433,24 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css interactive=False, scale=5, ) - version_checkbox = gr.Radio( + with gr.Column(): + version_checkbox = gr.Dropdown( label=i18n("训练模型的版本"), value=version, - choices=["v1", "v2", "v4", "v2Pro", "v2ProPlus"], + choices=[ + ("V1", "v1"), + ("V2", "v2"), + ("V4", "v4"), + ("V2 Pro", "v2Pro"), + ("V2 Pro Plus", "v2ProPlus"), + ], scale=5, ) + with gr.Column(): + n_processes = gr.Slider(0, 10, 2, step=1, label=i18n("每卡预处理进程数")) + with gr.Accordion(label=i18n("预训练模型路径"), open=False): - with gr.Row(): + with gr.Row(equal_height=True): with gr.Row(equal_height=True): pretrained_s1 = gr.Textbox( label=i18n("预训练GPT模型路径"), @@ -1529,15 +1479,15 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css with gr.TabItem("1A-" + i18n("训练集格式化工具")): with gr.Accordion(label=i18n("输出logs/实验名目录下应有23456开头的文件和文件夹")): - with gr.Row(): - with gr.Row(): + with gr.Row(equal_height=True): + with gr.Row(equal_height=True): inp_text = gr.Textbox( label=i18n("*文本标注文件"), value=r"D:\RVC1006\GPT-SoVITS\raw\xxx.list", interactive=True, scale=10, ) - with gr.Row(): + with gr.Row(equal_height=True): inp_wav_dir = gr.Textbox( label=i18n("*训练集音频文件目录"), # value=r"D:\RVC1006\GPT-SoVITS\raw\xxx", @@ -1549,90 +1499,99 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css ) with gr.Accordion(label="1Aa-" + process_name_1a): - with gr.Row(): - with gr.Row(): - gpu_numbers1a = gr.Textbox( - label=i18n("GPU卡号以-分割，每个卡号一个进程"), - value="%s-%s" % (gpus, gpus), + with gr.Row(equal_height=True): + with gr.Row(equal_height=True): + gpu_numbers1a = gr.Dropdown( + label=i18n("GPU卡号"), + choices=GPU_INDEX_LIST, + value=GPU_INDEX_LIST, interactive=True, + multiselect=True, + allow_custom_value=False, ) - with gr.Row(): + with gr.Row(equal_height=True): bert_pretrained_dir = gr.Textbox( label=i18n("预训练中文BERT模型路径"), value="GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large", interactive=False, lines=2, ) - with gr.Row(): + with gr.Row(equal_height=True): button1a_open = gr.Button( value=process_info(process_name_1a, "open"), variant="primary", visible=True ) button1a_close = gr.Button( value=process_info(process_name_1a, "close"), variant="primary", visible=False ) - with gr.Row(): + with gr.Row(equal_height=True): info1a = gr.Textbox(label=process_info(process_name_1a, "info")) with gr.Accordion(label="1Ab-" + process_name_1b): - with gr.Row(): - with gr.Row(): - gpu_numbers1Ba = gr.Textbox( - label=i18n("GPU卡号以-分割，每个卡号一个进程"), - value="%s-%s" % (gpus, gpus), + with gr.Row(equal_height=True): + with gr.Row(equal_height=True): + gpu_numbers1Ba = gr.Dropdown( + label=i18n("GPU卡号"), + choices=GPU_INDEX_LIST, + value=GPU_INDEX_LIST, interactive=True, + multiselect=True, + allow_custom_value=False, ) - with gr.Row(): + with gr.Row(equal_height=True): cnhubert_base_dir = gr.Textbox( label=i18n("预训练SSL模型路径"), value="GPT_SoVITS/pretrained_models/chinese-hubert-base", interactive=False, lines=2, ) - with gr.Row(): + with gr.Row(equal_height=True): button1b_open = gr.Button( value=process_info(process_name_1b, "open"), variant="primary", visible=True ) button1b_close = gr.Button( value=process_info(process_name_1b, "close"), variant="primary", visible=False ) - with gr.Row(): + with gr.Row(equal_height=True): info1b = gr.Textbox(label=process_info(process_name_1b, "info")) with gr.Accordion(label="1Ac-" + process_name_1c): - with gr.Row(): - with gr.Row(): - gpu_numbers1c = gr.Textbox( - label=i18n("GPU卡号以-分割，每个卡号一个进程"), - value="%s-%s" % (gpus, gpus), + with gr.Row(equal_height=True): + with gr.Row(equal_height=True): + gpu_numbers1c = gr.Dropdown( + label=i18n("GPU卡号"), + choices=GPU_INDEX_LIST, + value=GPU_INDEX_LIST, interactive=True, + multiselect=True, + allow_custom_value=False, ) - with gr.Row(): + with gr.Row(equal_height=True): pretrained_s2G_ = gr.Textbox( label=i18n("预训练SoVITS-G模型路径"), value=pretrained_sovits_name[version], interactive=False, lines=2, ) - with gr.Row(): + with gr.Row(equal_height=True): button1c_open = gr.Button( value=process_info(process_name_1c, "open"), variant="primary", visible=True ) button1c_close = gr.Button( value=process_info(process_name_1c, "close"), variant="primary", visible=False ) - with gr.Row(): + with gr.Row(equal_height=True): info1c = gr.Textbox(label=process_info(process_name_1c, "info")) with gr.Accordion(label="1Aabc-" + process_name_1abc): - with gr.Row(): - with gr.Row(): + with gr.Row(equal_height=True): + with gr.Row(equal_height=True): button1abc_open = gr.Button( value=process_info(process_name_1abc, "open"), variant="primary", visible=True ) button1abc_close = gr.Button( value=process_info(process_name_1abc, "close"), variant="primary", visible=False ) - with gr.Row(): + with gr.Row(equal_height=True): info1abc = gr.Textbox(label=process_info(process_name_1abc, "info")) pretrained_s2G.change(sync, [pretrained_s2G], [pretrained_s2G_]) @@ -1656,19 +1615,13 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css alpha, n_process, ], - [slicer_info, open_slicer_button, close_slicer_button, asr_inp_dir, denoise_input_dir, inp_wav_dir], + [slicer_info, open_slicer_button, close_slicer_button, asr_inp_dir, inp_wav_dir], ) close_slicer_button.click(close_slice, [], [slicer_info, open_slicer_button, close_slicer_button]) - open_denoise_button.click( - open_denoise, - [denoise_input_dir, denoise_output_dir], - [denoise_info, open_denoise_button, close_denoise_button, asr_inp_dir, inp_wav_dir], - ) - close_denoise_button.click(close_denoise, [], [denoise_info, open_denoise_button, close_denoise_button]) button1a_open.click( open1a, - [inp_text, inp_wav_dir, exp_name, gpu_numbers1a, bert_pretrained_dir], + [inp_text, inp_wav_dir, exp_name, gpu_numbers1a, bert_pretrained_dir, version_checkbox, n_processes], [info1a, button1a_open, button1a_close], ) button1a_close.click(close1a, [], [info1a, button1a_open, button1a_close]) @@ -1704,149 +1657,152 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css with gr.TabItem("1B-" + i18n("微调训练")): with gr.Accordion(label="1Ba-" + i18n("SoVITS 训练: 模型权重文件在 SoVITS_weights/")): - with gr.Row(): + with gr.Row(equal_height=True): + batch_size = gr.Slider( + minimum=1, + maximum=default_max_batch_size, + step=1, + label=i18n("每张显卡的batch_size"), + value=default_batch_size, + interactive=True, + ) + total_epoch = gr.Slider( + minimum=1, + maximum=max_sovits_epoch, + step=1, + label=i18n("总训练轮数total_epoch，不建议太高"), + value=default_sovits_epoch, + interactive=True, + ) + with gr.Column(scale=2): + if_save_latest = gr.Checkbox( + label=i18n("是否仅保存最新的权重文件以节省硬盘空间"), + value=True, + interactive=True, + show_label=True, + ) + if_save_every_weights = gr.Checkbox( + label=i18n("是否在每次保存时间点将最终小模型保存至weights文件夹"), + value=True, + interactive=True, + show_label=True, + ) + if_grad_ckpt = gr.Checkbox( + label="v3是否开启梯度检查点节省显存占用", + value=False, + interactive=True if version in v3v4set else False, + show_label=True, + visible=False, + ) # 只有V3s2可以用 + with gr.Row(equal_height=True): + text_low_lr_rate = gr.Slider( + minimum=0.2, + maximum=0.6, + step=0.05, + label=i18n("文本模块学习率权重"), + value=0.4, + visible=True if version not in v3v4set else False, + ) # v3v4 not need + lora_rank = gr.Radio( + label=i18n("LoRA秩"), + value="32", + choices=["16", "32", "64", "128"], + visible=True if version in v3v4set else False, + ) # v1v2 not need + save_every_epoch = gr.Slider( + minimum=1, + maximum=max_sovits_save_every_epoch, + step=1, + label=i18n("保存频率save_every_epoch"), + value=default_sovits_save_every_epoch, + interactive=True, + ) + with gr.Column(scale=3): + gpu_numbers1Ba = gr.Dropdown( + label=i18n("GPU卡号"), + choices=GPU_INDEX_LIST, + value=GPU_INDEX_LIST, + interactive=True, + multiselect=True, + allow_custom_value=False, + ) + with gr.Row(equal_height=True): with gr.Column(): - with gr.Row(): - batch_size = gr.Slider( - minimum=1, - maximum=default_max_batch_size, - step=1, - label=i18n("每张显卡的batch_size"), - value=default_batch_size, - interactive=True, - ) - total_epoch = gr.Slider( - minimum=1, - maximum=max_sovits_epoch, - step=1, - label=i18n("总训练轮数total_epoch，不建议太高"), - value=default_sovits_epoch, - interactive=True, - ) - with gr.Row(): - text_low_lr_rate = gr.Slider( - minimum=0.2, - maximum=0.6, - step=0.05, - label=i18n("文本模块学习率权重"), - value=0.4, - visible=True if version not in v3v4set else False, - ) # v3v4 not need - lora_rank = gr.Radio( - label=i18n("LoRA秩"), - value="32", - choices=["16", "32", "64", "128"], - visible=True if version in v3v4set else False, - ) # v1v2 not need - save_every_epoch = gr.Slider( - minimum=1, - maximum=max_sovits_save_every_epoch, - step=1, - label=i18n("保存频率save_every_epoch"), - value=default_sovits_save_every_epoch, - interactive=True, - ) - with gr.Column(): - with gr.Column(): - if_save_latest = gr.Checkbox( - label=i18n("是否仅保存最新的权重文件以节省硬盘空间"), - value=True, - interactive=True, - show_label=True, - ) - if_save_every_weights = gr.Checkbox( - label=i18n("是否在每次保存时间点将最终小模型保存至weights文件夹"), - value=True, - interactive=True, - show_label=True, - ) - if_grad_ckpt = gr.Checkbox( - label="v3是否开启梯度检查点节省显存占用", - value=False, - interactive=True if version in v3v4set else False, - show_label=True, - visible=False, - ) # 只有V3s2可以用 - with gr.Row(): - gpu_numbers1Ba = gr.Textbox( - label=i18n("GPU卡号以-分割，每个卡号一个进程"), - value="%s" % (gpus), - interactive=True, - ) - with gr.Row(): - with gr.Row(): button1Ba_open = gr.Button( value=process_info(process_name_sovits, "open"), variant="primary", visible=True ) button1Ba_close = gr.Button( value=process_info(process_name_sovits, "close"), variant="primary", visible=False ) - with gr.Row(): + with gr.Column(): info1Ba = gr.Textbox(label=process_info(process_name_sovits, "info")) with gr.Accordion(label="1Bb-" + i18n("GPT 训练: 模型权重文件在 GPT_weights/")): - with gr.Row(): + with gr.Row(equal_height=True): + batch_size1Bb = gr.Slider( + minimum=1, + maximum=40, + step=1, + label=i18n("每张显卡的batch_size"), + value=default_batch_size_s1, + interactive=True, + ) + total_epoch1Bb = gr.Slider( + minimum=2, + maximum=50, + step=1, + label=i18n("总训练轮数total_epoch"), + value=15, + interactive=True, + ) + with gr.Column(scale=2): + if_save_latest1Bb = gr.Checkbox( + label=i18n("是否仅保存最新的权重文件以节省硬盘空间"), + value=True, + interactive=True, + show_label=True, + ) + if_save_every_weights1Bb = gr.Checkbox( + label=i18n("是否在每次保存时间点将最终小模型保存至weights文件夹"), + value=True, + interactive=True, + show_label=True, + ) + with gr.Row(equal_height=True): + # with gr.Column(): + save_every_epoch1Bb = gr.Slider( + minimum=1, + maximum=50, + step=1, + label=i18n("保存频率save_every_epoch"), + value=5, + interactive=True, + ) + # with gr.Column(): + if_dpo = gr.Checkbox( + label=i18n("是否开启DPO训练选项(实验性)"), + value=False, + interactive=True, + show_label=True, + ) + with gr.Column(scale=2): + gpu_numbers1Bb = gr.Dropdown( + label=i18n("GPU卡号"), + choices=GPU_INDEX_LIST, + value=GPU_INDEX_LIST, + interactive=True, + multiselect=True, + allow_custom_value=False, + ) + with gr.Row(equal_height=True): with gr.Column(): - with gr.Row(): - batch_size1Bb = gr.Slider( - minimum=1, - maximum=40, - step=1, - label=i18n("每张显卡的batch_size"), - value=default_batch_size_s1, - interactive=True, + with gr.Row(equal_height=True): + button1Bb_open = gr.Button( + value=process_info(process_name_gpt, "open"), variant="primary", visible=True ) - total_epoch1Bb = gr.Slider( - minimum=2, - maximum=50, - step=1, - label=i18n("总训练轮数total_epoch"), - value=15, - interactive=True, - ) - with gr.Row(): - save_every_epoch1Bb = gr.Slider( - minimum=1, - maximum=50, - step=1, - label=i18n("保存频率save_every_epoch"), - value=5, - interactive=True, - ) - if_dpo = gr.Checkbox( - label=i18n("是否开启DPO训练选项(实验性)"), - value=False, - interactive=True, - show_label=True, + button1Bb_close = gr.Button( + value=process_info(process_name_gpt, "close"), variant="primary", visible=False ) with gr.Column(): - with gr.Column(): - if_save_latest1Bb = gr.Checkbox( - label=i18n("是否仅保存最新的权重文件以节省硬盘空间"), - value=True, - interactive=True, - show_label=True, - ) - if_save_every_weights1Bb = gr.Checkbox( - label=i18n("是否在每次保存时间点将最终小模型保存至weights文件夹"), - value=True, - interactive=True, - show_label=True, - ) - with gr.Row(): - gpu_numbers1Bb = gr.Textbox( - label=i18n("GPU卡号以-分割，每个卡号一个进程"), - value="%s" % (gpus), - interactive=True, - ) - with gr.Row(): - with gr.Row(): - button1Bb_open = gr.Button( - value=process_info(process_name_gpt, "open"), variant="primary", visible=True - ) - button1Bb_close = gr.Button( - value=process_info(process_name_gpt, "close"), variant="primary", visible=False - ) - with gr.Row(): info1Bb = gr.Textbox(label=process_info(process_name_gpt, "info")) button1Ba_close.click(close1Ba, [], [info1Ba, button1Ba_open, button1Ba_close]) @@ -1858,41 +1814,62 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css "选择训练完存放在SoVITS_weights和GPT_weights下的模型。默认的几个是底模，体验5秒Zero Shot TTS不训练推理用。" ) ) - with gr.Row(): + with gr.Row(equal_height=True): with gr.Column(scale=2): - with gr.Row(): - GPT_dropdown = gr.Dropdown( - label=i18n("GPT模型列表"), - choices=GPT_names, - value=GPT_names[-1], - interactive=True, - ) - SoVITS_dropdown = gr.Dropdown( - label=i18n("SoVITS模型列表"), - choices=SoVITS_names, - value=SoVITS_names[0], - interactive=True, - ) + with gr.Row(equal_height=True): + with gr.Column(): + GPT_dropdown = gr.Dropdown( + label=i18n("GPT模型列表"), + choices=GPT_names, + value=GPT_names[-1], + interactive=True, + ) + with gr.Column(): + SoVITS_dropdown = gr.Dropdown( + label=i18n("SoVITS模型列表"), + choices=SoVITS_names, + value=SoVITS_names[0], + interactive=True, + ) with gr.Column(scale=2): - with gr.Row(): - gpu_number_1C = gr.Textbox( - label=i18n("GPU卡号,只能填1个整数"), value=gpus, interactive=True + with gr.Row(equal_height=True): + gpu_number_1C = gr.Dropdown( + label=i18n("GPU卡号"), + choices=GPU_INDEX_LIST, + value=infer_device.index, + interactive=True, + multiselect=True, + allow_custom_value=False, ) refresh_button = gr.Button(i18n("刷新模型路径"), variant="primary") refresh_button.click(fn=change_choices, inputs=[], outputs=[SoVITS_dropdown, GPT_dropdown]) with gr.Row(equal_height=True): - with gr.Row(): - batched_infer_enabled = gr.Checkbox( - label=i18n("启用并行推理版本"), value=False, interactive=True, show_label=True - ) + with gr.Row(equal_height=True): + with gr.Column(): + batched_infer_enabled = gr.Checkbox( + label=i18n("启用并行推理版本"), value=False, interactive=True, show_label=True + ) + with gr.Column(): + backends_dropdown = gr.Dropdown( + choices=backends_gradio, + label=i18n("推理后端"), + value=backends_gradio[-1][-1], + interactive=True, + ) + with gr.Row(equal_height=True): + tts_info = gr.Textbox(label=process_info(process_name_tts, "info")) open_tts = gr.Button( value=process_info(process_name_tts, "open"), variant="primary", visible=True ) close_tts = gr.Button( value=process_info(process_name_tts, "close"), variant="primary", visible=False ) - with gr.Column(): - tts_info = gr.Textbox(label=process_info(process_name_tts, "info"), scale=2) + + batched_infer_enabled.change( + changeBackend, + [batched_infer_enabled], + [backends_dropdown], + ) open_tts.click( change_tts_inference, [ @@ -1902,6 +1879,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css GPT_dropdown, SoVITS_dropdown, batched_infer_enabled, + backends_dropdown, ], [tts_info, open_tts, close_tts], ) @@ -1914,6 +1892,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css GPT_dropdown, SoVITS_dropdown, batched_infer_enabled, + backends_dropdown, ], [tts_info, open_tts, close_tts], )