Merge branch 'RVC-Boss:main' into dev

GPT_SoVITS/AR/models/t2s_model.py

@@ -1,8 +1,11 @@
 # modified from https://github.com/yangdongchao/SoundStorm/blob/master/soundstorm/s1/AR/models/t2s_model.py
 # reference: https://github.com/lifeiteng/vall-e
 import torch
-from tqdm import tqdm
+import random
+import numpy as np
 
+from tqdm import tqdm
+from typing import List
 from AR.models.utils import make_pad_mask
 from AR.models.utils import (
     topk_sampling,
@@ -35,6 +38,139 @@ default_config = {
 }
 
 
+@torch.jit.script
+class T2SMLP:
+    def __init__(self, w1, b1, w2, b2):
+        self.w1 = w1
+        self.b1 = b1
+        self.w2 = w2
+        self.b2 = b2
+
+    def forward(self, x):
+        x = F.relu(F.linear(x, self.w1, self.b1))
+        x = F.linear(x, self.w2, self.b2)
+        return x
+
+
+@torch.jit.script
+class T2SBlock:
+    def __init__(
+            self,
+            num_heads,
+            hidden_dim: int,
+            mlp: T2SMLP,
+            qkv_w,
+            qkv_b,
+            out_w,
+            out_b,
+            norm_w1,
+            norm_b1,
+            norm_eps1,
+            norm_w2,
+            norm_b2,
+            norm_eps2,
+    ):
+        self.num_heads = num_heads
+        self.mlp = mlp
+        self.hidden_dim: int = hidden_dim
+        self.qkv_w = qkv_w
+        self.qkv_b = qkv_b
+        self.out_w = out_w
+        self.out_b = out_b
+        self.norm_w1 = norm_w1
+        self.norm_b1 = norm_b1
+        self.norm_eps1 = norm_eps1
+        self.norm_w2 = norm_w2
+        self.norm_b2 = norm_b2
+        self.norm_eps2 = norm_eps2
+
+    def process_prompt(self, x, attn_mask: torch.Tensor):
+        q, k, v = F.linear(x, self.qkv_w, self.qkv_b).chunk(3, dim=-1)
+
+        batch_size = q.shape[0]
+        q_len = q.shape[1]
+        kv_len = k.shape[1]
+
+        k_cache = k
+        v_cache = v
+
+        q = q.view(batch_size, q_len, self.num_heads, -1).transpose(1, 2)
+        k = k_cache.view(batch_size, kv_len, self.num_heads, -1).transpose(1, 2)
+        v = v_cache.view(batch_size, kv_len, self.num_heads, -1).transpose(1, 2)
+
+        attn = F.scaled_dot_product_attention(q, k, v, ~attn_mask)
+
+        attn = attn.permute(2, 0, 1, 3).reshape(batch_size, -1, self.hidden_dim)
+        attn = F.linear(attn, self.out_w, self.out_b)
+
+        x = F.layer_norm(
+            x + attn, [self.hidden_dim], self.norm_w1, self.norm_b1, self.norm_eps1
+        )
+        x = F.layer_norm(
+            x + self.mlp.forward(x),
+            [self.hidden_dim],
+            self.norm_w2,
+            self.norm_b2,
+            self.norm_eps2,
+        )
+        return x, k_cache, v_cache
+
+    def decode_next_token(self, x, k_cache, v_cache):
+        q, k, v = F.linear(x, self.qkv_w, self.qkv_b).chunk(3, dim=-1)
+
+        k_cache = torch.cat([k_cache, k], dim=1)
+        v_cache = torch.cat([v_cache, v], dim=1)
+        kv_len = k_cache.shape[1]
+
+        batch_size = q.shape[0]
+        q_len = q.shape[1]
+
+        q = q.view(batch_size, q_len, self.num_heads, -1).transpose(1, 2)
+        k = k_cache.view(batch_size, kv_len, self.num_heads, -1).transpose(1, 2)
+        v = v_cache.view(batch_size, kv_len, self.num_heads, -1).transpose(1, 2)
+
+        attn = F.scaled_dot_product_attention(q, k, v)
+
+        attn = attn.permute(2, 0, 1, 3).reshape(batch_size, -1, self.hidden_dim)
+        attn = F.linear(attn, self.out_w, self.out_b)
+
+        x = F.layer_norm(
+            x + attn, [self.hidden_dim], self.norm_w1, self.norm_b1, self.norm_eps1
+        )
+        x = F.layer_norm(
+            x + self.mlp.forward(x),
+            [self.hidden_dim],
+            self.norm_w2,
+            self.norm_b2,
+            self.norm_eps2,
+        )
+        return x, k_cache, v_cache
+
+
+@torch.jit.script
+class T2STransformer:
+    def __init__(self, num_blocks: int, blocks: List[T2SBlock]):
+        self.num_blocks: int = num_blocks
+        self.blocks = blocks
+
+    def process_prompt(
+            self, x, attn_mask: torch.Tensor):
+        k_cache: List[torch.Tensor] = []
+        v_cache: List[torch.Tensor] = []
+        for i in range(self.num_blocks):
+            x, k_cache_, v_cache_ = self.blocks[i].process_prompt(x, attn_mask)
+            k_cache.append(k_cache_)
+            v_cache.append(v_cache_)
+        return x, k_cache, v_cache
+
+    def decode_next_token(
+            self, x, k_cache: List[torch.Tensor], v_cache: List[torch.Tensor]
+    ):
+        for i in range(self.num_blocks):
+            x, k_cache[i], v_cache[i] = self.blocks[i].decode_next_token(x, k_cache[i], v_cache[i])
+        return x, k_cache, v_cache
+
+
 class Text2SemanticDecoder(nn.Module):
     def __init__(self, config, norm_first=False, top_k=3):
         super(Text2SemanticDecoder, self).__init__()
@@ -89,6 +225,37 @@ class Text2SemanticDecoder(nn.Module):
             ignore_index=self.EOS,
         )
 
+        blocks = []
+
+        for i in range(self.num_layers):
+            layer = self.h.layers[i]
+            t2smlp = T2SMLP(
+                layer.linear1.weight,
+                layer.linear1.bias,
+                layer.linear2.weight,
+                layer.linear2.bias
+            )
+            # (layer.self_attn.in_proj_weight, layer.self_attn.in_proj_bias)
+            block = T2SBlock(
+                self.num_head,
+                self.model_dim,
+                t2smlp,
+                layer.self_attn.in_proj_weight,
+                layer.self_attn.in_proj_bias,
+                layer.self_attn.out_proj.weight,
+                layer.self_attn.out_proj.bias,
+                layer.norm1.weight,
+                layer.norm1.bias,
+                layer.norm1.eps,
+                layer.norm2.weight,
+                layer.norm2.bias,
+                layer.norm2.eps
+            )
+
+            blocks.append(block)
+
+        self.t2s_transformer = T2STransformer(self.num_layers, blocks)
+
     def make_input_data(self, x, x_lens, y, y_lens, bert_feature):
         x = self.ar_text_embedding(x)
         x = x + self.bert_proj(bert_feature.transpose(1, 2))
@@ -343,17 +510,9 @@ class Text2SemanticDecoder(nn.Module):
         x_attn_mask = torch.zeros((x_len, x_len), dtype=torch.bool)
         stop = False
         # print(1111111,self.num_layers)
-        cache = {
+
-            "all_stage": self.num_layers,
+        k_cache = None
-            "k": [None] * self.num_layers,  ###根据配置自己手写
+        v_cache = None
-            "v": [None] * self.num_layers,
-            # "xy_pos":None,##y_pos位置编码每次都不一样的没法缓存，每次都要重新拼xy_pos.主要还是写法原因，其实是可以历史统一一样的，但也没啥计算量就不管了
-            "y_emb": None,  ##只需要对最新的samples求emb，再拼历史的就行
-            # "logits":None,###原版就已经只对结尾求再拼接了，不用管
-            # "xy_dec":None,###不需要，本来只需要最后一个做logits
-            "first_infer": 1,
-            "stage": 0,
-        }
         ###################  first step ##########################
         if y is not None:
             y_emb = self.ar_audio_embedding(y)
@@ -361,7 +520,6 @@ class Text2SemanticDecoder(nn.Module):
             prefix_len = y.shape[1]
             y_pos = self.ar_audio_position(y_emb)
             xy_pos = torch.concat([x, y_pos], dim=1)
-            cache["y_emb"] = y_emb
             ref_free = False
         else:
             y_emb = None
@@ -370,6 +528,7 @@ class Text2SemanticDecoder(nn.Module):
             y_pos = None
             xy_pos = x
             y = torch.zeros(x.shape[0], 0, dtype=torch.int, device=x.device)
+            prompts = y
             ref_free = True
 
         x_attn_mask_pad = F.pad(
@@ -386,21 +545,23 @@ class Text2SemanticDecoder(nn.Module):
             x.device
         )
 
-
         for idx in tqdm(range(1500)):
+            if xy_attn_mask is not None:
+                xy_dec, k_cache, v_cache = self.t2s_transformer.process_prompt(xy_pos, xy_attn_mask)
+            else:
+                xy_dec, k_cache, v_cache = self.t2s_transformer.decode_next_token(xy_pos, k_cache, v_cache)
 
-            xy_dec, _ = self.h((xy_pos, None), mask=xy_attn_mask, cache=cache)
             logits = self.ar_predict_layer(
                 xy_dec[:, -1]
-            )  ##不用改，如果用了cache的默认就是只有一帧，取最后一帧一样的
+            )
-            # samples = topk_sampling(logits, top_k=top_k, top_p=1.0, temperature=temperature)
+
-            if(idx==0):###第一次跑不能EOS否则没有了
+            if idx == 0:
-                logits = logits[:, :-1]  ###刨除1024终止符号的概率
+                xy_attn_mask = None
+                logits = logits[:, :-1]
             samples = sample(
                 logits[0], y, top_k=top_k, top_p=top_p, repetition_penalty=1.35, temperature=temperature
             )[0].unsqueeze(0)
-            # 本次生成的 semantic_ids 和之前的 y 构成新的 y
+
-            # print(samples.shape)#[1,1]#第一个1是bs
             y = torch.concat([y, samples], dim=1)
 
             if early_stop_num != -1 and (y.shape[1] - prefix_len) > early_stop_num:
@@ -408,12 +569,8 @@ class Text2SemanticDecoder(nn.Module):
                 stop = True
 
             if torch.argmax(logits, dim=-1)[0] == self.EOS or samples[0, 0] == self.EOS:
-                # print(torch.argmax(logits, dim=-1)[0] == self.EOS, samples[0, 0] == self.EOS)
                 stop = True
             if stop:
-                # if prompts.shape[1] == y.shape[1]:
-                #     y = torch.concat([y, torch.zeros_like(samples)], dim=1)
-                #     print("bad zero prediction")
                 if y.shape[1] == 0:
                     y = torch.concat([y, torch.zeros_like(samples)], dim=1)
                     print("bad zero prediction")
@@ -421,28 +578,9 @@ class Text2SemanticDecoder(nn.Module):
                 break
 
             ####################### update next step ###################################
-            cache["first_infer"] = 0
-            if cache["y_emb"] is not None:
-                y_emb = torch.cat(
-                    [cache["y_emb"], self.ar_audio_embedding(y[:, -1:])], dim = 1
-                )
-                cache["y_emb"] = y_emb
-                y_pos = self.ar_audio_position(y_emb)
-                xy_pos = y_pos[:, -1:]
-            else:
             y_emb = self.ar_audio_embedding(y[:, -1:])
-                cache["y_emb"] = y_emb
+            xy_pos = y_emb * self.ar_audio_position.x_scale + self.ar_audio_position.alpha * self.ar_audio_position.pe[:, y_len + idx].to(dtype=y_emb.dtype,device=y_emb.device)
-                y_pos = self.ar_audio_position(y_emb)
-                xy_pos = y_pos
-            y_len = y_pos.shape[1]
 
-            ###最右边一列（是错的）
-            # xy_attn_mask=torch.ones((1, x_len+y_len), dtype=torch.bool,device=xy_pos.device)
-            # xy_attn_mask[:,-1]=False
-            ###最下面一行（是对的）
-            xy_attn_mask = torch.zeros(
-                (1, x_len + y_len), dtype=torch.bool, device=xy_pos.device
-            )
         if ref_free:
             return y[:, :-1], 0
         return y[:, :-1], idx - 1

GPT_SoVITS/inference_webui.py

@@ -65,7 +65,7 @@ from text import cleaned_text_to_sequence
 from text.cleaner import clean_text
 from time import time as ttime
 from module.mel_processing import spectrogram_torch
-from my_utils import load_audio
+from tools.my_utils import load_audio
 from tools.i18n.i18n import I18nAuto
 
 i18n = I18nAuto()
@@ -331,6 +331,7 @@ def get_tts_wav(ref_wav_path, prompt_text, prompt_language, text, text_language,
         int(hps.data.sampling_rate * 0.3),
         dtype=np.float16 if is_half == True else np.float32,
     )
+    if not ref_free:
         with torch.no_grad():
             wav16k, sr = librosa.load(ref_wav_path, sr=16000)
             if (wav16k.shape[0] > 160000 or wav16k.shape[0] < 48000):
@@ -350,8 +351,9 @@ def get_tts_wav(ref_wav_path, prompt_text, prompt_language, text, text_language,
                 1, 2
             )  # .float()
             codes = vq_model.extract_latent(ssl_content)
-   
             prompt_semantic = codes[0, 0]
+            prompt = prompt_semantic.unsqueeze(0).to(device)
+
     t1 = ttime()
 
     if (how_to_cut == i18n("凑四句一切")):
@@ -391,7 +393,7 @@ def get_tts_wav(ref_wav_path, prompt_text, prompt_language, text, text_language,
 
         bert = bert.to(device).unsqueeze(0)
         all_phoneme_len = torch.tensor([all_phoneme_ids.shape[-1]]).to(device)
-        prompt = prompt_semantic.unsqueeze(0).to(device)
+
         t2 = ttime()
         with torch.no_grad():
             # pred_semantic = t2s_model.model.infer(
@@ -510,16 +512,26 @@ def cut4(inp):
 
 # contributed by https://github.com/AI-Hobbyist/GPT-SoVITS/blob/main/GPT_SoVITS/inference_webui.py
 def cut5(inp):
-    # if not re.search(r'[^\w\s]', inp[-1]):
-    # inp += '。'
     inp = inp.strip("\n")
-    punds = r'[,.;?!、，。？！;：…]'
+    punds = {',', '.', ';', '?', '!', '、', '，', '。', '？', '！', ';', '：', '…'}
-    items = re.split(f'({punds})', inp)
+    mergeitems = []
-    mergeitems = ["".join(group) for group in zip(items[::2], items[1::2])]
+    items = []
-    # 在句子不存在符号或句尾无符号的时候保证文本完整
+
-    if len(items)%2 == 1:
+    for i, char in enumerate(inp):
-        mergeitems.append(items[-1])
+        if char in punds:
-    opt = [item for item in mergeitems if not set(item).issubset(punctuation)]
+            if char == '.' and i > 0 and i < len(inp) - 1 and inp[i - 1].isdigit() and inp[i + 1].isdigit():
+                items.append(char)
+            else:
+                items.append(char)
+                mergeitems.append("".join(items))
+                items = []
+        else:
+            items.append(char)
+
+    if items:
+        mergeitems.append("".join(items))
+
+    opt = [item for item in mergeitems if not set(item).issubset(punds)]
     return "\n".join(opt)
 
 

GPT_SoVITS/module/data_utils.py

@@ -17,7 +17,7 @@ from functools import lru_cache
 import requests
 from scipy.io import wavfile
 from io import BytesIO
-from my_utils import load_audio
+from tools.my_utils import load_audio
 
 # ZeroDivisionError fixed by Tybost (https://github.com/RVC-Boss/GPT-SoVITS/issues/79)
 class TextAudioSpeakerLoader(torch.utils.data.Dataset):

GPT_SoVITS/my_utils.py

@@ -1,21 +0,0 @@
-import ffmpeg
-import numpy as np
-
-
-def load_audio(file, sr):
-    try:
-        # https://github.com/openai/whisper/blob/main/whisper/audio.py#L26
-        # This launches a subprocess to decode audio while down-mixing and resampling as necessary.
-        # Requires the ffmpeg CLI and `ffmpeg-python` package to be installed.
-        file = (
-            file.strip(" ").strip('"').strip("\n").strip('"').strip(" ")
-        )  # 防止小白拷路径头尾带了空格和"和回车
-        out, _ = (
-            ffmpeg.input(file, threads=0)
-            .output("-", format="f32le", acodec="pcm_f32le", ac=1, ar=sr)
-            .run(cmd=["ffmpeg", "-nostdin"], capture_stdout=True, capture_stderr=True)
-        )
-    except Exception as e:
-        raise RuntimeError(f"Failed to load audio: {e}")
-
-    return np.frombuffer(out, np.float32).flatten()

GPT_SoVITS/onnx_export.py

@@ -9,7 +9,7 @@ cnhubert.cnhubert_base_path=cnhubert_base_path
 ssl_model = cnhubert.get_model()
 from text import cleaned_text_to_sequence
 import soundfile
-from my_utils import load_audio
+from tools.my_utils import load_audio
 import os
 import json
 

GPT_SoVITS/prepare_datasets/2-get-hubert-wav32k.py

@@ -17,7 +17,7 @@ from scipy.io import wavfile
 import librosa,torch
 now_dir = os.getcwd()
 sys.path.append(now_dir)
-from my_utils import load_audio
+from tools.my_utils import load_audio
 
 # from config import cnhubert_base_path
 # cnhubert.cnhubert_base_path=cnhubert_base_path

GPT_SoVITS/s1_train.py

@@ -79,6 +79,8 @@ class my_model_ckpt(ModelCheckpoint):
                     to_save_od["config"] = self.config
                     to_save_od["info"] = "GPT-e%s" % (trainer.current_epoch + 1)
                     # torch.save(
+                    # print(os.environ)
+                    if(os.environ.get("LOCAL_RANK","0")=="0"):
                         my_save(
                             to_save_od,
                             "%s/%s-e%s.ckpt"

GPT_SoVITS/text/cleaner.py

@@ -22,6 +22,11 @@ def clean_text(text, language):
         phones, word2ph = language_module.g2p(norm_text)
         assert len(phones) == sum(word2ph)
         assert len(norm_text) == len(word2ph)
+    elif language == "en":
+        phones = language_module.g2p(norm_text)
+        if len(phones) < 4:
+            phones = [','] * (4 - len(phones)) + phones
+        word2ph = None
     else:
         phones = language_module.g2p(norm_text)
         word2ph = None

GPT_SoVITS/text/engdict-hot.rep

@@ -1,2 +1,3 @@
 CHATGPT CH AE1 T JH IY1 P IY1 T IY1
 JSON JH EY1 S AH0 N
+CONDA K AA1 N D AH0

api.py

@@ -143,7 +143,7 @@ from AR.models.t2s_lightning_module import Text2SemanticLightningModule
 from text import cleaned_text_to_sequence
 from text.cleaner import clean_text
 from module.mel_processing import spectrogram_torch
-from my_utils import load_audio
+from tools.my_utils import load_audio
 import config as global_config
 import logging
 import subprocess

docs/cn/Changelog_CN.md

@@ -183,13 +183,36 @@
 
 4-修复了webui的GPT中文微调没读到bert导致和推理不一致，训练太多可能效果还会变差的问题。如果大量数据微调的建议重新微调模型得到质量优化 [#99f09c8](https://github.com/RVC-Boss/GPT-SoVITS/commit/99f09c8bdc155c1f4272b511940717705509582a)
 
+### 20240706
+
+小问题修复：
+
+1-修正CPU推理默认bs小数 https://github.com/RVC-Boss/GPT-SoVITS/commit/db50670598f0236613eefa6f2d5a23a271d82041
+
+2-修复降噪、asr中途遇到异常跳出所有需处理的音频文件的问题 https://github.com/RVC-Boss/GPT-SoVITS/pull/1258 https://github.com/RVC-Boss/GPT-SoVITS/pull/1265 https://github.com/RVC-Boss/GPT-SoVITS/pull/1267
+
+3-修复按标点符号切分时小数会被切分 https://github.com/RVC-Boss/GPT-SoVITS/pull/1253
+
+4-多卡训练多进程保存逻辑修复
+
+https://github.com/RVC-Boss/GPT-SoVITS/commit/a208698e775155efc95b187b746d153d0f2847ca
+
+5-移除冗余my_utils https://github.com/RVC-Boss/GPT-SoVITS/pull/1251
+
+重点：
+
+6-倍速推理代码经过验证后推理效果和base完全一致，合并进main。使用的代码：https://github.com/RVC-Boss/GPT-SoVITS/pull/672。支持无参考文本模式也倍速。
+
+后面会逐渐验证快速推理分支的推理改动的一致性
+
+
 
 todolist：
 
 1-中文多音字推理优化(有没有人来测试的，欢迎把测试结果写在pr评论区里) https://github.com/RVC-Boss/GPT-SoVITS/pull/488
 （v2底模训练已经合了，下个版本发布就要合了）
 
-2-正在尝试解决低音质参考音频导致音质较差的问题，v2再试试如果能解决就发了，节点暂定高考后吧
+2-正在尝试解决低音质参考音频导致音质较差的问题，v2再试试如果能解决就发了，节点暂定7月吧
 
 
 

tools/asr/fasterwhisper_asr.py

@@ -78,7 +78,7 @@ def execute_asr(input_folder, output_folder, model_size, language, precision):
                     text += segment.text
             output.append(f"{file_path}|{output_file_name}|{info.language.upper()}|{text}")
         except:
-            return print(traceback.format_exc())
+            print(traceback.format_exc())
     
     output_folder = output_folder or "output/asr_opt"
     os.makedirs(output_folder, exist_ok=True)

tools/cmd-denoise.py

@@ -1,4 +1,5 @@
 import os,argparse
+import traceback
 
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
@@ -12,7 +13,10 @@ def execute_denoise(input_folder,output_folder):
     # print(input_folder)
     # print(list(os.listdir(input_folder).sort()))
     for name in tqdm(os.listdir(input_folder)):
+        try:
             ans("%s/%s"%(input_folder,name),output_path='%s/%s'%(output_folder,name))
+        except:
+            traceback.print_exc()
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()

tools/slice_audio.py

@@ -3,7 +3,7 @@ import traceback
 from scipy.io import wavfile
 # parent_directory = os.path.dirname(os.path.abspath(__file__))
 # sys.path.append(parent_directory)
-from my_utils import load_audio
+from tools.my_utils import load_audio
 from slicer2 import Slicer
 
 def slice(inp,opt_root,threshold,min_length,min_interval,hop_size,max_sil_kept,_max,alpha,i_part,all_part):