Merge 47bc83bd5fc171bde797e10c64cbc31617d21764 into ee4a466f79b4f643251aa2f873f541f85df11d91

* Added the instruction for AMD GPU in English

* Added the instruction for AMD GPU in Chinese

* Update install.sh, now it will check wether user are using cuda or rocm

* 恢复原来的readme，已经更新了install.sh

* 恢复中文readme

* 将n卡的判断条件由nvcc改成nvidia-smi

GPT_SoVITS/AR/modules/patched_mha_with_cache.py

@@ -12,33 +12,33 @@ import torch
 
 
 def multi_head_attention_forward_patched(
-    query: Tensor,
-    key: Tensor,
-    value: Tensor,
-    embed_dim_to_check: int,
-    num_heads: int,
-    in_proj_weight: Optional[Tensor],
-    in_proj_bias: Optional[Tensor],
-    bias_k: Optional[Tensor],
-    bias_v: Optional[Tensor],
-    add_zero_attn: bool,
+    query,
+    key,
+    value,
+    embed_dim_to_check,
+    num_heads,
+    in_proj_weight,
+    in_proj_bias,
+    bias_k,
+    bias_v,
+    add_zero_attn,
     dropout_p: float,
-    out_proj_weight: Tensor,
-    out_proj_bias: Optional[Tensor],
-    training: bool = True,
-    key_padding_mask: Optional[Tensor] = None,
-    need_weights: bool = True,
-    attn_mask: Optional[Tensor] = None,
-    use_separate_proj_weight: bool = False,
-    q_proj_weight: Optional[Tensor] = None,
-    k_proj_weight: Optional[Tensor] = None,
-    v_proj_weight: Optional[Tensor] = None,
-    static_k: Optional[Tensor] = None,
-    static_v: Optional[Tensor] = None,
-    average_attn_weights: bool = True,
-    is_causal: bool = False,
+    out_proj_weight,
+    out_proj_bias,
+    training = True,
+    key_padding_mask = None,
+    need_weights = True,
+    attn_mask = None,
+    use_separate_proj_weight = False,
+    q_proj_weight = None,
+    k_proj_weight = None,
+    v_proj_weight = None,
+    static_k = None,
+    static_v = None,
+    average_attn_weights = True,
+    is_causal = False,
     cache=None,
-) -> Tuple[Tensor, Optional[Tensor]]:
+):
     r"""
     Args:
         query, key, value: map a query and a set of key-value pairs to an output.

GPT_SoVITS/TTS_infer_pack/TextPreprocessor.py

@@ -1,5 +1,6 @@
 
 import os, sys
+import threading
 
 from tqdm import tqdm
 now_dir = os.getcwd()
@@ -54,6 +55,7 @@ class TextPreprocessor:
         self.bert_model = bert_model
         self.tokenizer = tokenizer
         self.device = device
+        self.bert_lock = threading.RLock()
 
     def preprocess(self, text:str, lang:str, text_split_method:str, version:str="v2")->List[Dict]:
         print(f'############ {i18n("切分文本")} ############')
@@ -117,70 +119,71 @@ class TextPreprocessor:
         return self.get_phones_and_bert(text, language, version)
 
     def get_phones_and_bert(self, text:str, language:str, version:str, final:bool=False):
-        if language in {"en", "all_zh", "all_ja", "all_ko", "all_yue"}:
-            # language = language.replace("all_","")
-            formattext = text
-            while "  " in formattext:
-                formattext = formattext.replace("  ", " ")
-            if language == "all_zh":
-                if re.search(r'[A-Za-z]', formattext):
-                    formattext = re.sub(r'[a-z]', lambda x: x.group(0).upper(), formattext)
-                    formattext = chinese.mix_text_normalize(formattext)
-                    return self.get_phones_and_bert(formattext,"zh",version)
-                else:
-                    phones, word2ph, norm_text = self.clean_text_inf(formattext, language, version)
-                    bert = self.get_bert_feature(norm_text, word2ph).to(self.device)
-            elif language == "all_yue" and re.search(r'[A-Za-z]', formattext):
-                    formattext = re.sub(r'[a-z]', lambda x: x.group(0).upper(), formattext)
-                    formattext = chinese.mix_text_normalize(formattext)
-                    return self.get_phones_and_bert(formattext,"yue",version)
-            else:
-                phones, word2ph, norm_text = self.clean_text_inf(formattext, language, version)
-                bert = torch.zeros(
-                    (1024, len(phones)),
-                    dtype=torch.float32,
-                ).to(self.device)
-        elif language in {"zh", "ja", "ko", "yue", "auto", "auto_yue"}:
-            textlist=[]
-            langlist=[]
-            if language == "auto":
-                for tmp in LangSegmenter.getTexts(text):
-                    langlist.append(tmp["lang"])
-                    textlist.append(tmp["text"])
-            elif language == "auto_yue":
-                for tmp in LangSegmenter.getTexts(text):
-                    if tmp["lang"] == "zh":
-                        tmp["lang"] = "yue"
-                    langlist.append(tmp["lang"])
-                    textlist.append(tmp["text"])
-            else:
-                for tmp in LangSegmenter.getTexts(text):
-                    if tmp["lang"] == "en":
-                        langlist.append(tmp["lang"])
-                    else:
-                        # 因无法区别中日韩文汉字,以用户输入为准
-                        langlist.append(language)
-                    textlist.append(tmp["text"])
-            # print(textlist)
-            # print(langlist)
-            phones_list = []
-            bert_list = []
-            norm_text_list = []
-            for i in range(len(textlist)):
-                lang = langlist[i]
-                phones, word2ph, norm_text = self.clean_text_inf(textlist[i], lang, version)
-                bert = self.get_bert_inf(phones, word2ph, norm_text, lang)
-                phones_list.append(phones)
-                norm_text_list.append(norm_text)
-                bert_list.append(bert)
-            bert = torch.cat(bert_list, dim=1)
-            phones = sum(phones_list, [])
-            norm_text = ''.join(norm_text_list)
+        with self.bert_lock:
+          if language in {"en", "all_zh", "all_ja", "all_ko", "all_yue"}:
+              # language = language.replace("all_","")
+              formattext = text
+              while "  " in formattext:
+                  formattext = formattext.replace("  ", " ")
+              if language == "all_zh":
+                  if re.search(r'[A-Za-z]', formattext):
+                      formattext = re.sub(r'[a-z]', lambda x: x.group(0).upper(), formattext)
+                      formattext = chinese.mix_text_normalize(formattext)
+                      return self.get_phones_and_bert(formattext,"zh",version)
+                  else:
+                      phones, word2ph, norm_text = self.clean_text_inf(formattext, language, version)
+                      bert = self.get_bert_feature(norm_text, word2ph).to(self.device)
+              elif language == "all_yue" and re.search(r'[A-Za-z]', formattext):
+                      formattext = re.sub(r'[a-z]', lambda x: x.group(0).upper(), formattext)
+                      formattext = chinese.mix_text_normalize(formattext)
+                      return self.get_phones_and_bert(formattext,"yue",version)
+              else:
+                  phones, word2ph, norm_text = self.clean_text_inf(formattext, language, version)
+                  bert = torch.zeros(
+                      (1024, len(phones)),
+                      dtype=torch.float32,
+                  ).to(self.device)
+          elif language in {"zh", "ja", "ko", "yue", "auto", "auto_yue"}:
+              textlist=[]
+              langlist=[]
+              if language == "auto":
+                  for tmp in LangSegmenter.getTexts(text):
+                      langlist.append(tmp["lang"])
+                      textlist.append(tmp["text"])
+              elif language == "auto_yue":
+                  for tmp in LangSegmenter.getTexts(text):
+                      if tmp["lang"] == "zh":
+                          tmp["lang"] = "yue"
+                      langlist.append(tmp["lang"])
+                      textlist.append(tmp["text"])
+              else:
+                  for tmp in LangSegmenter.getTexts(text):
+                      if tmp["lang"] == "en":
+                          langlist.append(tmp["lang"])
+                      else:
+                          # 因无法区别中日韩文汉字,以用户输入为准
+                          langlist.append(language)
+                      textlist.append(tmp["text"])
+              # print(textlist)
+              # print(langlist)
+              phones_list = []
+              bert_list = []
+              norm_text_list = []
+              for i in range(len(textlist)):
+                  lang = langlist[i]
+                  phones, word2ph, norm_text = self.clean_text_inf(textlist[i], lang, version)
+                  bert = self.get_bert_inf(phones, word2ph, norm_text, lang)
+                  phones_list.append(phones)
+                  norm_text_list.append(norm_text)
+                  bert_list.append(bert)
+              bert = torch.cat(bert_list, dim=1)
+              phones = sum(phones_list, [])
+              norm_text = ''.join(norm_text_list)
 
-        if not final and len(phones) < 6:
-            return self.get_phones_and_bert("." + text,language,version,final=True)
+          if not final and len(phones) < 6:
+              return self.get_phones_and_bert("." + text,language,version,final=True)
 
-        return phones, bert, norm_text
+          return phones, bert, norm_text
 
 
     def get_bert_feature(self, text:str, word2ph:list)->torch.Tensor:

GPT_SoVITS/prepare_datasets/3-get-semantic.py

@@ -81,7 +81,7 @@ if os.path.exists(semantic_path) == False:
     # utils.load_checkpoint(pretrained_s2G, vq_model, None, True)
     print(
         vq_model.load_state_dict(
-            torch.load(pretrained_s2G, map_location="cpu")["weight"], strict=False
+            torch.load(pretrained_s2G, map_location="cpu", weights_only=False)["weight"], strict=False
         )
     )
 

api_v2.py

@@ -23,7 +23,8 @@ POST:
 {
     "text": "",                   # str.(required) text to be synthesized
     "text_lang: "",               # str.(required) language of the text to be synthesized
-    "ref_audio_path": "",         # str.(required) reference audio path
+    "ref_audio_path": "",         # str.(optional) reference audio path, required if `ref_audio` not set
+    "ref_audio": ""               # str.(optional) reference audio encoded as base64, required if `ref_audio_path` not set
     "aux_ref_audio_paths": [],    # list.(optional) auxiliary reference audio paths for multi-speaker tone fusion
     "prompt_text": "",            # str.(optional) prompt text for the reference audio
     "prompt_lang": "",            # str.(required) language of the prompt text for the reference audio
@@ -44,6 +45,53 @@ POST:
 }
 ```
 
+#### 使用base64发送reference audio示例代码
+```python
+import requests
+import base64
+import json
+
+# 定义音频文件路径和文本内容
+audio_file_path = "/src/GPT-SoVITS/reference/news_reporter.wav"
+audio_text = "系统内随后发售的其它车次车票依然十分抢手，仅个别车次仍有余票。"
+
+# 读取音频文件并进行Base64编码
+with open(audio_file_path, "rb") as audio_file:
+    audio_data = audio_file.read()
+    audio_base64 = base64.b64encode(audio_data).decode('utf-8')
+
+# 构建请求数据
+payload = {
+    "text": "四渡赤水中，毛泽东险些辞职。因为他的一个打法，违反了最基本的战场原则——先弱后强，遭到了所有人的反对。但在周恩来的支持下，其他人还是按他说的打了。果然，这不仅避开了蒋中正的陷阱，还让红军找到了生机。",
+    "text_lang": "zh",
+    "ref_audio": audio_base64,
+    "prompt_text": audio_text,
+    "prompt_lang": "zh",
+    "top_k": 15,
+    "top_p": 0.7,
+    "temperature": 0.85,
+    "text_split_method": "cut5",
+    "batch_size": 1,
+    "media_type": "wav",
+    "streaming_mode": False,
+    "speed_factor": 1.1
+}
+
+# 发送POST请求
+url = 'http://localhost:9880/tts'
+headers = {'Content-Type': 'application/json'}
+response = requests.post(url, headers=headers, data=json.dumps(payload))
+
+# 检查响应状态码
+if response.status_code == 200:
+    with open('output.wav', 'wb') as output_file:
+        output_file.write(response.content)
+    print("音频文件已保存为 output.wav")
+else:
+    print(f"请求失败，状态码: {response.status_code}")
+    print(f"响应内容: {response.text}")
+```
+
 RESP:
 成功: 直接返回 wav 音频流， http code 200
 失败: 返回包含错误信息的 json, http code 400
@@ -101,6 +149,8 @@ import os
 import sys
 import traceback
 from typing import Generator
+import base64
+import tempfile
 
 now_dir = os.getcwd()
 sys.path.append(now_dir)
@@ -149,6 +199,7 @@ class TTS_Request(BaseModel):
     text: str = None
     text_lang: str = None
     ref_audio_path: str = None
+    ref_audio: str = None
     aux_ref_audio_paths: list = None
     prompt_lang: str = None
     prompt_text: str = ""
@@ -250,8 +301,8 @@ def check_params(req:dict):
     prompt_lang:str = req.get("prompt_lang", "")
     text_split_method:str = req.get("text_split_method", "cut5")
 
-    if ref_audio_path in [None, ""]:
-        return JSONResponse(status_code=400, content={"message": "ref_audio_path is required"})
+    if ref_audio_path in [None, ""] and ref_audio in [None, ""]:
+        return JSONResponse(status_code=400, content={"message": "ref_audio_path or ref_audio is required"})
     if text in [None, ""]:
         return JSONResponse(status_code=400, content={"message": "text is required"})
     if (text_lang in [None, ""]) :
@@ -377,6 +428,7 @@ async def tts_get_endpoint(
                         sample_steps:int =32,
                         super_sampling:bool = False
                         ):
+
     req = {
         "text": text,
         "text_lang": text_lang.lower(),
@@ -407,6 +459,13 @@ async def tts_get_endpoint(
 @APP.post("/tts")
 async def tts_post_endpoint(request: TTS_Request):
     req = request.dict()
+    if req['ref_audio']:
+        print(req['ref_audio'])
+        # 上传了以base64编码的参考音频，解码后存入临时文件并将文件路径赋值给ref_audio_path
+        decoded_audio = base64.b64decode(req['ref_audio'])
+        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+            temp_file.write(decoded_audio)
+            req['ref_audio_path'] = temp_file.name
     return await tts_handle(req)
 
 

install.sh

@@ -2,8 +2,13 @@
 
 # 安装构建工具
 # Install build tools
+echo "Installing GCC..."
 conda install -c conda-forge gcc=14
+
+echo "Installing G++..."
 conda install -c conda-forge gxx
+
+echo "Installing ffmpeg and cmake..."
 conda install ffmpeg cmake
 
 # 设置编译环境
@@ -12,10 +17,60 @@ export CMAKE_MAKE_PROGRAM="$CONDA_PREFIX/bin/cmake"
 export CC="$CONDA_PREFIX/bin/gcc"
 export CXX="$CONDA_PREFIX/bin/g++"
 
-conda install pytorch==2.1.1 torchvision==0.16.1 torchaudio==2.1.1 pytorch-cuda=11.8 -c pytorch -c nvidia
+echo "Checking for CUDA installation..."
+if command -v nvidia-smi &> /dev/null; then
+    USE_CUDA=true
+    echo "CUDA found."
+else
+    echo "CUDA not found."
+    USE_CUDA=false
+fi
+
+
+if [ "$USE_CUDA" = false ]; then
+    echo "Checking for ROCm installation..."
+    if [ -d "/opt/rocm" ]; then
+        USE_ROCM=true
+        echo "ROCm found."
+        if grep -qi "microsoft" /proc/version; then
+            echo "You are running WSL."
+            IS_WSL=true
+        else
+            echo "You are NOT running WSL."
+            IS_WSL=false
+        fi
+    else
+        echo "ROCm not found."
+        USE_ROCM=false
+    fi
+fi
+
+if [ "$USE_CUDA" = true ]; then
+    echo "Installing PyTorch with CUDA support..."
+    conda install pytorch==2.1.1 torchvision==0.16.1 torchaudio==2.1.1 pytorch-cuda=11.8 -c pytorch -c nvidia
+elif [ "$USE_ROCM" = true ] ; then
+    echo "Installing PyTorch with ROCm support..."
+    pip install torch==2.5.1 torchvision==0.20.1 torchaudio==2.5.1 --index-url https://download.pytorch.org/whl/rocm6.2
+else
+    echo "Installing PyTorch for CPU..."
+    conda install pytorch==2.1.1 torchvision==0.16.1 torchaudio==2.1.1 cpuonly -c pytorch
+fi
+
+
+echo "Installing Python dependencies from requirements.txt..."
 
 # 刷新环境
 # Refresh environment
 hash -r
+pip install -r requirements.txt
+
+if [ "$USE_ROCM" = true ] && [ "$IS_WSL" = true ] ; then
+    echo "Update to WSL compatible runtime lib..."
+    location=`pip show torch | grep Location | awk -F ": " '{print $2}'`
+    cd ${location}/torch/lib/
+    rm libhsa-runtime64.so*
+    cp /opt/rocm/lib/libhsa-runtime64.so.1.2 libhsa-runtime64.so
+fi
+
+echo "Installation completed successfully!"
 
-pip install -r requirements.txt

tools/my_utils.py

@@ -32,7 +32,7 @@ def clean_path(path_str:str):
     if path_str.endswith(('\\','/')):
         return clean_path(path_str[0:-1])
     path_str = path_str.replace('/', os.sep).replace('\\', os.sep)
-    return path_str.strip(" ").strip('\'').strip("\n").strip('"').strip(" ").strip("\u202a")
+    return path_str.strip(" \'\n\"\u202a")#path_str.strip(" ").strip('\'').strip("\n").strip('"').strip(" ").strip("\u202a")
 
 
 def check_for_existance(file_list:list=None,is_train=False,is_dataset_processing=False):