From 2cafde159c8102b7495827ad6445ad397971843c Mon Sep 17 00:00:00 2001 From: ChasonJiang <46401978+ChasonJiang@users.noreply.github.com> Date: Sun, 19 May 2024 16:40:13 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=8A=A5=E9=94=99=20Type?= =?UTF-8?q?Error:'type'=20object=20is=20not=20subscriptable.=20(#1087)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- GPT_SoVITS/TTS_infer_pack/TTS.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GPT_SoVITS/TTS_infer_pack/TTS.py b/GPT_SoVITS/TTS_infer_pack/TTS.py index 4befc0c4..cd1df0be 100644 --- a/GPT_SoVITS/TTS_infer_pack/TTS.py +++ b/GPT_SoVITS/TTS_infer_pack/TTS.py @@ -9,7 +9,7 @@ now_dir = os.getcwd() sys.path.append(now_dir) import ffmpeg import os -from typing import Generator, List, Union +from typing import Generator, List, Tuple, Union import numpy as np import torch import torch.nn.functional as F @@ -591,7 +591,7 @@ class TTS: "repetition_penalty": 1.35 # float. repetition penalty for T2S model. } returns: - tuple[int, np.ndarray]: sampling rate and audio data. + Tuple[int, np.ndarray]: sampling rate and audio data. """ ########## variables initialization ########### self.stop_flag:bool = False @@ -874,7 +874,7 @@ class TTS: speed_factor:float=1.0, split_bucket:bool=True, fragment_interval:float=0.3 - )->tuple[int, np.ndarray]: + )->Tuple[int, np.ndarray]: zero_wav = torch.zeros( int(self.configs.sampling_rate * fragment_interval), dtype=self.precision, From 50c3664496205d79cfa4853d849d765dd619fe8e Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Sun, 19 May 2024 17:15:56 +0800 Subject: [PATCH 2/5] chore: add the ability of lru cache for api v3 to improve the inference speed when exchange model weights (#1058) * chore: add the ability of lru cache for api v3 to improve the inference speed when exchange model weights * chore: Dockerfile start api v3 * chore: api default port from 127.0.0.1 to 0.0.0.0 * chore: make gpu happy when do tts * chore: rollback Dockerfile * chore: fix * chore: fix --------- Co-authored-by: kevin.zhang --- GPT_SoVITS/TTS_infer_pack/TTS.py | 6 + api_v3.py | 467 +++++++++++++++++++++++++++++++ 2 files changed, 473 insertions(+) create mode 100644 api_v3.py diff --git a/GPT_SoVITS/TTS_infer_pack/TTS.py b/GPT_SoVITS/TTS_infer_pack/TTS.py index cd1df0be..eaacb529 100644 --- a/GPT_SoVITS/TTS_infer_pack/TTS.py +++ b/GPT_SoVITS/TTS_infer_pack/TTS.py @@ -182,6 +182,12 @@ class TTS_Config: def __repr__(self): return self.__str__() + def __hash__(self): + return hash(self.configs_path) + + def __eq__(self, other): + return isinstance(other, TTS_Config) and self.configs_path == other.configs_path + class TTS: def __init__(self, configs: Union[dict, str, TTS_Config]): diff --git a/api_v3.py b/api_v3.py new file mode 100644 index 00000000..a121dfc4 --- /dev/null +++ b/api_v3.py @@ -0,0 +1,467 @@ +""" +# WebAPI文档 (3.0) - 使用了缓存技术，初始化时使用LRU Cache TTS 实例，缓存加载模型的世界，达到减少切换不同语音时的推理时间 + +` python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/tts_infer.yaml ` + +## 执行参数: + `-a` - `绑定地址, 默认"127.0.0.1"` + `-p` - `绑定端口, 默认9880` + `-c` - `TTS配置文件路径, 默认"GPT_SoVITS/configs/tts_infer.yaml"` + +## 调用: + +### 推理 + +endpoint: `/tts` +GET: +``` +http://127.0.0.1:9880/tts?text=先帝创业未半而中道崩殂，今天下三分，益州疲弊，此诚危急存亡之秋也。&text_lang=zh&ref_audio_path=archive_jingyuan_1.wav&prompt_lang=zh&prompt_text=我是「罗浮」云骑将军景元。不必拘谨，「将军」只是一时的身份，你称呼我景元便可&text_split_method=cut5&batch_size=1&media_type=wav&streaming_mode=true +``` + +POST: +```json +{ + "text": "", # str.(required) text to be synthesized + "text_lang": "", # str.(required) language of the text to be synthesized + "ref_audio_path": "", # str.(required) reference audio path. + "prompt_text": "", # str.(optional) prompt text for the reference audio + "prompt_lang": "", # str.(required) language of the prompt text for the reference audio + "top_k": 5, # int.(optional) top k sampling + "top_p": 1, # float.(optional) top p sampling + "temperature": 1, # float.(optional) temperature for sampling + "text_split_method": "cut5", # str.(optional) text split method, see text_segmentation_method.py for details. + "batch_size": 1, # int.(optional) batch size for inference + "batch_threshold": 0.75, # float.(optional) threshold for batch splitting. + "split_bucket": true, # bool.(optional) whether to split the batch into multiple buckets. + "speed_factor":1.0, # float.(optional) control the speed of the synthesized audio. + "fragment_interval":0.3, # float.(optional) to control the interval of the audio fragment. + "seed": -1, # int.(optional) random seed for reproducibility. + "media_type": "wav", # str.(optional) media type of the output audio, support "wav", "raw", "ogg", "aac". + "streaming_mode": false, # bool.(optional) whether to return a streaming response. + "parallel_infer": True, # bool.(optional) whether to use parallel inference. + "repetition_penalty": 1.35, # float.(optional) repetition penalty for T2S model. + "tts_infer_yaml_path": “GPT_SoVITS/configs/tts_infer.yaml” # str.(optional) tts infer yaml path +} +``` + +RESP: +成功: 直接返回 wav 音频流， http code 200 +失败: 返回包含错误信息的 json, http code 400 + +### 命令控制 + +endpoint: `/control` + +command: +"restart": 重新运行 +"exit": 结束运行 + +GET: +``` +http://127.0.0.1:9880/control?command=restart +``` +POST: +```json +{ + "command": "restart" +} +``` + +RESP: 无 + + +### 切换GPT模型 + +endpoint: `/set_gpt_weights` + +GET: +``` +http://127.0.0.1:9880/set_gpt_weights?weights_path=GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt +``` +RESP: +成功: 返回"success", http code 200 +失败: 返回包含错误信息的 json, http code 400 + + +### 切换Sovits模型 + +endpoint: `/set_sovits_weights` + +GET: +``` +http://127.0.0.1:9880/set_sovits_weights?weights_path=GPT_SoVITS/pretrained_models/s2G488k.pth +``` + +RESP: +成功: 返回"success", http code 200 +失败: 返回包含错误信息的 json, http code 400 + +""" +import os +import sys +import traceback +from typing import Generator + +import torch + +now_dir = os.getcwd() +sys.path.append(now_dir) +sys.path.append("%s/GPT_SoVITS" % (now_dir)) + +import argparse +import subprocess +import wave +import signal +import numpy as np +import soundfile as sf +from fastapi import Response +from fastapi.responses import JSONResponse +from fastapi import FastAPI +import uvicorn +from io import BytesIO +from GPT_SoVITS.TTS_infer_pack.TTS import TTS, TTS_Config +from GPT_SoVITS.TTS_infer_pack.text_segmentation_method import get_method_names as get_cut_method_names +from fastapi.responses import StreamingResponse +from pydantic import BaseModel +from functools import lru_cache + +cut_method_names = get_cut_method_names() + +parser = argparse.ArgumentParser(description="GPT-SoVITS api") +parser.add_argument("-a", "--bind_addr", type=str, default="0.0.0.0", help="default: 0.0.0.0") +parser.add_argument("-p", "--port", type=int, default="9880", help="default: 9880") +args = parser.parse_args() +port = args.port +host = args.bind_addr +argv = sys.argv + +APP = FastAPI() + + +class TTS_Request(BaseModel): + text: str = None + text_lang: str = None + ref_audio_path: str = None + prompt_lang: str = None + prompt_text: str = "" + top_k: int = 5 + top_p: float = 1 + temperature: float = 1 + text_split_method: str = "cut5" + batch_size: int = 1 + batch_threshold: float = 0.75 + split_bucket: bool = True + speed_factor: float = 1.0 + fragment_interval: float = 0.3 + seed: int = -1 + media_type: str = "wav" + streaming_mode: bool = False + parallel_infer: bool = True + repetition_penalty: float = 1.35 + tts_infer_yaml_path: str = None + """推理时需要加载的声音模型的yaml配置文件路径，如：GPT_SoVITS/configs/tts_infer.yaml""" + + +@lru_cache(maxsize=10) +def get_tts_instance(tts_config: TTS_Config) -> TTS: + print(f"load tts config from {tts_config.configs_path}") + return TTS(tts_config) + + +def pack_ogg(io_buffer: BytesIO, data: np.ndarray, rate: int): + """modify from https://github.com/RVC-Boss/GPT-SoVITS/pull/894/files""" + with sf.SoundFile(io_buffer, mode='w', samplerate=rate, channels=1, format='ogg') as audio_file: + audio_file.write(data) + return io_buffer + + +def pack_raw(io_buffer: BytesIO, data: np.ndarray, rate: int): + io_buffer.write(data.tobytes()) + return io_buffer + + +def pack_wav(io_buffer: BytesIO, data: np.ndarray, rate: int): + io_buffer = BytesIO() + sf.write(io_buffer, data, rate, format='wav') + return io_buffer + + +def pack_aac(io_buffer: BytesIO, data: np.ndarray, rate: int): + process = subprocess.Popen([ + 'ffmpeg', + '-f', 's16le', # 输入16位有符号小端整数PCM + '-ar', str(rate), # 设置采样率 + '-ac', '1', # 单声道 + '-i', 'pipe:0', # 从管道读取输入 + '-c:a', 'aac', # 音频编码器为AAC + '-b:a', '192k', # 比特率 + '-vn', # 不包含视频 + '-f', 'adts', # 输出AAC数据流格式 + 'pipe:1' # 将输出写入管道 + ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, _ = process.communicate(input=data.tobytes()) + io_buffer.write(out) + return io_buffer + + +def pack_audio(io_buffer: BytesIO, data: np.ndarray, rate: int, media_type: str): + if media_type == "ogg": + io_buffer = pack_ogg(io_buffer, data, rate) + elif media_type == "aac": + io_buffer = pack_aac(io_buffer, data, rate) + elif media_type == "wav": + io_buffer = pack_wav(io_buffer, data, rate) + else: + io_buffer = pack_raw(io_buffer, data, rate) + io_buffer.seek(0) + return io_buffer + + +# from https://huggingface.co/spaces/coqui/voice-chat-with-mistral/blob/main/app.py +def wave_header_chunk(frame_input=b"", channels=1, sample_width=2, sample_rate=32000): + # This will create a wave header then append the frame input + # It should be first on a streaming wav file + # Other frames better should not have it (else you will hear some artifacts each chunk start) + wav_buf = BytesIO() + with wave.open(wav_buf, "wb") as vfout: + vfout.setnchannels(channels) + vfout.setsampwidth(sample_width) + vfout.setframerate(sample_rate) + vfout.writeframes(frame_input) + + wav_buf.seek(0) + return wav_buf.read() + + +def handle_control(command: str): + if command == "restart": + os.execl(sys.executable, sys.executable, *argv) + elif command == "exit": + os.kill(os.getpid(), signal.SIGTERM) + exit(0) + + +def check_params(req: dict, tts_config: TTS_Config): + text: str = req.get("text", "") + text_lang: str = req.get("text_lang", "") + ref_audio_path: str = req.get("ref_audio_path", "") + streaming_mode: bool = req.get("streaming_mode", False) + media_type: str = req.get("media_type", "wav") + prompt_lang: str = req.get("prompt_lang", "") + text_split_method: str = req.get("text_split_method", "cut5") + + if ref_audio_path in [None, ""]: + return JSONResponse(status_code=400, content={"message": "ref_audio_path is required"}) + if text in [None, ""]: + return JSONResponse(status_code=400, content={"message": "text is required"}) + if (text_lang in [None, ""]): + return JSONResponse(status_code=400, content={"message": "text_lang is required"}) + elif text_lang.lower() not in tts_config.languages: + return JSONResponse(status_code=400, content={"message": "text_lang is not supported"}) + if (prompt_lang in [None, ""]): + return JSONResponse(status_code=400, content={"message": "prompt_lang is required"}) + elif prompt_lang.lower() not in tts_config.languages: + return JSONResponse(status_code=400, content={"message": "prompt_lang is not supported"}) + if media_type not in ["wav", "raw", "ogg", "aac"]: + return JSONResponse(status_code=400, content={"message": "media_type is not supported"}) + elif media_type == "ogg" and not streaming_mode: + return JSONResponse(status_code=400, content={"message": "ogg format is not supported in non-streaming mode"}) + + if text_split_method not in cut_method_names: + return JSONResponse(status_code=400, + content={"message": f"text_split_method:{text_split_method} is not supported"}) + + return None + + +async def tts_handle(req: dict): + """ + Text to speech handler. + + Args: + req (dict): + { + "text": "", # str.(required) text to be synthesized + "text_lang: "", # str.(required) language of the text to be synthesized + "ref_audio_path": "", # str.(required) reference audio path + "prompt_text": "", # str.(optional) prompt text for the reference audio + "prompt_lang": "", # str.(required) language of the prompt text for the reference audio + "top_k": 5, # int. top k sampling + "top_p": 1, # float. top p sampling + "temperature": 1, # float. temperature for sampling + "text_split_method": "cut5", # str. text split method, see text_segmentation_method.py for details. + "batch_size": 1, # int. batch size for inference + "batch_threshold": 0.75, # float. threshold for batch splitting. + "split_bucket: True, # bool. whether to split the batch into multiple buckets. + "speed_factor":1.0, # float. control the speed of the synthesized audio. + "fragment_interval":0.3, # float. to control the interval of the audio fragment. + "seed": -1, # int. random seed for reproducibility. + "media_type": "wav", # str. media type of the output audio, support "wav", "raw", "ogg", "aac". + "streaming_mode": False, # bool. whether to return a streaming response. + "parallel_infer": True, # bool.(optional) whether to use parallel inference. + "repetition_penalty": 1.35 # float.(optional) repetition penalty for T2S model. + } + returns: + StreamingResponse: audio stream response. + """ + + streaming_mode = req.get("streaming_mode", False) + media_type = req.get("media_type", "wav") + tts_infer_yaml_path = req.get("tts_infer_yaml_path", "GPT_SoVITS/configs/tts_infer.yaml") + + tts_config = TTS_Config(tts_infer_yaml_path) + check_res = check_params(req, tts_config) + if check_res is not None: + return check_res + + if streaming_mode: + req["return_fragment"] = True + + try: + tts_instance = get_tts_instance(tts_config) + + move_to_gpu(tts_instance, tts_config) + + tts_generator = tts_instance.run(req) + + if streaming_mode: + def streaming_generator(tts_generator: Generator, media_type: str): + if media_type == "wav": + yield wave_header_chunk() + media_type = "raw" + for sr, chunk in tts_generator: + yield pack_audio(BytesIO(), chunk, sr, media_type).getvalue() + move_to_cpu(tts_instance) + + # _media_type = f"audio/{media_type}" if not (streaming_mode and media_type in ["wav", "raw"]) else f"audio/x-{media_type}" + return StreamingResponse(streaming_generator(tts_generator, media_type, ), media_type=f"audio/{media_type}") + + else: + sr, audio_data = next(tts_generator) + audio_data = pack_audio(BytesIO(), audio_data, sr, media_type).getvalue() + move_to_cpu(tts_instance) + return Response(audio_data, media_type=f"audio/{media_type}") + except Exception as e: + return JSONResponse(status_code=400, content={"message": f"tts failed", "Exception": str(e)}) + + +def move_to_cpu(tts): + cpu_device = torch.device('cpu') + tts.set_device(cpu_device) + print("Moved TTS models to CPU to save GPU memory.") + + +def move_to_gpu(tts: TTS, tts_config: TTS_Config): + tts.set_device(tts_config.device) + print("Moved TTS models back to GPU for performance.") + + +@APP.get("/control") +async def control(command: str = None): + if command is None: + return JSONResponse(status_code=400, content={"message": "command is required"}) + handle_control(command) + + +@APP.get("/tts") +async def tts_get_endpoint( + text: str = None, + text_lang: str = None, + ref_audio_path: str = None, + prompt_lang: str = None, + prompt_text: str = "", + top_k: int = 5, + top_p: float = 1, + temperature: float = 1, + text_split_method: str = "cut0", + batch_size: int = 1, + batch_threshold: float = 0.75, + split_bucket: bool = True, + speed_factor: float = 1.0, + fragment_interval: float = 0.3, + seed: int = -1, + media_type: str = "wav", + streaming_mode: bool = False, + parallel_infer: bool = True, + repetition_penalty: float = 1.35, + tts_infer_yaml_path: str = "GPT_SoVITS/configs/tts_infer.yaml" +): + req = { + "text": text, + "text_lang": text_lang.lower(), + "ref_audio_path": ref_audio_path, + "prompt_text": prompt_text, + "prompt_lang": prompt_lang.lower(), + "top_k": top_k, + "top_p": top_p, + "temperature": temperature, + "text_split_method": text_split_method, + "batch_size": int(batch_size), + "batch_threshold": float(batch_threshold), + "speed_factor": float(speed_factor), + "split_bucket": split_bucket, + "fragment_interval": fragment_interval, + "seed": seed, + "media_type": media_type, + "streaming_mode": streaming_mode, + "parallel_infer": parallel_infer, + "repetition_penalty": float(repetition_penalty), + "tts_infer_yaml_path": tts_infer_yaml_path + } + + return await tts_handle(req) + + +@APP.post("/tts") +async def tts_post_endpoint(request: TTS_Request): + req = request.dict() + return await tts_handle(req) + + +@APP.get("/set_refer_audio") +async def set_refer_audio(refer_audio_path: str = None, tts_infer_yaml_path: str = "GPT_SoVITS/configs/tts_infer.yaml"): + try: + tts_config = TTS_Config(tts_infer_yaml_path) + tts_instance = get_tts_instance(tts_config) + tts_instance.set_ref_audio(refer_audio_path) + except Exception as e: + return JSONResponse(status_code=400, content={"message": f"set refer audio failed", "Exception": str(e)}) + return JSONResponse(status_code=200, content={"message": "success"}) + + +@APP.get("/set_gpt_weights") +async def set_gpt_weights(weights_path: str = None, tts_infer_yaml_path: str = "GPT_SoVITS/configs/tts_infer.yaml"): + try: + if weights_path in ["", None]: + return JSONResponse(status_code=400, content={"message": "gpt weight path is required"}) + + tts_config = TTS_Config(tts_infer_yaml_path) + tts_instance = get_tts_instance(tts_config) + tts_instance.init_t2s_weights(weights_path) + except Exception as e: + return JSONResponse(status_code=400, content={"message": f"change gpt weight failed", "Exception": str(e)}) + + return JSONResponse(status_code=200, content={"message": "success"}) + + +@APP.get("/set_sovits_weights") +async def set_sovits_weights(weights_path: str = None, tts_infer_yaml_path: str = "GPT_SoVITS/configs/tts_infer.yaml"): + try: + if weights_path in ["", None]: + return JSONResponse(status_code=400, content={"message": "sovits weight path is required"}) + + tts_config = TTS_Config(tts_infer_yaml_path) + tts_instance = get_tts_instance(tts_config) + tts_instance.init_vits_weights(weights_path) + except Exception as e: + return JSONResponse(status_code=400, content={"message": f"change sovits weight failed", "Exception": str(e)}) + return JSONResponse(status_code=200, content={"message": "success"}) + + +if __name__ == "__main__": + try: + uvicorn.run(APP, host=host, port=port, workers=1) + except Exception as e: + traceback.print_exc() + os.kill(os.getpid(), signal.SIGTERM) + exit(0) From f822b9588f4748342e91861c88dbf438f658b022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=82=A6?= Date: Fri, 24 May 2024 18:32:44 +0800 Subject: [PATCH 3/5] Update api_v2.py (#1104) --- api_v2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api_v2.py b/api_v2.py index 9f45ac53..aaa56e0b 100644 --- a/api_v2.py +++ b/api_v2.py @@ -446,8 +446,8 @@ async def set_sovits_weights(weights_path: str = None): if __name__ == "__main__": try: - uvicorn.run(APP, host=host, port=port, workers=1) + uvicorn.run(app="api_v2:APP", host=host, port=port, workers=1) except Exception as e: traceback.print_exc() os.kill(os.getpid(), signal.SIGTERM) - exit(0) \ No newline at end of file + exit(0) From 8fc1e34f5863cb6deb9395262b47e2870ef721a1 Mon Sep 17 00:00:00 2001 From: XXXXRT666 <157766680+XXXXRT666@users.noreply.github.com> Date: Sun, 26 May 2024 17:45:20 +0100 Subject: [PATCH 4/5] name2go fix for fast_inference (#1133) --- GPT_SoVITS/prepare_datasets/2-get-hubert-wav32k.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GPT_SoVITS/prepare_datasets/2-get-hubert-wav32k.py b/GPT_SoVITS/prepare_datasets/2-get-hubert-wav32k.py index 9a2f73c0..61c933a4 100644 --- a/GPT_SoVITS/prepare_datasets/2-get-hubert-wav32k.py +++ b/GPT_SoVITS/prepare_datasets/2-get-hubert-wav32k.py @@ -82,7 +82,7 @@ def name2go(wav_name,wav_path): tensor_wav16 = tensor_wav16.to(device) ssl=model.model(tensor_wav16.unsqueeze(0))["last_hidden_state"].transpose(1,2).cpu()#torch.Size([1, 768, 215]) if np.isnan(ssl.detach().numpy()).sum()!= 0: - nan_fails.append(wav_name) + nan_fails.append((wav_name,wav_path)) print("nan filtered:%s"%wav_name) return wavfile.write( @@ -90,7 +90,7 @@ def name2go(wav_name,wav_path): 32000, tmp_audio32.astype("int16"), ) - my_save(ssl,hubert_path ) + my_save(ssl,hubert_path) with open(inp_text,"r",encoding="utf8")as f: lines=f.read().strip("\n").split("\n") @@ -113,8 +113,8 @@ for line in lines[int(i_part)::int(all_parts)]: if(len(nan_fails)>0 and is_half==True): is_half=False model=model.float() - for wav_name in nan_fails: + for wav in nan_fails: try: - name2go(wav_name) + name2go(wav[0],wav[1]) except: print(wav_name,traceback.format_exc()) From 3e1288bdd6812c58091ad9ebcc640e33ffed61d2 Mon Sep 17 00:00:00 2001 From: XXXXRT666 <157766680+XXXXRT666@users.noreply.github.com> Date: Mon, 27 May 2024 04:24:24 +0100 Subject: [PATCH 5/5] Sync with main branch (#1134) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * api接口，修复文本切分符号设定中，中文分号错写为英文分号的问题 (#1001) * 一些小问题修复 (#1021) * fix import error. It may happen when calling api.py * Update README.md * Update gpt-sovits_kaggle.ipynb * Update gpt-sovits_kaggle.ipynb * fix path error delete useless line wraps * 删除重复的 COPY 指令 (#1073) * [优化] 1Aa-文本获取 (#1102) * Filter unsupported languages * add feedback * simplify modification * fix detail * Update english.py (#1106) copy but not ref the phones list becoz it will be extend later, if not do so,it will affect the self.cmu dict values. * Update models.py * modify freeze_quantizer mode, avoid quantizer's codebook updating (#953) --------- Co-authored-by: FengQingYunDan Co-authored-by: Kenn Zhang Co-authored-by: 蓝梦实 <36986837+SapphireLab@users.noreply.github.com> Co-authored-by: lyris Co-authored-by: hcwu1993 <15855138469@163.com> --- .gitignore | 5 ----- Dockerfile | 3 --- GPT_SoVITS/module/models.py | 13 ++++++++++--- GPT_SoVITS/prepare_datasets/1-get-text.py | 9 ++++++--- GPT_SoVITS/text/english.py | 4 ++-- api.py | 13 +++++++------ docs/ja/README.md | 2 +- gpt-sovits_kaggle.ipynb | 4 ++-- tools/my_utils.py | 2 +- webui.py | 7 +++++-- 10 files changed, 34 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 28b8a7a5..754b06b7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,8 +10,3 @@ reference GPT_weights SoVITS_weights TEMP -PortableGit -ffmpeg.exe -ffprobe.exe -tmp_audio -trained diff --git a/Dockerfile b/Dockerfile index 74e282c4..80cd9f3a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,9 +34,6 @@ RUN if [ "$IMAGE_TYPE" != "elite" ]; then \ fi -# Copy the rest of the application -COPY . /workspace - # Copy the rest of the application COPY . /workspace diff --git a/GPT_SoVITS/module/models.py b/GPT_SoVITS/module/models.py index b14e7c81..26840ccc 100644 --- a/GPT_SoVITS/module/models.py +++ b/GPT_SoVITS/module/models.py @@ -16,6 +16,7 @@ from module.mrte_model import MRTE from module.quantize import ResidualVectorQuantizer from text import symbols from torch.cuda.amp import autocast +import contextlib class StochasticDurationPredictor(nn.Module): @@ -891,9 +892,10 @@ class SynthesizerTrn(nn.Module): self.ssl_proj = nn.Conv1d(ssl_dim, ssl_dim, 1, stride=1) self.quantizer = ResidualVectorQuantizer(dimension=ssl_dim, n_q=1, bins=1024) - if freeze_quantizer: - self.ssl_proj.requires_grad_(False) - self.quantizer.requires_grad_(False) + self.freeze_quantizer = freeze_quantizer + # if freeze_quantizer: + # self.ssl_proj.requires_grad_(False) + # self.quantizer.requires_grad_(False) #self.quantizer.eval() # self.enc_p.text_embedding.requires_grad_(False) # self.enc_p.encoder_text.requires_grad_(False) @@ -906,6 +908,11 @@ class SynthesizerTrn(nn.Module): ge = self.ref_enc(y * y_mask, y_mask) with autocast(enabled=False): + maybe_no_grad = torch.no_grad() if self.freeze_quantizer else contextlib.nullcontext() + with maybe_no_grad: + if self.freeze_quantizer: + self.ssl_proj.eval() + self.quantizer.eval() ssl = self.ssl_proj(ssl) quantized, codes, commit_loss, quantized_list = self.quantizer( ssl, layers=[0] diff --git a/GPT_SoVITS/prepare_datasets/1-get-text.py b/GPT_SoVITS/prepare_datasets/1-get-text.py index b2413826..e01a63b9 100644 --- a/GPT_SoVITS/prepare_datasets/1-get-text.py +++ b/GPT_SoVITS/prepare_datasets/1-get-text.py @@ -117,9 +117,12 @@ if os.path.exists(txt_path) == False: try: wav_name, spk_name, language, text = line.split("|") # todo.append([name,text,"zh"]) - todo.append( - [wav_name, text, language_v1_to_language_v2.get(language, language)] - ) + if language in language_v1_to_language_v2.keys(): + todo.append( + [wav_name, text, language_v1_to_language_v2.get(language, language)] + ) + else: + print(f"\033[33m[Waring] The {language = } of {wav_name} is not supported for training.\033[0m") except: print(line, traceback.format_exc()) diff --git a/GPT_SoVITS/text/english.py b/GPT_SoVITS/text/english.py index 68ce7896..30fafb51 100644 --- a/GPT_SoVITS/text/english.py +++ b/GPT_SoVITS/text/english.py @@ -320,7 +320,7 @@ class en_G2p(G2p): # 尝试分离所有格 if re.match(r"^([a-z]+)('s)$", word): - phones = self.qryword(word[:-2]) + phones = self.qryword(word[:-2])[:] # P T K F TH HH 无声辅音结尾 's 发 ['S'] if phones[-1] in ['P', 'T', 'K', 'F', 'TH', 'HH']: phones.extend(['S']) @@ -359,4 +359,4 @@ def g2p(text): if __name__ == "__main__": print(g2p("hello")) print(g2p(text_normalize("e.g. I used openai's AI tool to draw a picture."))) - print(g2p(text_normalize("In this; paper, we propose 1 DSPGAN, a GAN-based universal vocoder."))) \ No newline at end of file + print(g2p(text_normalize("In this; paper, we propose 1 DSPGAN, a GAN-based universal vocoder."))) diff --git a/api.py b/api.py index ea0e39d0..ea3e123f 100644 --- a/api.py +++ b/api.py @@ -120,6 +120,11 @@ RESP: 无 import argparse import os,re import sys + +now_dir = os.getcwd() +sys.path.append(now_dir) +sys.path.append("%s/GPT_SoVITS" % (now_dir)) + import signal import LangSegment from time import time as ttime @@ -381,7 +386,7 @@ def read_clean_buffer(audio_bytes): def cut_text(text, punc): - punc_list = [p for p in punc if p in {",", ".", ";", "?", "!", "、", "，", "。", "？", "！", ";", "：", "…"}] + punc_list = [p for p in punc if p in {",", ".", ";", "?", "!", "、", "，", "。", "？", "！", "；", "：", "…"}] if len(punc_list) > 0: punds = r"[" + "".join(punc_list) + r"]" text = text.strip("\n") @@ -536,10 +541,6 @@ def handle(refer_wav_path, prompt_text, prompt_language, text, text_language, cu # -------------------------------- # 初始化部分 # -------------------------------- -now_dir = os.getcwd() -sys.path.append(now_dir) -sys.path.append("%s/GPT_SoVITS" % (now_dir)) - dict_language = { "中文": "all_zh", "英文": "en", @@ -579,7 +580,7 @@ parser.add_argument("-hp", "--half_precision", action="store_true", default=Fals # 此时 full_precision==True, half_precision==False parser.add_argument("-sm", "--stream_mode", type=str, default="close", help="流式返回模式, close / normal / keepalive") parser.add_argument("-mt", "--media_type", type=str, default="wav", help="音频编码格式, wav / ogg / aac") -parser.add_argument("-cp", "--cut_punc", type=str, default="", help="文本切分符号设定, 符号范围,.;?!、，。？！;：…") +parser.add_argument("-cp", "--cut_punc", type=str, default="", help="文本切分符号设定, 符号范围,.;?!、，。？！；：…") # 切割常用分句符为 `python ./api.py -cp ".?!。？！"` parser.add_argument("-hb", "--hubert_path", type=str, default=g_config.cnhubert_path, help="覆盖config.cnhubert_path") parser.add_argument("-b", "--bert_path", type=str, default=g_config.bert_path, help="覆盖config.bert_path") diff --git a/docs/ja/README.md b/docs/ja/README.md index 02d1b836..a910f94d 100644 --- a/docs/ja/README.md +++ b/docs/ja/README.md @@ -159,7 +159,7 @@ D:\GPT-SoVITS\xxx/xxx.wav|xxx|en|I like playing Genshin. - [ ] **優先度 高:** - [x] 日本語と英語でのローカライズ。 - - [] ユーザーガイド。 + - [ ] ユーザーガイド。 - [x] 日本語データセットと英語データセットのファインチューニングトレーニング。 - [ ] **機能:** diff --git a/gpt-sovits_kaggle.ipynb b/gpt-sovits_kaggle.ipynb index 1980a77a..84ecd89c 100644 --- a/gpt-sovits_kaggle.ipynb +++ b/gpt-sovits_kaggle.ipynb @@ -54,11 +54,11 @@ "source": [ "# @title Download pretrained models 下载预训练模型\n", "!mkdir -p /kaggle/working/GPT-SoVITS/GPT_SoVITS/pretrained_models\n", - "!mkdir -p /kaggle/working/GPT-SoVITS/tools/damo_asr/models\n", + "!mkdir -p /kaggle/working/GPT-SoVITS/tools/asr/models\n", "!mkdir -p /kaggle/working/GPT-SoVITS/tools/uvr5\n", "%cd /kaggle/working/GPT-SoVITS/GPT_SoVITS/pretrained_models\n", "!git clone https://huggingface.co/lj1995/GPT-SoVITS\n", - "%cd /kaggle/working/GPT-SoVITS/tools/damo_asr/models\n", + "%cd /kaggle/working/GPT-SoVITS/tools/asr/models\n", "!git clone https://www.modelscope.cn/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch.git\n", "!git clone https://www.modelscope.cn/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch.git\n", "!git clone https://www.modelscope.cn/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch.git\n", diff --git a/tools/my_utils.py b/tools/my_utils.py index a7755d6d..de79f3b5 100644 --- a/tools/my_utils.py +++ b/tools/my_utils.py @@ -28,4 +28,4 @@ def load_audio(file, sr): def clean_path(path_str): if platform.system() == 'Windows': path_str = path_str.replace('/', '\\') - return path_str.strip(" ").strip('"').strip("\n").strip('"').strip(" ") + return path_str.strip(" ").strip('"').strip("\n").strip('"').strip(" ").strip("\u202a") diff --git a/webui.py b/webui.py index e1c36e1e..c71c1ca4 100644 --- a/webui.py +++ b/webui.py @@ -418,7 +418,10 @@ def open1a(inp_text,inp_wav_dir,exp_name,gpu_numbers,bert_pretrained_dir): with open(path_text, "w", encoding="utf8") as f: f.write("\n".join(opt) + "\n") ps1a=[] - yield "文本进程结束",{"__type__":"update","visible":True},{"__type__":"update","visible":False} + if len("".join(opt)) > 0: + yield "文本进程成功", {"__type__": "update", "visible": True}, {"__type__": "update", "visible": False} + else: + yield "文本进程失败", {"__type__": "update", "visible": True}, {"__type__": "update", "visible": False} else: yield "已有正在进行的文本任务，需先终止才能开启下一次任务", {"__type__": "update", "visible": False}, {"__type__": "update", "visible": True} @@ -583,7 +586,7 @@ def open1abc(inp_text,inp_wav_dir,exp_name,gpu_numbers1a,gpu_numbers1Ba,gpu_numb os.remove(txt_path) with open(path_text, "w",encoding="utf8") as f: f.write("\n".join(opt) + "\n") - + assert len("".join(opt)) > 0, "1Aa-文本获取进程失败" yield "进度：1a-done", {"__type__": "update", "visible": False}, {"__type__": "update", "visible": True} ps1abc=[] #############################1b