diff --git a/GPT_SoVITS/TTS_infer_pack/TTS.py b/GPT_SoVITS/TTS_infer_pack/TTS.py index eaacb529..34498789 100644 --- a/GPT_SoVITS/TTS_infer_pack/TTS.py +++ b/GPT_SoVITS/TTS_infer_pack/TTS.py @@ -70,7 +70,7 @@ def set_seed(seed:int): except: pass return seed - + class TTS_Config: default_configs={ "device": "cpu", @@ -82,30 +82,30 @@ class TTS_Config: } configs:dict = None def __init__(self, configs: Union[dict, str]=None): - + # 设置默认配置文件路径 configs_base_path:str = "GPT_SoVITS/configs/" os.makedirs(configs_base_path, exist_ok=True) self.configs_path:str = os.path.join(configs_base_path, "tts_infer.yaml") - + if configs in ["", None]: if not os.path.exists(self.configs_path): self.save_configs() print(f"Create default config file at {self.configs_path}") configs:dict = {"default": deepcopy(self.default_configs)} - + if isinstance(configs, str): self.configs_path = configs configs:dict = self._load_configs(self.configs_path) - + assert isinstance(configs, dict) default_configs:dict = configs.get("default", None) if default_configs is not None: self.default_configs = default_configs - + self.configs:dict = configs.get("custom", deepcopy(self.default_configs)) - - + + self.device = self.configs.get("device", torch.device("cpu")) self.is_half = self.configs.get("is_half", False) self.t2s_weights_path = self.configs.get("t2s_weights_path", None) @@ -113,7 +113,7 @@ class TTS_Config: self.bert_base_path = self.configs.get("bert_base_path", None) self.cnhuhbert_base_path = self.configs.get("cnhuhbert_base_path", None) - + if (self.t2s_weights_path in [None, ""]) or (not os.path.exists(self.t2s_weights_path)): self.t2s_weights_path = self.default_configs['t2s_weights_path'] print(f"fall back to default t2s_weights_path: {self.t2s_weights_path}") @@ -127,8 +127,8 @@ class TTS_Config: self.cnhuhbert_base_path = self.default_configs['cnhuhbert_base_path'] print(f"fall back to default cnhuhbert_base_path: {self.cnhuhbert_base_path}") self.update_configs() - - + + self.max_sec = None self.hz:int = 50 self.semantic_frame_rate:str = "25hz" @@ -138,14 +138,14 @@ class TTS_Config: self.hop_length:int = 640 self.win_length:int = 2048 self.n_speakers:int = 300 - + self.languages:list = ["auto", "en", "zh", "ja", "all_zh", "all_ja"] - + def _load_configs(self, configs_path: str)->dict: with open(configs_path, 'r') as f: configs = yaml.load(f, Loader=yaml.FullLoader) - + return configs def save_configs(self, configs_path:str=None)->None: @@ -154,7 +154,7 @@ class TTS_Config: } if self.configs is not None: configs["custom"] = self.update_configs() - + if configs_path is None: configs_path = self.configs_path with open(configs_path, 'w') as f: @@ -170,7 +170,7 @@ class TTS_Config: "cnhuhbert_base_path": self.cnhuhbert_base_path, } return self.config - + def __str__(self): self.configs = self.update_configs() string = "TTS Config".center(100, '-') + '\n' @@ -178,7 +178,7 @@ class TTS_Config: string += f"{str(k).ljust(20)}: {str(v)}\n" string += "-" * 100 + '\n' return string - + def __repr__(self): return self.__str__() @@ -195,21 +195,21 @@ class TTS: self.configs = configs else: self.configs:TTS_Config = TTS_Config(configs) - + self.t2s_model:Text2SemanticLightningModule = None self.vits_model:SynthesizerTrn = None self.bert_tokenizer:AutoTokenizer = None self.bert_model:AutoModelForMaskedLM = None self.cnhuhbert_model:CNHubert = None - + self._init_models() - + self.text_preprocessor:TextPreprocessor = \ - TextPreprocessor(self.bert_model, - self.bert_tokenizer, + TextPreprocessor(self.bert_model, + self.bert_tokenizer, self.configs.device) - - + + self.prompt_cache:dict = { "ref_audio_path" : None, "prompt_semantic": None, @@ -220,8 +220,8 @@ class TTS: "bert_features" : None, "norm_text" : None, } - - + + self.stop_flag:bool = False self.precision:torch.dtype = torch.float16 if self.configs.is_half else torch.float32 @@ -231,9 +231,9 @@ class TTS: self.init_bert_weights(self.configs.bert_base_path) self.init_cnhuhbert_weights(self.configs.cnhuhbert_base_path) # self.enable_half_precision(self.configs.is_half) - - - + + + def init_cnhuhbert_weights(self, base_path: str): print(f"Loading CNHuBERT weights from {base_path}") self.cnhuhbert_model = CNHubert(base_path) @@ -241,9 +241,9 @@ class TTS: self.cnhuhbert_model = self.cnhuhbert_model.to(self.configs.device) if self.configs.is_half and str(self.configs.device)!="cpu": self.cnhuhbert_model = self.cnhuhbert_model.half() - - - + + + def init_bert_weights(self, base_path: str): print(f"Loading BERT weights from {base_path}") self.bert_tokenizer = AutoTokenizer.from_pretrained(base_path) @@ -252,16 +252,17 @@ class TTS: self.bert_model = self.bert_model.to(self.configs.device) if self.configs.is_half and str(self.configs.device)!="cpu": self.bert_model = self.bert_model.half() - - def init_vits_weights(self, weights_path: str): + + def init_vits_weights(self, weights_path: str, save_configs: bool = True): print(f"Loading VITS weights from {weights_path}") self.configs.vits_weights_path = weights_path - self.configs.save_configs() + if save_configs: + self.configs.save_configs() dict_s2 = torch.load(weights_path, map_location=self.configs.device) hps = dict_s2["config"] self.configs.filter_length = hps["data"]["filter_length"] self.configs.segment_size = hps["train"]["segment_size"] - self.configs.sampling_rate = hps["data"]["sampling_rate"] + self.configs.sampling_rate = hps["data"]["sampling_rate"] self.configs.hop_length = hps["data"]["hop_length"] self.configs.win_length = hps["data"]["win_length"] self.configs.n_speakers = hps["data"]["n_speakers"] @@ -276,7 +277,7 @@ class TTS: # if ("pretrained" not in weights_path): if hasattr(vits_model, "enc_q"): del vits_model.enc_q - + vits_model = vits_model.to(self.configs.device) vits_model = vits_model.eval() vits_model.load_state_dict(dict_s2["weight"], strict=False) @@ -284,11 +285,12 @@ class TTS: if self.configs.is_half and str(self.configs.device)!="cpu": self.vits_model = self.vits_model.half() - - def init_t2s_weights(self, weights_path: str): + + def init_t2s_weights(self, weights_path: str, save_configs: bool = True): print(f"Loading Text2Semantic weights from {weights_path}") self.configs.t2s_weights_path = weights_path - self.configs.save_configs() + if save_configs: + self.configs.save_configs() self.configs.hz = 50 dict_s1 = torch.load(weights_path, map_location=self.configs.device) config = dict_s1["config"] @@ -300,7 +302,7 @@ class TTS: self.t2s_model = t2s_model if self.configs.is_half and str(self.configs.device)!="cpu": self.t2s_model = self.t2s_model.half() - + def enable_half_precision(self, enable: bool = True): ''' To enable half precision for the TTS model. @@ -311,7 +313,7 @@ class TTS: if str(self.configs.device) == "cpu" and enable: print("Half precision is not supported on CPU.") return - + self.configs.is_half = enable self.precision = torch.float16 if enable else torch.float32 self.configs.save_configs() @@ -333,15 +335,16 @@ class TTS: self.bert_model = self.bert_model.float() if self.cnhuhbert_model is not None: self.cnhuhbert_model = self.cnhuhbert_model.float() - - def set_device(self, device: torch.device): + + def set_device(self, device: torch.device, save_configs: bool = True): ''' To set the device for all models. Args: device: torch.device, the device to use for all models. ''' self.configs.device = device - self.configs.save_configs() + if save_configs: + self.configs.save_configs() if self.t2s_model is not None: self.t2s_model = self.t2s_model.to(device) if self.vits_model is not None: @@ -350,7 +353,7 @@ class TTS: self.bert_model = self.bert_model.to(device) if self.cnhuhbert_model is not None: self.cnhuhbert_model = self.cnhuhbert_model.to(device) - + def set_ref_audio(self, ref_audio_path:str): ''' To set the reference audio for the TTS model, @@ -360,7 +363,7 @@ class TTS: ''' self._set_prompt_semantic(ref_audio_path) self._set_ref_spec(ref_audio_path) - + def _set_ref_spec(self, ref_audio_path): audio = load_audio(ref_audio_path, int(self.configs.sampling_rate)) audio = torch.FloatTensor(audio) @@ -379,8 +382,8 @@ class TTS: spec = spec.half() # self.refer_spec = spec self.prompt_cache["refer_spec"] = spec - - + + def _set_prompt_semantic(self, ref_wav_path:str): zero_wav = np.zeros( int(self.configs.sampling_rate * 0.3), @@ -405,10 +408,10 @@ class TTS: 1, 2 ) # .float() codes = self.vits_model.extract_latent(hubert_feature) - + prompt_semantic = codes[0, 0].to(self.configs.device) self.prompt_cache["prompt_semantic"] = prompt_semantic - + def batch_sequences(self, sequences: List[torch.Tensor], axis: int = 0, pad_value: int = 0, max_length:int=None): seq = sequences[0] ndim = seq.dim() @@ -429,12 +432,12 @@ class TTS: padded_sequences.append(padded_seq) batch = torch.stack(padded_sequences) return batch - - def to_batch(self, data:list, - prompt_data:dict=None, - batch_size:int=5, - threshold:float=0.75, - split_bucket:bool=True, + + def to_batch(self, data:list, + prompt_data:dict=None, + batch_size:int=5, + threshold:float=0.75, + split_bucket:bool=True, device:torch.device=torch.device("cpu"), precision:torch.dtype=torch.float32, ): @@ -447,8 +450,8 @@ class TTS: batch_index_list = [] if split_bucket: index_and_len_list.sort(key=lambda x: x[1]) - index_and_len_list = np.array(index_and_len_list, dtype=np.int64) - + index_and_len_list = np.array(index_and_len_list, dtype=np.int64) + batch_index_list_len = 0 pos = 0 while pos list: ''' Recovery the order of the audio according to the batch_index_list. @@ -568,7 +571,7 @@ class TTS: Stop the inference process. ''' self.stop_flag = True - + @torch.no_grad() def run(self, inputs:dict): """ @@ -668,7 +671,7 @@ class TTS: self.prompt_cache["prompt_lang"] = prompt_lang phones, bert_features, norm_text = \ self.text_preprocessor.segment_and_extract_feature_for_text( - prompt_text, + prompt_text, prompt_lang) self.prompt_cache["phones"] = phones self.prompt_cache["bert_features"] = bert_features @@ -685,9 +688,9 @@ class TTS: return batch_index_list:list = None - data, batch_index_list = self.to_batch(data, - prompt_data=self.prompt_cache if not no_prompt_text else None, - batch_size=batch_size, + data, batch_index_list = self.to_batch(data, + prompt_data=self.prompt_cache if not no_prompt_text else None, + batch_size=batch_size, threshold=batch_threshold, split_bucket=split_bucket, device=self.configs.device, @@ -701,7 +704,7 @@ class TTS: if i%batch_size == 0: data.append([]) data[-1].append(texts[i]) - + def make_batch(batch_texts): batch_data = [] print(i18n("############ 提取文本Bert特征 ############")) @@ -717,9 +720,9 @@ class TTS: batch_data.append(res) if len(batch_data) == 0: return None - batch, _ = self.to_batch(batch_data, - prompt_data=self.prompt_cache if not no_prompt_text else None, - batch_size=batch_size, + batch, _ = self.to_batch(batch_data, + prompt_data=self.prompt_cache if not no_prompt_text else None, + batch_size=batch_size, threshold=batch_threshold, split_bucket=False, device=self.configs.device, @@ -781,7 +784,7 @@ class TTS: # 这里要记得加 torch.no_grad() 不然速度慢一大截 # with torch.no_grad(): - + # ## vits并行推理 method 1 # pred_semantic_list = [item[-idx:] for item, idx in zip(pred_semantic_list, idx_list)] # pred_semantic_len = torch.LongTensor([item.shape[0] for item in pred_semantic_list]).to(self.configs.device) @@ -823,10 +826,10 @@ class TTS: t_45 += t5 - t4 if return_fragment: print("%.3f\t%.3f\t%.3f\t%.3f" % (t1 - t0, t2 - t1, t4 - t3, t5 - t4)) - yield self.audio_postprocess([batch_audio_fragment], - self.configs.sampling_rate, - None, - speed_factor, + yield self.audio_postprocess([batch_audio_fragment], + self.configs.sampling_rate, + None, + speed_factor, False, fragment_interval ) @@ -840,10 +843,10 @@ class TTS: if not return_fragment: print("%.3f\t%.3f\t%.3f\t%.3f" % (t1 - t0, t2 - t1, t_34, t_45)) - yield self.audio_postprocess(audio, - self.configs.sampling_rate, - batch_index_list, - speed_factor, + yield self.audio_postprocess(audio, + self.configs.sampling_rate, + batch_index_list, + speed_factor, split_bucket, fragment_interval ) @@ -863,21 +866,21 @@ class TTS: raise e finally: self.empty_cache() - + def empty_cache(self): - try: + try: if "cuda" in str(self.configs.device): torch.cuda.empty_cache() elif str(self.configs.device) == "mps": torch.mps.empty_cache() except: - pass - - def audio_postprocess(self, - audio:List[torch.Tensor], - sr:int, - batch_index_list:list=None, - speed_factor:float=1.0, + pass + + def audio_postprocess(self, + audio:List[torch.Tensor], + sr:int, + batch_index_list:list=None, + speed_factor:float=1.0, split_bucket:bool=True, fragment_interval:float=0.3 )->Tuple[int, np.ndarray]: @@ -886,36 +889,36 @@ class TTS: dtype=self.precision, device=self.configs.device ) - + for i, batch in enumerate(audio): for j, audio_fragment in enumerate(batch): max_audio=torch.abs(audio_fragment).max()#简单防止16bit爆音 if max_audio>1: audio_fragment/=max_audio audio_fragment:torch.Tensor = torch.cat([audio_fragment, zero_wav], dim=0) audio[i][j] = audio_fragment.cpu().numpy() - - + + if split_bucket: audio = self.recovery_order(audio, batch_index_list) else: # audio = [item for batch in audio for item in batch] audio = sum(audio, []) - - + + audio = np.concatenate(audio, 0) - audio = (audio * 32768).astype(np.int16) - + audio = (audio * 32768).astype(np.int16) + try: if speed_factor != 1.0: audio = speed_change(audio, speed=speed_factor, sr=int(sr)) except Exception as e: print(f"Failed to change speed of audio: \n{e}") - + return sr, audio - - - - + + + + def speed_change(input_audio:np.ndarray, speed:float, sr:int): # 将 NumPy 数组转换为原始 PCM 流 raw_audio = input_audio.astype(np.int16).tobytes() diff --git a/GPT_SoVITS/TTS_infer_pack/tts_instance_pool.py b/GPT_SoVITS/TTS_infer_pack/tts_instance_pool.py new file mode 100644 index 00000000..1afb8d9b --- /dev/null +++ b/GPT_SoVITS/TTS_infer_pack/tts_instance_pool.py @@ -0,0 +1,133 @@ +import threading +from time import perf_counter +import traceback +from typing import Dict, Union + +from GPT_SoVITS.TTS_infer_pack.TTS import TTS, TTS_Config + + +class TTSWrapper(TTS): + heat: float = 0 + usage_count: int = 0 + usage_counter: int = 0 + usage_time: float = 0.0 + first_used_time: float = 0.0 + + def __init__(self, configs: Union[dict, str, TTS_Config]): + super(TTSWrapper, self).__init__(configs) + self.first_used_time = perf_counter() + + def __hash__(self) -> int: + return hash(self.first_used_time) + + def run(self, *args, **kwargs): + self.usage_counter += 1 + t0 = perf_counter() + for result in super(TTSWrapper, self).run(*args, **kwargs): + yield result + t1 = perf_counter() + self.usage_time += t1 - t0 + idle_time = self.usage_time - self.first_used_time + self.heat = self.usage_counter / idle_time + + def reset_heat(self): + self.heat: int = 0 + self.usage_count: int = 0 + self.usage_time: float = 0.0 + self.first_used_time: float = perf_counter() + + +class TTSInstancePool: + def __init__(self, max_size): + self.max_size: int = max_size + self.semaphore: threading.Semaphore = threading.Semaphore(max_size) + self.pool_lock: threading.Lock = threading.Lock() + self.pool: Dict[int, TTSWrapper] = dict() + self.current_index: int = 0 + self.size: int = 0 + + def acquire(self, configs: TTS_Config): + + self.semaphore.acquire() + try: + with self.pool_lock: + # 查询最匹配的实例 + indexed_key = None + rank = [] + for key, tts_instance in self.pool.items(): + if tts_instance.configs.vits_weights_path == configs.vits_weights_path \ + and tts_instance.configs.t2s_weights_path == configs.t2s_weights_path: + indexed_key = key + rank.append((tts_instance.heat, key)) + rank.sort(key=lambda x: x[0]) + matched_key = None if len(rank) == 0 else rank[0][1] + + # 如果已有实例匹配，则直接复用 + if indexed_key is not None: + tts_instance = self._reuse_instance(indexed_key, configs) + print(f"如果已有实例匹配，则直接复用: {configs.vits_weights_path} {configs.t2s_weights_path}") + return tts_instance + + # 如果pool未满，则创建一个新实例 + if self.size < self.max_size: + tts_instance = TTSWrapper(configs) + self.size += 1 + print(f"如果pool未满，则创建一个新实例: {configs.vits_weights_path} {configs.t2s_weights_path}") + return tts_instance + else: + # 否则用最合适的实例进行复用 + tts_instance = self._reuse_instance(matched_key, configs) + print(f"否则用最合适的实例进行复用: {configs.vits_weights_path} {configs.t2s_weights_path}") + return tts_instance + except Exception as e: + self.semaphore.release() + traceback.print_exc() + raise e + + def release(self, tts_instance: TTSWrapper): + assert tts_instance is not None + with self.pool_lock: + key = hash(tts_instance) + if key in self.pool.keys(): + return + self.pool[key] = tts_instance + self.semaphore.release() + + def clear_pool(self): + for i in range(self.max_size): + self.semaphore.acquire() + with self.pool_lock: + self.pool.clear() + # for i in range(self.max_size): + self.semaphore.release(self.max_size) + + def _reuse_instance(self, instance_key: int, configs: TTS_Config) -> TTSWrapper: + """ + 复用已有实例 + args: + instance_key: int, 已有实例的Key + config: TTS_Config + return: + TTS_Wrapper: 返回复用的TTS实例 + """ + + # 复用已有实例 + tts_instance = self.pool.pop(instance_key, None) + if tts_instance is None: + raise ValueError("Instance not found") + + tts_instance.configs.device = configs.device + if tts_instance.configs.vits_weights_path != configs.vits_weights_path \ + or tts_instance.configs.t2s_weights_path != configs.t2s_weights_path: + tts_instance.reset_heat() + + if tts_instance.configs.vits_weights_path != configs.vits_weights_path: + tts_instance.init_vits_weights(configs.vits_weights_path, False) + tts_instance.configs.vits_weights_path = configs.vits_weights_path + + if tts_instance.configs.t2s_weights_path != configs.t2s_weights_path: + tts_instance.init_t2s_weights(configs.t2s_weights_path, False) + tts_instance.configs.t2s_weights_path = configs.t2s_weights_path + + tts_instance.set_device(configs.device, False) + return tts_instance diff --git a/GPT_SoVITS/configs/voices/voice1.yaml b/GPT_SoVITS/configs/voices/voice1.yaml new file mode 100644 index 00000000..c5a9c081 --- /dev/null +++ b/GPT_SoVITS/configs/voices/voice1.yaml @@ -0,0 +1,14 @@ +custom: + bert_base_path: GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large + cnhuhbert_base_path: GPT_SoVITS/pretrained_models/chinese-hubert-base + device: cpu + is_half: false + t2s_weights_path: GPT_weights/liyunlong-e15.ckpt + vits_weights_path: SoVITS_weights/liyunlong_e8_s176.pth +default: + bert_base_path: GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large + cnhuhbert_base_path: GPT_SoVITS/pretrained_models/chinese-hubert-base + device: cpu + is_half: false + t2s_weights_path: GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt + vits_weights_path: GPT_SoVITS/pretrained_models/s2G488k.pth diff --git a/GPT_SoVITS/configs/voices/voice2.yaml b/GPT_SoVITS/configs/voices/voice2.yaml new file mode 100644 index 00000000..f7432a8f --- /dev/null +++ b/GPT_SoVITS/configs/voices/voice2.yaml @@ -0,0 +1,14 @@ +custom: + bert_base_path: GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large + cnhuhbert_base_path: GPT_SoVITS/pretrained_models/chinese-hubert-base + device: cpu + is_half: false + t2s_weights_path: GPT_weights/jackma-e10.ckpt + vits_weights_path: SoVITS_weights/jackma_e8_s192.pth +default: + bert_base_path: GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large + cnhuhbert_base_path: GPT_SoVITS/pretrained_models/chinese-hubert-base + device: cpu + is_half: false + t2s_weights_path: GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt + vits_weights_path: GPT_SoVITS/pretrained_models/s2G488k.pth diff --git a/GPT_SoVITS/configs/voices/voice3.yaml b/GPT_SoVITS/configs/voices/voice3.yaml new file mode 100644 index 00000000..add0c797 --- /dev/null +++ b/GPT_SoVITS/configs/voices/voice3.yaml @@ -0,0 +1,14 @@ +custom: + bert_base_path: GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large + cnhuhbert_base_path: GPT_SoVITS/pretrained_models/chinese-hubert-base + device: cpu + is_half: false + t2s_weights_path: GPT_weights/stephenchow-e15.ckpt + vits_weights_path: SoVITS_weights/stephenchow_e8_s112.pth +default: + bert_base_path: GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large + cnhuhbert_base_path: GPT_SoVITS/pretrained_models/chinese-hubert-base + device: cpu + is_half: false + t2s_weights_path: GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt + vits_weights_path: GPT_SoVITS/pretrained_models/s2G488k.pth diff --git a/api_v3.py b/api_v3.py index a121dfc4..01b51f27 100644 --- a/api_v3.py +++ b/api_v3.py @@ -104,6 +104,8 @@ from typing import Generator import torch +from TTS_infer_pack.tts_instance_pool import TTSInstancePool, TTSWrapper + now_dir = os.getcwd() sys.path.append(now_dir) sys.path.append("%s/GPT_SoVITS" % (now_dir)) @@ -119,11 +121,10 @@ from fastapi.responses import JSONResponse from fastapi import FastAPI import uvicorn from io import BytesIO -from GPT_SoVITS.TTS_infer_pack.TTS import TTS, TTS_Config +from GPT_SoVITS.TTS_infer_pack.TTS import TTS_Config from GPT_SoVITS.TTS_infer_pack.text_segmentation_method import get_method_names as get_cut_method_names from fastapi.responses import StreamingResponse from pydantic import BaseModel -from functools import lru_cache cut_method_names = get_cut_method_names() @@ -137,6 +138,9 @@ argv = sys.argv APP = FastAPI() +max_size = 10 +tts_instance_pool = TTSInstancePool(max_size) + class TTS_Request(BaseModel): text: str = None @@ -162,12 +166,6 @@ class TTS_Request(BaseModel): """推理时需要加载的声音模型的yaml配置文件路径，如：GPT_SoVITS/configs/tts_infer.yaml""" -@lru_cache(maxsize=10) -def get_tts_instance(tts_config: TTS_Config) -> TTS: - print(f"load tts config from {tts_config.configs_path}") - return TTS(tts_config) - - def pack_ogg(io_buffer: BytesIO, data: np.ndarray, rate: int): """modify from https://github.com/RVC-Boss/GPT-SoVITS/pull/894/files""" with sf.SoundFile(io_buffer, mode='w', samplerate=rate, channels=1, format='ogg') as audio_file: @@ -318,7 +316,7 @@ async def tts_handle(req: dict): req["return_fragment"] = True try: - tts_instance = get_tts_instance(tts_config) + tts_instance = tts_instance_pool.acquire(tts_config) move_to_gpu(tts_instance, tts_config) @@ -332,27 +330,30 @@ async def tts_handle(req: dict): for sr, chunk in tts_generator: yield pack_audio(BytesIO(), chunk, sr, media_type).getvalue() move_to_cpu(tts_instance) + tts_instance_pool.release(tts_instance) - # _media_type = f"audio/{media_type}" if not (streaming_mode and media_type in ["wav", "raw"]) else f"audio/x-{media_type}" + # _media_type = f"audio/{media_type}" if not (streaming_mode and media_type in ["wav", "raw"]) else + # f"audio/x-{media_type}" return StreamingResponse(streaming_generator(tts_generator, media_type, ), media_type=f"audio/{media_type}") else: sr, audio_data = next(tts_generator) audio_data = pack_audio(BytesIO(), audio_data, sr, media_type).getvalue() move_to_cpu(tts_instance) + tts_instance_pool.release(tts_instance) return Response(audio_data, media_type=f"audio/{media_type}") except Exception as e: return JSONResponse(status_code=400, content={"message": f"tts failed", "Exception": str(e)}) -def move_to_cpu(tts): +def move_to_cpu(tts: TTSWrapper): cpu_device = torch.device('cpu') - tts.set_device(cpu_device) + tts.set_device(cpu_device, False) print("Moved TTS models to CPU to save GPU memory.") -def move_to_gpu(tts: TTS, tts_config: TTS_Config): - tts.set_device(tts_config.device) +def move_to_gpu(tts: TTSWrapper, tts_config: TTS_Config): + tts.set_device(tts_config.device, False) print("Moved TTS models back to GPU for performance.") @@ -422,7 +423,7 @@ async def tts_post_endpoint(request: TTS_Request): async def set_refer_audio(refer_audio_path: str = None, tts_infer_yaml_path: str = "GPT_SoVITS/configs/tts_infer.yaml"): try: tts_config = TTS_Config(tts_infer_yaml_path) - tts_instance = get_tts_instance(tts_config) + tts_instance = tts_instance_pool.acquire(tts_config) tts_instance.set_ref_audio(refer_audio_path) except Exception as e: return JSONResponse(status_code=400, content={"message": f"set refer audio failed", "Exception": str(e)}) @@ -436,7 +437,7 @@ async def set_gpt_weights(weights_path: str = None, tts_infer_yaml_path: str = " return JSONResponse(status_code=400, content={"message": "gpt weight path is required"}) tts_config = TTS_Config(tts_infer_yaml_path) - tts_instance = get_tts_instance(tts_config) + tts_instance = tts_instance_pool.acquire(tts_config) tts_instance.init_t2s_weights(weights_path) except Exception as e: return JSONResponse(status_code=400, content={"message": f"change gpt weight failed", "Exception": str(e)}) @@ -451,7 +452,7 @@ async def set_sovits_weights(weights_path: str = None, tts_infer_yaml_path: str return JSONResponse(status_code=400, content={"message": "sovits weight path is required"}) tts_config = TTS_Config(tts_infer_yaml_path) - tts_instance = get_tts_instance(tts_config) + tts_instance = tts_instance_pool.acquire(tts_config) tts_instance.init_vits_weights(weights_path) except Exception as e: return JSONResponse(status_code=400, content={"message": f"change sovits weight failed", "Exception": str(e)})