Merge 7604f36bb270d0a897df0b8d4dd9d35f860d06cb into 9ec3a60f30d228719e5ec6cd6796c5b2d888dd1a

* Add ModelScope Snapshot Download For ASR

* Typo Fix

* Remove YUE in whisper

* Remove HF ENDPOINT

* Add FunASR Download

GPT_SoVITS/inference_webui.py

@@ -27,11 +27,14 @@ import re
 import sys
 import traceback
 import warnings
+import soundfile  # 新增导入
 
 import torch
 import torchaudio
 from text.LangSegmenter import LangSegmenter
 
+
+
 logging.getLogger("markdown_it").setLevel(logging.ERROR)
 logging.getLogger("urllib3").setLevel(logging.ERROR)
 logging.getLogger("httpcore").setLevel(logging.ERROR)
@@ -1001,6 +1004,255 @@ def get_tts_wav(
     yield opt_sr, (audio_opt * 32767).astype(np.int16)
 
 
+import uuid
+import shutil
+from pydub import AudioSegment
+
+TEMP_FOLDER = "TEMP"  # 临时文件夹路径
+os.makedirs(TEMP_FOLDER, exist_ok=True)
+
+def clean_temp_folder_on_startup():
+    """启动时清理临时文件夹"""
+    try:
+        if os.path.exists(TEMP_FOLDER):
+            shutil.rmtree(TEMP_FOLDER)
+            os.makedirs(TEMP_FOLDER, exist_ok=True)
+            print("启动时已清理临时文件夹")
+        else:
+            os.makedirs(TEMP_FOLDER, exist_ok=True)
+            print("临时文件夹已创建")
+    except Exception as e:
+        print(f"启动时清理临时文件夹失败: {str(e)}")
+
+def split_text_by_punctuation(text, period_pause=0.3, comma_pause=0.15):
+    """改进的文本分割函数"""
+    if not text or not isinstance(text, str):
+        print("收到空或非字符串文本输入")
+        return []
+    
+    segments = []
+    current_segment = ""
+    punctuation_marks = ['，', ',', '。', '.']
+    
+    for char in text:
+        current_segment += char
+        if char in punctuation_marks:
+            # 根据标点类型设置停顿时间
+            pause = period_pause if char in ['。', '.'] else comma_pause
+            segments.append({
+                "text": current_segment.strip(),
+                "pause": pause,
+                "punctuation": char
+            })
+            current_segment = ""
+    
+    if current_segment:
+        segments.append({
+            "text": current_segment.strip(),
+            "pause": comma_pause,  # 默认使用非句号停顿时间
+            "punctuation": ""
+        })
+    
+    print(f"分割结果: {[seg['text'] for seg in segments]}")
+    return segments
+
+def generate_segment_audio(segment_data, ref_wav_path, prompt_text, prompt_language, text_language, top_k, top_p, temperature):
+    """增强的音频生成函数"""
+    try:
+        if not os.path.exists(ref_wav_path):
+            raise FileNotFoundError(f"参考音频不存在: {ref_wav_path}")
+        
+        # 生成音频
+        sr, audio_data = next(get_tts_wav(
+            ref_wav_path=ref_wav_path,
+            prompt_text=prompt_text,
+            prompt_language=prompt_language,
+            text=segment_data["text"],
+            text_language=text_language,
+            how_to_cut=i18n("不切"),
+            top_k=top_k,
+            top_p=top_p,
+            temperature=temperature,
+            pause_second=0  # 不在内部添加停顿
+        ))
+        
+        # 这里不再生成ID，由调用者提供
+        temp_path = os.path.join(TEMP_FOLDER, "temp_generate.wav")
+        soundfile.write(temp_path, audio_data, sr)
+        
+        # 添加停顿
+        audio = AudioSegment.from_wav(temp_path)
+        pause = AudioSegment.silent(duration=int(segment_data["pause"]*1000))
+        final_audio = audio + pause
+        final_audio.export(temp_path, format="wav")
+        
+        return {
+            "success": True,
+            "audio_path": temp_path,
+            "text": segment_data["text"],
+            "pause": segment_data["pause"],
+            "message": "生成成功"
+        }
+    except Exception as e:
+        print(f"生成片段失败: {str(e)}", exc_info=True)
+        return {
+            "success": False,
+            "audio_path": None,
+            "text": segment_data["text"],
+            "pause": segment_data["pause"],
+            "message": f"生成失败: {str(e)}"
+        }
+
+def process_all_segments(text, ref_wav_path, prompt_text, prompt_language, text_language, 
+                        top_k, top_p, temperature, period_pause, comma_pause):
+    """完整处理流程"""
+    # 输入验证
+    if not text or not isinstance(text, str):
+        error_msg = "输入文本无效"
+        print(error_msg)
+        return [[1, error_msg, "错误"]], None
+    
+    if not os.path.exists(ref_wav_path):
+        error_msg = f"参考音频不存在: {ref_wav_path}"
+        print(error_msg)
+        return [[1, error_msg, "错误"]], None
+    
+    # 处理分段
+    segments = split_text_by_punctuation(text, period_pause, comma_pause)
+    if not segments:
+        error_msg = "无法分割文本"
+        print(error_msg)
+        return [[1, error_msg, "错误"]], None
+    
+    results = []
+    audio_files = []
+    
+    # 修改这里：使用enumerate从1开始编号，而不是基于文件夹内容
+    for i, segment in enumerate(segments, 1):
+        result = generate_segment_audio(
+            segment, ref_wav_path, prompt_text,
+            prompt_language, text_language, top_k, top_p, temperature
+        )
+        
+        # 更新结果中的segment_id
+        result["segment_id"] = f"{i}temp"
+        if result["success"] and result["audio_path"]:
+            # 重命名文件以匹配新的编号
+            new_path = os.path.join(TEMP_FOLDER, f"{result['segment_id']}.wav")
+            os.rename(result["audio_path"], new_path)
+            result["audio_path"] = new_path
+            audio_files.append(new_path)
+        
+        results.append(result)
+        print(f"处理进度: {i}/{len(segments)} - {result['message']}")
+
+    # 准备显示数据
+    df_data = []
+    for i, result in enumerate(results, 1):
+        df_data.append([
+            f"{i}temp",
+            result["text"],
+            result["message"]
+        ])
+    
+    first_audio = audio_files[0] if audio_files else None
+    return df_data, first_audio
+
+def regenerate_segment(segment_id, new_text, ref_wav_path, prompt_text, 
+                      prompt_language, text_language, top_k, top_p, temperature,
+                      period_pause, comma_pause):
+    try:
+        if not segment_id or not new_text:
+            raise ValueError("缺少片段ID或新文本内容")
+        
+        # 从文件名解析原始停顿时间
+        try:
+            pause = 0.25 if segment_id.endswith(("。", ".")) else 0.1
+        except:
+            pause = 0.1  # 默认值
+        
+        is_period = segment_id.endswith(("。", "."))
+        pause = period_pause if is_period else comma_pause
+        
+        segment_data = {
+            "text": new_text,
+            "pause": pause,
+            "punctuation": "。" if is_period else "，"
+        }
+        result = generate_segment_audio(
+            segment_data, ref_wav_path, prompt_text,
+            prompt_language, text_language, top_k, top_p, temperature
+        )
+        
+        # 更新文件
+        if result["success"]:
+            old_path = os.path.join(TEMP_FOLDER, f"{segment_id}.wav")
+            if os.path.exists(old_path):
+                os.remove(old_path)
+            os.rename(result["audio_path"], old_path)
+            result["audio_path"] = old_path
+        
+        return (
+            result["audio_path"],
+            segment_id,
+            result["message"]
+        )
+    except Exception as e:
+        print(f"重新生成片段失败: {str(e)}", exc_info=True)
+        return None, segment_id, f"重新生成失败: {str(e)}"
+
+def merge_all_segments():
+    try:
+        # 获取并按编号排序片段
+        segments = sorted(
+            [f for f in os.listdir(TEMP_FOLDER) if f.endswith(".wav") and f != "final_output.wav"],
+            key=lambda x: int(x.split("temp")[0])
+        )
+        if not segments:
+            raise ValueError("没有找到可合并的音频片段")
+        
+        combined = AudioSegment.empty()
+        for seg in segments:
+            seg_path = os.path.join(TEMP_FOLDER, seg)
+            audio = AudioSegment.from_wav(seg_path)
+            combined += audio
+        
+        # 保存最终结果
+        output_path = os.path.join(TEMP_FOLDER, "final_output.wav")
+        combined.export(output_path, format="wav")
+        
+        print(f"成功合并 {len(segments)} 个片段")
+        return output_path, "合并成功"
+    except Exception as e:
+        print(f"合并片段失败: {str(e)}", exc_info=True)
+        return None, f"合并失败: {str(e)}"
+
+def clean_temp_files():
+    """清理临时文件函数"""
+    try:
+        for filename in os.listdir(TEMP_FOLDER):
+            file_path = os.path.join(TEMP_FOLDER, filename)
+            try:
+                if os.path.isfile(file_path):
+                    os.unlink(file_path)
+            except Exception as e:
+                print(f"删除文件 {file_path} 失败: {e}")
+        return "临时文件已清理"
+    except Exception as e:
+        return f"清理失败: {str(e)}"
+
+def on_segment_select(df, evt: gr.SelectData):
+    """当选择分段列表中的项目时更新显示"""
+    if evt.index:
+        selected_row = df.iloc[evt.index[0]]
+        audio_path = os.path.join(TEMP_FOLDER, f"{selected_row['编号']}.wav")
+        return (
+            selected_row["编号"],
+            selected_row["文本内容"],
+            audio_path if os.path.exists(audio_path) else None
+        )
+    return "1temp", "", None
+
 def split(todo_text):
     todo_text = todo_text.replace("……", "。").replace("——", "，")
     if todo_text[-1] not in splits:
@@ -1018,6 +1270,236 @@ def split(todo_text):
         else:
             i_split_head += 1
     return todo_texts
+# ======================== 合并功能实现 ========================
+def merge_selected_segments(merge_range, segment_list_data, ref_wav_path, prompt_text, 
+                           prompt_language, text_language, top_k, top_p, temperature,
+                           pause_period, pause_comma):
+    """合并选中的句子并立即生成新音频"""
+    try:
+        if not merge_range:
+            return segment_list_data, "请输入合并范围（例如：1-3）", None
+        
+        # 解析合并范围
+        start, end = map(int, merge_range.split('-'))
+        if start <= 0 or end <= 0 or start > end:
+            return segment_list_data, "无效的合并范围", None
+        
+        # 检查范围是否有效
+        if end > len(segment_list_data):
+            return segment_list_data, f"结束编号 {end} 超过总段数 {len(segment_list_data)}", None
+        
+        # === 第一步：收集要删除的文件并立即删除 ===
+        files_to_delete = []
+        for i in range(start-1, end):
+            file_id = segment_list_data.iloc[i, 0]
+            file_path = os.path.join(TEMP_FOLDER, f"{file_id}.wav")
+            if os.path.exists(file_path):
+                os.remove(file_path)
+                files_to_delete.append(file_id)
+        
+        # === 第二步：合并文本 ===
+        merged_text = ""
+        for i in range(start-1, end):
+            merged_text += segment_list_data.iloc[i, 1]  # 文本内容在第二列
+        
+        # 确定合并后的停顿类型（取最后一个片段的标点）
+        last_punctuation = segment_list_data.iloc[end-1, 1][-1] if segment_list_data.iloc[end-1, 1] else ""
+        is_period = last_punctuation in ["。", "."]
+        pause = pause_period if is_period else pause_comma
+        
+        # === 第三步：立即生成合并后的音频 ===
+        segment_data = {
+            "text": merged_text,
+            "pause": pause,
+            "punctuation": last_punctuation
+        }
+        # 生成音频
+        result = generate_segment_audio(
+            segment_data, ref_wav_path, prompt_text,
+            prompt_language, text_language, top_k, top_p, temperature
+        )
+        if not result["success"]:
+            return segment_list_data, result["message"], None
+        
+        # === 第四步：构建新的分段列表 ===
+        # 创建新的合并条目 - 使用起始编号
+        new_id = f"{start}temp"
+        merged_entry = [new_id, merged_text, "已生成"]
+        
+        # 移动生成的音频到正确位置
+        new_path = os.path.join(TEMP_FOLDER, f"{new_id}.wav")
+        shutil.move(result["audio_path"], new_path)
+        
+        # 构建新的分段列表
+        new_segment_list = []
+        
+        # 添加合并前的部分
+        if start > 1:
+            new_segment_list.extend(segment_list_data.iloc[:start-1].values.tolist())
+        
+        # 添加合并条目
+        new_segment_list.append(merged_entry)
+        
+        # 添加合并后的部分
+        if end < len(segment_list_data):
+            new_segment_list.extend(segment_list_data.iloc[end:].values.tolist())
+        
+        # === 第五步：重新编号 ===
+        reindexed_list = []
+        new_id_counter = 1
+        for segment in new_segment_list:
+            old_id = segment[0]
+            # 为新列表生成连续编号
+            new_id = f"{new_id_counter}temp"
+            
+            # 重命名文件（如果存在）
+            old_path = os.path.join(TEMP_FOLDER, f"{old_id}.wav")
+            new_path = os.path.join(TEMP_FOLDER, f"{new_id}.wav")
+            
+            if os.path.exists(old_path) and old_id != new_id:
+                os.rename(old_path, new_path)
+            
+            # 更新ID
+            segment[0] = new_id
+            reindexed_list.append(segment)
+            new_id_counter += 1
+        
+        return reindexed_list, "合并成功", new_id
+    
+    except Exception as e:
+        traceback.print_exc()
+        return segment_list_data, f"合并失败: {str(e)}", None
+
+# ======================== 实现拆分逻辑函数 ========================
+def split_selected_segment(split_id, segment_list_data, ref_wav_path, prompt_text, 
+                         prompt_language, text_language, top_k, top_p, temperature,
+                         pause_period, pause_comma):
+    """拆分选中的句子并重新生成 - 使用倒序重命名避免冲突"""
+    try:
+        if not split_id:
+            return segment_list_data, "请输入要拆分的句子编号", None
+        
+        # 从文件名解析原始停顿时间
+        try:
+            pause = 0.25 if split_id.endswith(("。", ".")) else 0.1
+        except:
+            pause = 0.1  # 默认值
+        
+        # 查找要拆分的句子
+        df = segment_list_data
+        target_row = None
+        target_idx = None
+        
+        # 查找匹配的句子
+        for idx, row in enumerate(df.itertuples()):
+            if str(row[1]) == split_id:  # 第一列是ID
+                target_row = row
+                target_idx = idx
+                break
+        
+        if target_row is None:
+            return segment_list_data, f"未找到编号为 {split_id} 的句子", None
+        
+        # 获取原始文本
+        original_text = target_row[2]  # 第二列是文本
+        
+        # 根据标点符号拆分文本
+        segments = []
+        current_segment = ""
+        punctuation_marks = ['，', ',', '。', '.', '？', '?', '！', '!', '；', ';']
+        
+        for char in original_text:
+            current_segment += char
+            if char in punctuation_marks:
+                # 根据标点类型设置停顿时间
+                pause = pause_period if char in ['。', '.', '？', '?', '！', '!'] else pause_comma
+                segments.append({
+                    "text": current_segment.strip(),
+                    "pause": pause,
+                    "punctuation": char
+                })
+                current_segment = ""
+        
+        if current_segment:
+            segments.append({
+                "text": current_segment.strip(),
+                "pause": pause_comma,  # 默认使用非句号停顿时间
+                "punctuation": ""
+            })
+        
+        if len(segments) <= 1:
+            return segment_list_data, "句子无法拆分（没有标点符号）", None
+        
+        # 计算拆分后新增的句子数量
+        num_new_segments = len(segments)
+        offset = num_new_segments - 1  # 拆分后增加的句子数
+        
+        # === 第一步：删除原始音频文件 ===
+        original_path = os.path.join(TEMP_FOLDER, f"{split_id}.wav")
+        if os.path.exists(original_path):
+            os.remove(original_path)
+        
+        # === 第二步：倒序重命名后续句子避免冲突 ===
+        new_segment_list = []
+        
+        # 添加拆分前的句子
+        for i in range(target_idx):
+            new_segment_list.append(df.iloc[i].tolist())
+        
+        # 倒序重命名从最后一个句子开始，避免冲突
+        total_segments = len(df)
+        
+        # 从最后一个句子开始倒序遍历
+        for i in range(total_segments - 1, target_idx, -1):
+            old_id = df.iloc[i, 0]
+            old_num = int(old_id.replace("temp", ""))
+            new_id_num = old_num + offset
+            new_id = f"{new_id_num}temp"
+            
+            # 重命名音频文件
+            old_path = os.path.join(TEMP_FOLDER, f"{old_id}.wav")
+            new_path = os.path.join(TEMP_FOLDER, f"{new_id}.wav")
+            if os.path.exists(old_path):
+                os.rename(old_path, new_path)
+            
+            # 更新列表条目
+            new_segment_list.append([new_id, df.iloc[i, 1], df.iloc[i, 2]])
+        
+        # === 第三步：添加拆分后的新句子 ===
+        for i, segment in enumerate(segments):
+            new_id = f"{target_idx+1+i}temp"  # 新ID从原始位置开始
+            new_segment_list.append([new_id, segment["text"], "待生成"])
+        
+        # === 第四步：重新排序列表 ===
+        # 按照ID中的数字排序
+        new_segment_list.sort(key=lambda x: int(x[0].replace("temp", "")))
+        
+        # === 第五步：生成拆分后的新句子 ===
+        for i, segment in enumerate(segments):
+            segment_id = f"{target_idx+1+i}temp"
+            result = generate_segment_audio(
+                segment, ref_wav_path, prompt_text,
+                prompt_language, text_language, top_k, top_p, temperature
+            )
+            
+            if result["success"]:
+                # 更新文件
+                new_path = os.path.join(TEMP_FOLDER, f"{segment_id}.wav")
+                if os.path.exists(result["audio_path"]):
+                    os.rename(result["audio_path"], new_path)
+                
+                # 更新状态
+                for j, item in enumerate(new_segment_list):
+                    if item[0] == segment_id:
+                        new_segment_list[j][2] = "已生成"
+                        break
+        
+        return new_segment_list, f"成功拆分为 {num_new_segments} 个句子", f"{target_idx+1}temp"
+    
+    except Exception as e:
+        traceback.print_exc()
+        return segment_list_data, f"拆分失败: {str(e)}", None
+
 
 
 def cut1(inp):
@@ -1327,6 +1809,160 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
         )
         GPT_dropdown.change(change_gpt_weights, [GPT_dropdown], [])
 
+
+
+        # ======================== 插入分段合成UI开始 ========================
+    with gr.Tab(i18n("分段合成模式")):
+        with gr.Row():
+            with gr.Column():
+                segmented_text = gr.Textbox(label=i18n("需要分段的文本"), lines=10, value="")
+                segment_button = gr.Button(i18n("分割文本并生成所有片段"), variant="primary")
+                segmented_output = gr.Audio(label=i18n("当前选中片段"), interactive=False)
+                
+                with gr.Row():
+                    segment_index = gr.Textbox(label=i18n("片段编号"), interactive=False, visible=False)
+                    new_segment_text = gr.Textbox(label=i18n("新文本内容"), lines=2, max_lines=4)
+                    regenerate_button = gr.Button(i18n("重新生成当前片段"), variant="primary")
+                with gr.Row():
+                    pause_period = gr.Slider(
+                        minimum=0.1, 
+                        maximum=1.0, 
+                        step=0.05, 
+                        label=i18n("句号停顿时间(秒)"), 
+                        value=0.3,
+                        interactive=True
+                    )
+                    pause_comma = gr.Slider(
+                        minimum=0.1, 
+                        maximum=0.5, 
+                        step=0.05, 
+                        label=i18n("非句号停顿时间(秒)"), 
+                        value=0.15,
+                        interactive=True
+                    )
+                with gr.Row():
+                    clean_button = gr.Button(i18n("清理临时文件"), variant="secondary", scale=1)
+                    confirm_button = gr.Button(i18n("确认并合并所有片段"), variant="primary", scale=2)
+                
+                final_output = gr.Audio(label=i18n("最终合成结果"), interactive=False)
+                segment_status = gr.Textbox(label=i18n("状态"), interactive=False)
+                # 添加合并功能
+                with gr.Row():
+                    merge_range = gr.Textbox(label=i18n("合并范围（例如：1-3）"), scale=3)
+                    merge_button = gr.Button(i18n("合并句子"), variant="primary", scale=1)
+                # 添加拆分功能控件
+                with gr.Row():
+                    split_index = gr.Textbox(label=i18n("要拆分的句子编号"), scale=1)
+                    split_button = gr.Button(i18n("拆分句子"), variant="primary", scale=1)
+            with gr.Column():
+                segment_list = gr.Dataframe(
+                    headers=["编号", "文本内容", "状态"],
+                    datatype=["str", "str", "str"],
+                    interactive=False,
+                    label=i18n("分段列表"),
+                    value=[]
+                )
+                # 在分段合成UI部分添加以下代码（大约在line 3200附近）
+
+    # ======================== 插入分段合成UI结束 ========================
+
+    # ======================== 插入分段合成事件绑定开始 ========================
+    # 分割文本并生成所有片段
+    segment_button.click(
+        process_all_segments,
+        inputs=[
+            segmented_text, 
+            inp_ref, 
+            prompt_text, 
+            prompt_language, 
+            text_language, 
+            top_k, 
+            top_p, 
+            temperature,
+            pause_period,  # 新增句号停顿参数
+            pause_comma    # 新增非句号停顿参数
+        ],
+        outputs=[segment_list, segmented_output]
+    )
+    # 重新生成当前片段
+    regenerate_button.click(
+        regenerate_segment,
+        inputs=[
+            segment_index,  # 第一个参数应该是segment_id
+            new_segment_text, 
+            inp_ref, 
+            prompt_text, 
+            prompt_language, 
+            text_language, 
+            top_k, 
+            top_p, 
+            temperature,
+            pause_period,
+            pause_comma
+        ],
+        outputs=[segmented_output, segment_index, segment_status]
+    )
+
+    # 合并所有片段
+    confirm_button.click(
+        merge_all_segments,
+        inputs=[],
+        outputs=[final_output, segment_status]
+    )
+
+    # 清理临时文件
+    clean_button.click(
+        clean_temp_files,
+        inputs=[],
+        outputs=[segment_status]
+    )
+
+    # 当选择分段列表中的项目时
+    segment_list.select(
+        fn=on_segment_select,
+        inputs=[segment_list],
+        outputs=[segment_index, new_segment_text, segmented_output],
+    )
+    merge_button.click(
+        fn=merge_selected_segments,
+        inputs=[
+            merge_range, 
+            segment_list,
+            inp_ref,
+            prompt_text,
+            prompt_language,
+            text_language,
+            top_k,
+            top_p,
+            temperature,
+            pause_period,
+            pause_comma
+        ],
+        outputs=[segment_list, segment_status, segment_index]
+    )
+    # ======================== 添加拆分按钮的事件绑定 ========================
+    split_button.click(
+        fn=split_selected_segment,
+        inputs=[
+            split_index, 
+            segment_list,
+            inp_ref,
+            prompt_text,
+            prompt_language,
+            text_language,
+            top_k,
+            top_p,
+            temperature,
+            pause_period,
+            pause_comma
+        ],
+        outputs=[segment_list, segment_status, segment_index]
+    )
+    # ======================== 插入分段合成事件绑定结束 ========================
+
+
+
+
         # gr.Markdown(value=i18n("文本切分工具。太长的文本合成出来效果不一定好，所以太长建议先切。合成会根据文本的换行分开合成再拼起来。"))
         # with gr.Row():
         #     text_inp = gr.Textbox(label=i18n("需要合成的切分前文本"), value="")
@@ -1351,3 +1987,4 @@ if __name__ == "__main__":
         server_port=infer_ttswebui,
         # quiet=True,
     )
+

GPT_SoVITS/module/core_vq.py

@@ -37,6 +37,10 @@ from einops import rearrange, repeat
 import torch
 from torch import nn
 import torch.nn.functional as F
+import torch.distributed as dist
+
+from module.distrib import broadcast_tensors, is_distributed
+from module.ddp_utils import SyncFunction
 from tqdm import tqdm
 
 
@@ -69,27 +73,40 @@ def sample_vectors(samples, num: int):
     return samples[indices]
 
 
-def kmeans(samples, num_clusters: int, num_iters: int = 10):
-    dim, dtype = samples.shape[-1], samples.dtype
-    max_kmeans_samples = 500
-    samples = samples[:max_kmeans_samples, :]
+def kmeans(samples, num_clusters: int, num_iters: int = 10, frames_to_use: int = 10_000, batch_size: int = 64):
+    N, D = samples.shape
+    dtype, device = samples.dtype, samples.device
+
+    if frames_to_use < N:
+        indices = torch.randperm(N, device=device)[:frames_to_use]
+        samples = samples[indices]
+
     means = sample_vectors(samples, num_clusters)
 
     print("kmeans start ... ")
     for _ in tqdm(range(num_iters)):
-        diffs = rearrange(samples, "n d -> n () d") - rearrange(means, "c d -> () c d")
-        dists = -(diffs**2).sum(dim=-1)
+        # Store cluster assignments
+        all_assignments = []
 
-        buckets = dists.max(dim=-1).indices
+        for i in range(0, samples.shape[0], batch_size):
+            batch = samples[i : i + batch_size]  # [B, D]
+            dists = torch.cdist(batch, means, p=2)  # [B, C]
+            assignments = dists.argmin(dim=1)  # [B]
+            all_assignments.append(assignments)
+
+        buckets = torch.cat(all_assignments, dim=0)  # [N]
         bins = torch.bincount(buckets, minlength=num_clusters)
         zero_mask = bins == 0
         bins_min_clamped = bins.masked_fill(zero_mask, 1)
 
-        new_means = buckets.new_zeros(num_clusters, dim, dtype=dtype)
-        new_means.scatter_add_(0, repeat(buckets, "n -> n d", d=dim), samples)
-        new_means = new_means / bins_min_clamped[..., None]
+        # Compute new means
+        new_means = torch.zeros_like(means)
+        for i in range(num_clusters):
+            mask = buckets == i
+            if mask.any():
+                new_means[i] = samples[mask].mean(dim=0)
 
-        means = torch.where(zero_mask[..., None], means, new_means)
+        means = torch.where(zero_mask[:, None], means, new_means)
 
     return means, bins
 
@@ -141,13 +158,24 @@ class EuclideanCodebook(nn.Module):
         if self.inited:
             return
 
-        embed, cluster_size = kmeans(data, self.codebook_size, self.kmeans_iters)
+        if dist.is_available() and dist.is_initialized():
+            # [B * T * world_size, D]
+            data = SyncFunction.apply(data)
+
+        if dist.get_rank() == 0:
+            embed, cluster_size = kmeans(data, self.codebook_size, self.kmeans_iters)
+        else:
+            embed = torch.empty_like(self.embed)
+            cluster_size = torch.empty_like(self.cluster_size)
+        dist.broadcast(embed, src=0)
+        dist.broadcast(cluster_size, src=0)
+
         self.embed.data.copy_(embed)
         self.embed_avg.data.copy_(embed.clone())
         self.cluster_size.data.copy_(cluster_size)
         self.inited.data.copy_(torch.Tensor([True]))
         # Make sure all buffers across workers are in sync after initialization
-        # broadcast_tensors(self.buffers())
+        broadcast_tensors(self.buffers())
 
     def replace_(self, samples, mask):
         modified_codebook = torch.where(mask[..., None], sample_vectors(samples, self.codebook_size), self.embed)
@@ -161,9 +189,17 @@ class EuclideanCodebook(nn.Module):
         if not torch.any(expired_codes):
             return
 
-        batch_samples = rearrange(batch_samples, "... d -> (...) d")
-        self.replace_(batch_samples, mask=expired_codes)
-        # broadcast_tensors(self.buffers())
+        if is_distributed():
+            # [B * T * world_size, D]
+            batch_samples = SyncFunction.apply(batch_samples)
+
+        if dist.get_rank() == 0:
+            new_embeds = sample_vectors(batch_samples, expired_codes.sum())
+        else:
+            new_embeds = torch.zeros(expired_codes.sum(), self.embed.size(1), device=self.embed.device)
+        dist.broadcast(new_embeds, src=0)
+        self.embed.data[expired_codes] = new_embeds
+        broadcast_tensors(self.buffers())
 
     def preprocess(self, x):
         x = rearrange(x, "... d -> (...) d")
@@ -208,17 +244,26 @@ class EuclideanCodebook(nn.Module):
         quantize = self.dequantize(embed_ind)
 
         if self.training:
+            ### Update codebook by EMA
+            embed_onehot_sum = embed_onehot.sum(0)  # [cb-size,]
+            embed_sum = x.t() @ embed_onehot  # [D, cb-size]
+            if is_distributed():
+                dist.all_reduce(embed_onehot_sum)
+                dist.all_reduce(embed_sum)
+            # Update ema cluster count N_i^t, eq. (6) in vqvae paper
+            self.cluster_size.data.mul_(self.decay).add_(embed_onehot_sum, alpha=1 - self.decay)
+            # Update ema embed: eq. (7) in vqvae paper
+            self.embed_avg.data.mul_(self.decay).add_(embed_sum.t(), alpha=1 - self.decay)
+            # apply laplace smoothing
+            n = self.cluster_size.sum()
+            cluster_size = (self.cluster_size + self.epsilon) / (n + self.codebook_size * self.epsilon) * n
+            # Update ema embed: eq. (8) in vqvae paper
+            embed_normalized = self.embed_avg / cluster_size.unsqueeze(1)
+            self.embed.data.copy_(embed_normalized)
+
             # We do the expiry of code at that point as buffers are in sync
             # and all the workers will take the same decision.
             self.expire_codes_(x)
-            ema_inplace(self.cluster_size, embed_onehot.sum(0), self.decay)
-            embed_sum = x.t() @ embed_onehot
-            ema_inplace(self.embed_avg, embed_sum.t(), self.decay)
-            cluster_size = (
-                laplace_smoothing(self.cluster_size, self.codebook_size, self.epsilon) * self.cluster_size.sum()
-            )
-            embed_normalized = self.embed_avg / cluster_size.unsqueeze(1)
-            self.embed.data.copy_(embed_normalized)
 
         return quantize, embed_ind
 

GPT_SoVITS/module/ddp_utils.py

@@ -0,0 +1,181 @@
+import torch
+from torch.nn.parallel import DistributedDataParallel
+from torch.nn.parallel.distributed import _find_tensors
+from packaging import version
+
+
+# from https://github.com/Lightning-AI/lightning-bolts/blob/5d61197cd2f491f69e238137a5edabe80ae14ad9/pl_bolts/models/self_supervised/simclr/simclr_module.py#L20
+class SyncFunction(torch.autograd.Function):
+    @staticmethod
+    # @torch.no_grad()
+    def forward(ctx, tensor):
+        world_size = torch.distributed.get_world_size()
+
+        # Collect batch sizes from all processes
+        local_bs = torch.tensor([tensor.shape[0]], device=tensor.device)
+        batch_sizes = [torch.zeros_like(local_bs) for _ in range(world_size)]
+        torch.distributed.all_gather(batch_sizes, local_bs)
+
+        # Convert to integer list and find the minimum
+        batch_sizes_int = [bs.item() for bs in batch_sizes]
+        min_bs = min(batch_sizes_int)
+
+        # Crop the tensor to the minimum batch size if needed
+        cropped_tensor = tensor[:min_bs] if tensor.shape[0] > min_bs else tensor
+
+        # Prepare for gathering
+        out_shape = (min_bs * world_size,) + tensor.shape[1:]
+        gathered_tensor = torch.zeros(out_shape, dtype=tensor.dtype, device=tensor.device)
+
+        # Build tensor list for all_gather
+        tensor_list = list(torch.chunk(gathered_tensor, world_size))
+
+        # Perform all_gather using the cropped tensors
+        torch.distributed.all_gather(tensor_list, cropped_tensor)
+
+        # Save for backward pass
+        ctx.min_bs = min_bs
+        ctx.world_size = world_size
+        ctx.orig_shape = tensor.shape
+
+        return gathered_tensor
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        assert False
+        grad_input = grad_output.clone()
+        torch.distributed.all_reduce(grad_input, op=torch.distributed.ReduceOp.SUM, async_op=False)
+
+        idx_from = torch.distributed.get_rank() * ctx.batch_size
+        idx_to = (torch.distributed.get_rank() + 1) * ctx.batch_size
+        return grad_input[idx_from:idx_to]
+
+class DDP(DistributedDataParallel):
+    """
+    Override the forward call in lightning so it goes to training and validation step respectively
+    """
+
+    def forward(self, *inputs, **kwargs):  # pragma: no cover
+        if version.parse(torch.__version__[:6]) < version.parse("1.11"):
+            self._sync_params()
+            inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
+            assert len(self.device_ids) == 1
+            if self.module.training:
+                output = self.module.training_step(*inputs[0], **kwargs[0])
+            elif self.module.testing:
+                output = self.module.test_step(*inputs[0], **kwargs[0])
+            else:
+                output = self.module.validation_step(*inputs[0], **kwargs[0])
+            if torch.is_grad_enabled():
+                # We'll return the output object verbatim since it is a freeform
+                # object. We need to find any tensors in this object, though,
+                # because we need to figure out which parameters were used during
+                # this forward pass, to ensure we short circuit reduction for any
+                # unused parameters. Only if `find_unused_parameters` is set.
+                if self.find_unused_parameters:
+                    self.reducer.prepare_for_backward(list(_find_tensors(output)))
+                else:
+                    self.reducer.prepare_for_backward([])
+        else:
+            from torch.nn.parallel.distributed import (
+                Join,
+                _DDPSink,
+                _tree_flatten_with_rref,
+                _tree_unflatten_with_rref,
+            )
+
+            with torch.autograd.profiler.record_function("DistributedDataParallel.forward"):
+                if torch.is_grad_enabled() and self.require_backward_grad_sync:
+                    self.logger.set_runtime_stats_and_log()
+                    self.num_iterations += 1
+                    self.reducer.prepare_for_forward()
+
+                # Notify the join context that this process has not joined, if
+                # needed
+                work = Join.notify_join_context(self)
+                if work:
+                    self.reducer._set_forward_pass_work_handle(work, self._divide_by_initial_world_size)
+
+                # Calling _rebuild_buckets before forward compuation,
+                # It may allocate new buckets before deallocating old buckets
+                # inside _rebuild_buckets. To save peak memory usage,
+                # call _rebuild_buckets before the peak memory usage increases
+                # during forward computation.
+                # This should be called only once during whole training period.
+                if torch.is_grad_enabled() and self.reducer._rebuild_buckets():
+                    print("Reducer buckets have been rebuilt in this iteration.")
+                    self._has_rebuilt_buckets = True
+
+                # sync params according to location (before/after forward) user
+                # specified as part of hook, if hook was specified.
+                buffer_hook_registered = hasattr(self, "buffer_hook")
+                if self._check_sync_bufs_pre_fwd():
+                    self._sync_buffers()
+
+                if self._join_config.enable:
+                    # Notify joined ranks whether they should sync in backwards pass or not.
+                    self._check_global_requires_backward_grad_sync(is_joined_rank=False)
+
+                inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
+                if self.module.training:
+                    output = self.module.training_step(*inputs[0], **kwargs[0])
+                elif self.module.testing:
+                    output = self.module.test_step(*inputs[0], **kwargs[0])
+                else:
+                    output = self.module.validation_step(*inputs[0], **kwargs[0])
+
+                # sync params according to location (before/after forward) user
+                # specified as part of hook, if hook was specified.
+                if self._check_sync_bufs_post_fwd():
+                    self._sync_buffers()
+
+                if torch.is_grad_enabled() and self.require_backward_grad_sync:
+                    self.require_forward_param_sync = True
+                    # We'll return the output object verbatim since it is a freeform
+                    # object. We need to find any tensors in this object, though,
+                    # because we need to figure out which parameters were used during
+                    # this forward pass, to ensure we short circuit reduction for any
+                    # unused parameters. Only if `find_unused_parameters` is set.
+                    if self.find_unused_parameters and not self.static_graph:
+                        # Do not need to populate this for static graph.
+                        self.reducer.prepare_for_backward(list(_find_tensors(output)))
+                    else:
+                        self.reducer.prepare_for_backward([])
+                else:
+                    self.require_forward_param_sync = False
+
+            # TODO: DDPSink is currently enabled for unused parameter detection and
+            # static graph training for first iteration.
+            if (self.find_unused_parameters and not self.static_graph) or (
+                self.static_graph and self.num_iterations == 1
+            ):
+                state_dict = {
+                    "static_graph": self.static_graph,
+                    "num_iterations": self.num_iterations,
+                }
+
+                output_tensor_list, treespec, output_is_rref = _tree_flatten_with_rref(output)
+                output_placeholders = [None for _ in range(len(output_tensor_list))]
+                # Do not touch tensors that have no grad_fn, which can cause issues
+                # such as https://github.com/pytorch/pytorch/issues/60733
+                for i, output in enumerate(output_tensor_list):
+                    if torch.is_tensor(output) and output.grad_fn is None:
+                        output_placeholders[i] = output
+
+                # When find_unused_parameters=True, makes tensors which require grad
+                # run through the DDPSink backward pass. When not all outputs are
+                # used in loss, this makes those corresponding tensors receive
+                # undefined gradient which the reducer then handles to ensure
+                # param.grad field is not touched and we don't error out.
+                passthrough_tensor_list = _DDPSink.apply(
+                    self.reducer,
+                    state_dict,
+                    *output_tensor_list,
+                )
+                for i in range(len(output_placeholders)):
+                    if output_placeholders[i] is None:
+                        output_placeholders[i] = passthrough_tensor_list[i]
+
+                # Reconstruct output data structure.
+                output = _tree_unflatten_with_rref(output_placeholders, treespec, output_is_rref)
+        return output

GPT_SoVITS/module/distrib.py

@@ -0,0 +1,123 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Torch distributed utilities."""
+
+import typing as tp
+
+import torch
+
+
+def rank():
+    if torch.distributed.is_initialized():
+        return torch.distributed.get_rank()
+    else:
+        return 0
+
+
+def world_size():
+    if torch.distributed.is_initialized():
+        return torch.distributed.get_world_size()
+    else:
+        return 1
+
+
+def is_distributed():
+    return world_size() > 1
+
+
+def all_reduce(tensor: torch.Tensor, op=torch.distributed.ReduceOp.SUM):
+    if is_distributed():
+        return torch.distributed.all_reduce(tensor, op)
+
+
+def _is_complex_or_float(tensor):
+    return torch.is_floating_point(tensor) or torch.is_complex(tensor)
+
+
+def _check_number_of_params(params: tp.List[torch.Tensor]):
+    # utility function to check that the number of params in all workers is the same,
+    # and thus avoid a deadlock with distributed all reduce.
+    if not is_distributed() or not params:
+        return
+    # print('params[0].device ', params[0].device)
+    tensor = torch.tensor([len(params)], device=params[0].device, dtype=torch.long)
+    all_reduce(tensor)
+    if tensor.item() != len(params) * world_size():
+        # If not all the workers have the same number, for at least one of them,
+        # this inequality will be verified.
+        raise RuntimeError(
+            f"Mismatch in number of params: ours is {len(params)}, at least one worker has a different one."
+        )
+
+
+def broadcast_tensors(tensors: tp.Iterable[torch.Tensor], src: int = 0):
+    """Broadcast the tensors from the given parameters to all workers.
+    This can be used to ensure that all workers have the same model to start with.
+    """
+    if not is_distributed():
+        return
+    tensors = [tensor for tensor in tensors if _is_complex_or_float(tensor)]
+    _check_number_of_params(tensors)
+    handles = []
+    for tensor in tensors:
+        handle = torch.distributed.broadcast(tensor.data, src=src, async_op=True)
+        handles.append(handle)
+    for handle in handles:
+        handle.wait()
+
+
+def sync_buffer(buffers, average=True):
+    """
+    Sync grad for buffers. If average is False, broadcast instead of averaging.
+    """
+    if not is_distributed():
+        return
+    handles = []
+    for buffer in buffers:
+        if torch.is_floating_point(buffer.data):
+            if average:
+                handle = torch.distributed.all_reduce(buffer.data, op=torch.distributed.ReduceOp.SUM, async_op=True)
+            else:
+                handle = torch.distributed.broadcast(buffer.data, src=0, async_op=True)
+            handles.append((buffer, handle))
+    for buffer, handle in handles:
+        handle.wait()
+        if average:
+            buffer.data /= world_size
+
+
+def sync_grad(params):
+    """
+    Simpler alternative to DistributedDataParallel, that doesn't rely
+    on any black magic. For simple models it can also be as fast.
+    Just call this on your model parameters after the call to backward!
+    """
+    if not is_distributed():
+        return
+    handles = []
+    for p in params:
+        if p.grad is not None:
+            handle = torch.distributed.all_reduce(p.grad.data, op=torch.distributed.ReduceOp.SUM, async_op=True)
+            handles.append((p, handle))
+    for p, handle in handles:
+        handle.wait()
+        p.grad.data /= world_size()
+
+
+def average_metrics(metrics: tp.Dict[str, float], count=1.0):
+    """Average a dictionary of metrics across all workers, using the optional
+    `count` as unormalized weight.
+    """
+    if not is_distributed():
+        return metrics
+    keys, values = zip(*metrics.items())
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    tensor = torch.tensor(list(values) + [1], device=device, dtype=torch.float32)
+    tensor *= count
+    all_reduce(tensor)
+    averaged = (tensor[:-1] / tensor[-1]).cpu().tolist()
+    return dict(zip(keys, averaged))

GPT_SoVITS/s2_train.py

@@ -124,7 +124,7 @@ def run(rank, n_gpus, hps):
         collate_fn=collate_fn,
         batch_sampler=train_sampler,
         persistent_workers=True,
-        prefetch_factor=4,
+        prefetch_factor=3,
     )
     # if rank == 0:
     #     eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data, val=True)

GPT_SoVITS/s2_train_v3.py

@@ -118,13 +118,13 @@ def run(rank, n_gpus, hps):
     collate_fn = TextAudioSpeakerCollate()
     train_loader = DataLoader(
         train_dataset,
-        num_workers=6,
+        num_workers=5,
         shuffle=False,
         pin_memory=True,
         collate_fn=collate_fn,
         batch_sampler=train_sampler,
         persistent_workers=True,
-        prefetch_factor=4,
+        prefetch_factor=3,
     )
     # if rank == 0:
     #     eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data, val=True)

GPT_SoVITS/s2_train_v3_lora.py

@@ -120,13 +120,13 @@ def run(rank, n_gpus, hps):
     collate_fn = TextAudioSpeakerCollate()
     train_loader = DataLoader(
         train_dataset,
-        num_workers=6,
+        num_workers=5,
         shuffle=False,
         pin_memory=True,
         collate_fn=collate_fn,
         batch_sampler=train_sampler,
         persistent_workers=True,
-        prefetch_factor=4,
+        prefetch_factor=3,
     )
     save_root = "%s/logs_s2_%s_lora_%s" % (hps.data.exp_dir, hps.model.version, hps.train.lora_rank)
     os.makedirs(save_root, exist_ok=True)

api_v2.py

@@ -41,11 +41,9 @@ POST:
     "repetition_penalty": 1.35,   # float. repetition penalty for T2S model.
     "sample_steps": 32,           # int. number of sampling steps for VITS model V3.
     "super_sampling": False,      # bool. whether to use super-sampling for audio when using VITS model V3.
-    "return_fragment": False,     # bool. step by step return the audio fragment. (Best Quality, Slowest response speed. old version of streaming mode)
-    "streaming_mode": False,      # bool. return audio chunk by chunk. (Medium quality, Slow response speed)
+    "streaming_mode": False,      # bool or int. return audio chunk by chunk.T he available options are: 0,1,2,3 or True/False (0/False: Disabled | 1/True: Best Quality, Slowest response speed (old version streaming_mode) | 2: Medium Quality, Slow response speed | 3: Lower Quality, Faster response speed )
     "overlap_length": 2,          # int. overlap length of semantic tokens for streaming mode.
-    "min_chunk_length": 16,        # int. The minimum chunk length of semantic tokens for streaming mode. (affects audio chunk size)
-    "fixed_length_chunk": False,  # bool. When turned on, it can achieve faster streaming response, but with lower quality. (lower quality, faster response speed)
+    "min_chunk_length": 16,       # int. The minimum chunk length of semantic tokens for streaming mode. (affects audio chunk size)
 }
 ```
 
@@ -106,7 +104,7 @@ RESP:
 import os
 import sys
 import traceback
-from typing import Generator
+from typing import Generator, Union
 
 now_dir = os.getcwd()
 sys.path.append(now_dir)
@@ -171,15 +169,13 @@ class TTS_Request(BaseModel):
     fragment_interval: float = 0.3
     seed: int = -1
     media_type: str = "wav"
-    streaming_mode: bool = False
+    streaming_mode: Union[bool, int] = False
     parallel_infer: bool = True
     repetition_penalty: float = 1.35
     sample_steps: int = 32
     super_sampling: bool = False
     overlap_length: int = 2
     min_chunk_length: int = 16
-    return_fragment: bool = False
-    fixed_length_chunk: bool = False
 
 
 def pack_ogg(io_buffer: BytesIO, data: np.ndarray, rate: int):
@@ -373,11 +369,9 @@ async def tts_handle(req: dict):
                 "repetition_penalty": 1.35,   # float. repetition penalty for T2S model.
                 "sample_steps": 32,           # int. number of sampling steps for VITS model V3.
                 "super_sampling": False,      # bool. whether to use super-sampling for audio when using VITS model V3.
-                "return_fragment": False,     # bool. step by step return the audio fragment. (Best Quality, Slowest response speed. old version of streaming mode)
-                "streaming_mode": False,      # bool. return audio chunk by chunk. (Medium quality, Slow response speed)
+                "streaming_mode": False,      # bool or int. return audio chunk by chunk.T he available options are: 0,1,2,3 or True/False (0/False: Disabled | 1/True: Best Quality, Slowest response speed (old version streaming_mode) | 2: Medium Quality, Slow response speed | 3: Lower Quality, Faster response speed )
                 "overlap_length": 2,          # int. overlap length of semantic tokens for streaming mode.
-                "min_chunk_length": 16,        # int. The minimum chunk length of semantic tokens for streaming mode. (affects audio chunk size)
-                "fixed_length_chunk": False,  # bool. When turned on, it can achieve faster streaming response, but with lower quality. (lower quality, faster response speed)
+                "min_chunk_length": 16,       # int. The minimum chunk length of semantic tokens for streaming mode. (affects audio chunk size)
             }
     returns:
         StreamingResponse: audio stream response.
@@ -390,9 +384,33 @@ async def tts_handle(req: dict):
     check_res = check_params(req)
     if check_res is not None:
         return check_res
+    
+    if streaming_mode == 0:
+        streaming_mode = False
+        return_fragment = False
+        fixed_length_chunk = False
+    elif streaming_mode == 1:
+        streaming_mode = False
+        return_fragment = True
+        fixed_length_chunk = False
+    elif streaming_mode == 2:
+        streaming_mode = True
+        return_fragment = False
+        fixed_length_chunk = False
+    elif streaming_mode == 3:
+        streaming_mode = True
+        return_fragment = False
+        fixed_length_chunk = True
+
+    else:
+        return JSONResponse(status_code=400, content={"message": f"the value of streaming_mode must be 0, 1, 2, 3(int) or true/false(bool)"})
 
     req["streaming_mode"] = streaming_mode
     req["return_fragment"] = return_fragment
+    req["fixed_length_chunk"] = fixed_length_chunk
+
+    print(f"{streaming_mode} {return_fragment} {fixed_length_chunk}")
+
     streaming_mode = streaming_mode or return_fragment
 
 
@@ -457,11 +475,9 @@ async def tts_get_endpoint(
     repetition_penalty: float = 1.35,
     sample_steps: int = 32,
     super_sampling: bool = False,
-    return_fragment: bool = False,
-    streaming_mode: bool = False,
+    streaming_mode: Union[bool, int] = False,
     overlap_length: int = 2,
     min_chunk_length: int = 16,
-    fixed_length_chunk: bool = False,
 ):
     req = {
         "text": text,
@@ -488,8 +504,6 @@ async def tts_get_endpoint(
         "super_sampling": super_sampling,
         "overlap_length": int(overlap_length),
         "min_chunk_length": int(min_chunk_length),
-        "return_fragment": return_fragment,
-        "fixed_length_chunk": fixed_length_chunk
     }
     return await tts_handle(req)
 

requirements.txt

@@ -16,7 +16,7 @@ pypinyin
 pyopenjtalk>=0.4.1
 g2p_en
 torchaudio
-modelscope==1.10.0
+modelscope
 sentencepiece
 transformers>=4.43,<=4.50
 peft
@@ -39,7 +39,5 @@ x_transformers
 torchmetrics<=1.5
 pydantic<=2.10.6
 ctranslate2>=4.0,<5
-huggingface_hub>=0.13
-tokenizers>=0.13,<1
 av>=11
 tqdm

tools/asr/config.py

@@ -1,34 +1,13 @@
-import os
-
-
-def check_fw_local_models():
-    """
-    启动时检查本地是否有 Faster Whisper 模型.
-    """
-    model_size_list = [
-        "medium",
-        "medium.en",
-        "distil-large-v2",
-        "distil-large-v3",
-        "large-v1",
-        "large-v2",
-        "large-v3",
-    ]
-    for i, size in enumerate(model_size_list):
-        if os.path.exists(f"tools/asr/models/faster-whisper-{size}"):
-            model_size_list[i] = size + "-local"
-    return model_size_list
-
-
 def get_models():
     model_size_list = [
         "medium",
         "medium.en",
-        "distil-large-v2",
-        "distil-large-v3",
-        "large-v1",
         "large-v2",
         "large-v3",
+        "large-v3-turbo",
+        #"distil-large-v2",
+        #"distil-large-v3",
+        #"distil-large-v3.5",
     ]
     return model_size_list
 
@@ -36,7 +15,7 @@ def get_models():
 asr_dict = {
     "达摩 ASR (中文)": {"lang": ["zh", "yue"], "size": ["large"], "path": "funasr_asr.py", "precision": ["float32"]},
     "Faster Whisper (多语种)": {
-        "lang": ["auto", "zh", "en", "ja", "ko", "yue"],
+        "lang": ["auto", "en", "ja", "ko"],
         "size": get_models(),
         "path": "fasterwhisper_asr.py",
         "precision": ["float32", "float16", "int8"],

tools/asr/fasterwhisper_asr.py

@@ -1,12 +1,12 @@
 import argparse
 import os
-import time
 import traceback
 
+import requests
 import torch
 from faster_whisper import WhisperModel
-from huggingface_hub import snapshot_download
-from huggingface_hub.errors import LocalEntryNotFoundError
+from huggingface_hub import snapshot_download as snapshot_download_hf
+from modelscope import snapshot_download as snapshot_download_ms
 from tqdm import tqdm
 
 from tools.asr.config import get_models
@@ -40,11 +40,32 @@ language_code_list = [
 
 
 def download_model(model_size: str):
-    if "distil" in model_size:
-        repo_id = "Systran/faster-{}-whisper-{}".format(*model_size.split("-", maxsplit=1))
+    url = "https://huggingface.co/api/models/gpt2"
+    try:
+        requests.get(url, timeout=3)
+        source = "HF"
+    except Exception:
+        source = "ModelScope"
+
+    model_path = ""
+    if source == "HF":
+        if "distil" in model_size:
+            if "3.5" in model_size:
+                repo_id = "distil-whisper/distil-large-v3.5-ct2"
+                model_path = "tools/asr/models/faster-distil-whisper-large-v3.5"
+            else:
+                repo_id = "Systran/faster-{}-whisper-{}".format(*model_size.split("-", maxsplit=1))
+        elif model_size == "large-v3-turbo":
+            repo_id = "mobiuslabsgmbh/faster-whisper-large-v3-turbo"
+            model_path = "tools/asr/models/faster-whisper-large-v3-turbo"
+        else:
+            repo_id = f"Systran/faster-whisper-{model_size}"
+        model_path = (
+            model_path or f"tools/asr/models/{repo_id.replace('Systran/', '').replace('distil-whisper/', '', 1)}"
+        )
     else:
-        repo_id = f"Systran/faster-whisper-{model_size}"
-    model_path = f"tools/asr/models/{repo_id.strip('Systran/')}"
+        repo_id = "XXXXRT/faster-whisper"
+        model_path = "tools/asr/models"
 
     files: list[str] = [
         "config.json",
@@ -52,32 +73,31 @@ def download_model(model_size: str):
         "tokenizer.json",
         "vocabulary.txt",
     ]
-    if model_size == "large-v3" or "distil" in model_size:
+    if "large-v3" in model_size or "distil" in model_size:
         files.append("preprocessor_config.json")
         files.append("vocabulary.json")
 
         files.remove("vocabulary.txt")
 
-    for attempt in range(2):
-        try:
-            snapshot_download(
-                repo_id=repo_id,
-                allow_patterns=files,
-                local_dir=model_path,
-            )
-            break
-        except LocalEntryNotFoundError:
-            if attempt < 1:
-                time.sleep(2)
-            else:
-                print("[ERROR] LocalEntryNotFoundError and no fallback.")
-                traceback.print_exc()
-                exit(1)
-        except Exception as e:
-            print(f"[ERROR] Unexpected error on attempt {attempt + 1}: {e}")
-            traceback.print_exc()
-            exit(1)
+    if source == "ModelScope":
+        files = [f"faster-whisper-{model_size}/{file}".replace("whisper-distil", "distil-whisper") for file in files]
 
+    if source == "HF":
+        print(f"Downloading model from HuggingFace: {repo_id} to {model_path}")
+        snapshot_download_hf(
+            repo_id,
+            local_dir=model_path,
+            local_dir_use_symlinks=False,
+            allow_patterns=files,
+        )
+    else:
+        print(f"Downloading model from ModelScope: {repo_id} to {model_path}")
+        snapshot_download_ms(
+            repo_id,
+            local_dir=model_path,
+            allow_patterns=files,
+        )
+        return model_path + f"/faster-whisper-{model_size}".replace("whisper-distil", "distil-whisper")
     return model_path
 
 
@@ -106,7 +126,7 @@ def execute_asr(input_folder, output_folder, model_path, language, precision):
             )
             text = ""
 
-            if info.language == "zh":
+            if info.language in ["zh", "yue"]:
                 print("检测为中文文本, 转 FunASR 处理")
                 text = only_asr(file_path, language=info.language.lower())
 

tools/asr/funasr_asr.py

@@ -4,9 +4,8 @@ import argparse
 import os
 import traceback
 
-# from funasr.utils import version_checker
-# version_checker.check_for_update = lambda: None
 from funasr import AutoModel
+from modelscope import snapshot_download
 from tqdm import tqdm
 
 funasr_models = {}  # 存储模型避免重复加载
@@ -16,40 +15,43 @@ def only_asr(input_file, language):
     try:
         model = create_model(language)
         text = model.generate(input=input_file)[0]["text"]
-    except:
+    except Exception:
         text = ""
         print(traceback.format_exc())
     return text
 
 
 def create_model(language="zh"):
-    path_vad = "tools/asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch"
-    path_punc = "tools/asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
-    path_vad = path_vad if os.path.exists(path_vad) else "iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"
-    path_punc = path_punc if os.path.exists(path_punc) else "iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
-    vad_model_revision = punc_model_revision = "v2.0.4"
-
     if language == "zh":
+        path_vad = "tools/asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch"
+        path_punc = "tools/asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
         path_asr = "tools/asr/models/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
-        path_asr = (
-            path_asr
-            if os.path.exists(path_asr)
-            else "iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
+        snapshot_download(
+            "iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+            local_dir="tools/asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+        )
+        snapshot_download(
+            "iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
+            local_dir="tools/asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
+        )
+        snapshot_download(
+            "iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+            local_dir="tools/asr/models/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
         )
         model_revision = "v2.0.4"
     elif language == "yue":
         path_asr = "tools/asr/models/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
-        path_asr = (
-            path_asr
-            if os.path.exists(path_asr)
-            else "iic/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
+        snapshot_download(
+            "iic/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
+            local_dir="tools/asr/models/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
         )
-        model_revision = "master"
         path_vad = path_punc = None
-        vad_model_revision = punc_model_revision = None
-        ###友情提示：粤语带VAD识别可能会有少量shape不对报错的，但是不带VAD可以.不带vad只能分阶段单独加标点。不过标点模型对粤语效果真的不行…
+        vad_model_revision = punc_model_revision = ""
+        model_revision = "master"
     else:
-        raise ValueError("FunASR 不支持该语言" + ": " + language)
+        raise ValueError(f"{language} is not supported")
+
+    vad_model_revision = punc_model_revision = "v2.0.4"
 
     if language in funasr_models:
         return funasr_models[language]
@@ -83,7 +85,7 @@ def execute_asr(input_folder, output_folder, model_size, language):
             file_path = os.path.join(input_folder, file_name)
             text = model.generate(input=file_path)[0]["text"]
             output.append(f"{file_path}|{output_file_name}|{language.upper()}|{text}")
-        except:
+        except Exception:
             print(traceback.format_exc())
 
     output_folder = output_folder or "output/asr_opt"

tools/i18n/locale/en_US.json

@@ -38,7 +38,7 @@
     "hop_size:怎么算音量曲线，越小精度越大计算量越高（不是精度越大效果越好）": "hop_size: FO hop size, the smaller the value, the higher the accuracy）",
     "max:归一化后最大值多少": "Loudness multiplier after normalized",
     "max_sil_kept:切完后静音最多留多长": "Maximum length for silence to be kept",
-    "min_interval:最短切割间隔": "Minumum interval for audio cutting",
+    "min_interval:最短切割间隔": "Minimum interval for audio cutting",
     "min_length:每段最小多长，如果第一段太短一直和后面段连起来直到超过这个值": "min_length: the minimum length of each segment. If the first segment is too short, it will be concatenated with the next segment until it exceeds this value",
     "temperature": "temperature",
     "threshold:音量小于这个值视作静音的备选切割点": "Noise gate threshold (loudness below this value will be treated as noise",
@@ -176,7 +176,7 @@
     "语音降噪": "Speech Denoising",
     "请上传3~10秒内参考音频，超过会报错！": "Please upload a reference audio within the 3-10 second range; if it exceeds this duration, it will raise errors.",
     "请上传参考音频": "Please Upload the Reference Audio",
-    "请填入推理文本": "Please Fill in the Terget Text",
+    "请填入推理文本": "Please Fill in the Target Text",
     "请填入正确的List路径": "Please Fill in the Correct List Path",
     "请填入正确的音频文件夹路径": "Please Fill in the Correct Audio Folder Path",
     "请输入有效文本": "Please enter valid text.",

webui.py

@@ -1,3 +1,6 @@
+import os
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"  # 限制为单卡
+
 import os
 import sys
 
@@ -86,7 +89,6 @@ from config import (
 from tools import my_utils
 from tools.my_utils import check_details, check_for_existance
 
-os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
 os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
 
 # os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1' # 当遇到mps不支持的步骤时使用cpu
@@ -117,20 +119,20 @@ def set_default():
     gpu_info = "\n".join(gpu_infos)
     if is_gpu_ok:
         minmem = min(mem)
-        default_batch_size = minmem // 2 if version not in v3v4set else minmem // 8
-        default_batch_size_s1 = minmem // 2
+        default_batch_size = int(minmem // 2 if version not in v3v4set else minmem // 8)
+        default_batch_size_s1 = int(minmem // 2)
     else:
         default_batch_size = default_batch_size_s1 = int(psutil.virtual_memory().total / 1024 / 1024 / 1024 / 4)
     if version not in v3v4set:
         default_sovits_epoch = 8
         default_sovits_save_every_epoch = 4
-        max_sovits_epoch = 25  # 40
-        max_sovits_save_every_epoch = 25  # 10
+        max_sovits_epoch = 255  # 40
+        max_sovits_save_every_epoch = 255  # 10
     else:
         default_sovits_epoch = 2
         default_sovits_save_every_epoch = 1
-        max_sovits_epoch = 16  # 40 # 3 #训太多=作死
-        max_sovits_save_every_epoch = 10  # 10 # 3
+        max_sovits_epoch = 255  # 40 # 3 #训太多=作死
+        max_sovits_save_every_epoch = 255  # 10 # 3
 
     default_batch_size = max(1, default_batch_size)
     default_batch_size_s1 = max(1, default_batch_size_s1)
@@ -504,7 +506,7 @@ def open1Ba(
 ):
     global p_train_SoVITS
     if p_train_SoVITS == None:
-        exp_name = exp_name.rstrip(" ")
+        exp_name=exp_name.rstrip(" ")
         config_file = (
             "GPT_SoVITS/configs/s2.json"
             if version not in {"v2Pro", "v2ProPlus"}
@@ -601,7 +603,7 @@ def open1Bb(
 ):
     global p_train_GPT
     if p_train_GPT == None:
-        exp_name = exp_name.rstrip(" ")
+        exp_name=exp_name.rstrip(" ")
         with open(
             "GPT_SoVITS/configs/s1longer.yaml" if version == "v1" else "GPT_SoVITS/configs/s1longer-v2.yaml"
         ) as f:
@@ -1725,8 +1727,8 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
                                 )
                             with gr.Row():
                                 text_low_lr_rate = gr.Slider(
-                                    minimum=0.2,
-                                    maximum=0.6,
+                                    minimum=0,
+                                    maximum=1,
                                     step=0.05,
                                     label=i18n("文本模块学习率权重"),
                                     value=0.4,
@@ -1735,7 +1737,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
                                 lora_rank = gr.Radio(
                                     label=i18n("LoRA秩"),
                                     value="32",
-                                    choices=["16", "32", "64", "128"],
+                                    choices=["16", "32", "64", "128", "256", "512", "1024","2048", "4096"],
                                     visible=True if version in v3v4set else False,
                                 )  # v1v2 not need
                                 save_every_epoch = gr.Slider(
@@ -1797,7 +1799,7 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
                                 )
                                 total_epoch1Bb = gr.Slider(
                                     minimum=2,
-                                    maximum=50,
+                                    maximum=max_sovits_epoch,
                                     step=1,
                                     label=i18n("总训练轮数total_epoch"),
                                     value=15,
@@ -1980,4 +1982,3 @@ with gr.Blocks(title="GPT-SoVITS WebUI", analytics_enabled=False, js=js, css=css
         server_port=webui_port_main,
         # quiet=True,
     )
-