mirror of
https://github.com/RVC-Boss/GPT-SoVITS.git
synced 2026-04-29 21:00:42 +08:00
14191901cd
1. 修复 sync_buffer 中除以函数对象而非调用结果（distrib.py）
- `buffer.data /= world_size` 中 world_size 是函数，缺少 ()，
导致 TypeError 使分布式训练 buffer 同步失败
2. 修复 istft 函数缺少 return 语句（spec_utils.py）
- 函数计算了结果但未返回，调用者始终得到 None
3. 修复 cut0 返回字面量 "/n" 而非换行符 "\n"（text_segmentation_method.py）
- 导致后续 text.split("\n") 无法正确切分，字面 /n 被当作文本内容
4. 修复粤语 ASR 的 vad/punc model_revision 被无条件覆盖（funasr_asr.py）
- 粤语分支将 vad_model_revision 设为空（因不使用 VAD/标点模型），
但 if/else 外的赋值将其覆盖为 "v2.0.4"，传入错误的 revision 参数
Made-with: Cursor
124 lines
4.0 KiB
Python
124 lines
4.0 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
"""Torch distributed utilities."""
|
|
|
|
import typing as tp
|
|
|
|
import torch
|
|
|
|
|
|
def rank():
|
|
if torch.distributed.is_initialized():
|
|
return torch.distributed.get_rank()
|
|
else:
|
|
return 0
|
|
|
|
|
|
def world_size():
|
|
if torch.distributed.is_initialized():
|
|
return torch.distributed.get_world_size()
|
|
else:
|
|
return 1
|
|
|
|
|
|
def is_distributed():
|
|
return world_size() > 1
|
|
|
|
|
|
def all_reduce(tensor: torch.Tensor, op=torch.distributed.ReduceOp.SUM):
|
|
if is_distributed():
|
|
return torch.distributed.all_reduce(tensor, op)
|
|
|
|
|
|
def _is_complex_or_float(tensor):
|
|
return torch.is_floating_point(tensor) or torch.is_complex(tensor)
|
|
|
|
|
|
def _check_number_of_params(params: tp.List[torch.Tensor]):
|
|
# utility function to check that the number of params in all workers is the same,
|
|
# and thus avoid a deadlock with distributed all reduce.
|
|
if not is_distributed() or not params:
|
|
return
|
|
# print('params[0].device ', params[0].device)
|
|
tensor = torch.tensor([len(params)], device=params[0].device, dtype=torch.long)
|
|
all_reduce(tensor)
|
|
if tensor.item() != len(params) * world_size():
|
|
# If not all the workers have the same number, for at least one of them,
|
|
# this inequality will be verified.
|
|
raise RuntimeError(
|
|
f"Mismatch in number of params: ours is {len(params)}, at least one worker has a different one."
|
|
)
|
|
|
|
|
|
def broadcast_tensors(tensors: tp.Iterable[torch.Tensor], src: int = 0):
|
|
"""Broadcast the tensors from the given parameters to all workers.
|
|
This can be used to ensure that all workers have the same model to start with.
|
|
"""
|
|
if not is_distributed():
|
|
return
|
|
tensors = [tensor for tensor in tensors if _is_complex_or_float(tensor)]
|
|
_check_number_of_params(tensors)
|
|
handles = []
|
|
for tensor in tensors:
|
|
handle = torch.distributed.broadcast(tensor.data, src=src, async_op=True)
|
|
handles.append(handle)
|
|
for handle in handles:
|
|
handle.wait()
|
|
|
|
|
|
def sync_buffer(buffers, average=True):
|
|
"""
|
|
Sync grad for buffers. If average is False, broadcast instead of averaging.
|
|
"""
|
|
if not is_distributed():
|
|
return
|
|
handles = []
|
|
for buffer in buffers:
|
|
if torch.is_floating_point(buffer.data):
|
|
if average:
|
|
handle = torch.distributed.all_reduce(buffer.data, op=torch.distributed.ReduceOp.SUM, async_op=True)
|
|
else:
|
|
handle = torch.distributed.broadcast(buffer.data, src=0, async_op=True)
|
|
handles.append((buffer, handle))
|
|
for buffer, handle in handles:
|
|
handle.wait()
|
|
if average:
|
|
buffer.data /= world_size()
|
|
|
|
|
|
def sync_grad(params):
|
|
"""
|
|
Simpler alternative to DistributedDataParallel, that doesn't rely
|
|
on any black magic. For simple models it can also be as fast.
|
|
Just call this on your model parameters after the call to backward!
|
|
"""
|
|
if not is_distributed():
|
|
return
|
|
handles = []
|
|
for p in params:
|
|
if p.grad is not None:
|
|
handle = torch.distributed.all_reduce(p.grad.data, op=torch.distributed.ReduceOp.SUM, async_op=True)
|
|
handles.append((p, handle))
|
|
for p, handle in handles:
|
|
handle.wait()
|
|
p.grad.data /= world_size()
|
|
|
|
|
|
def average_metrics(metrics: tp.Dict[str, float], count=1.0):
|
|
"""Average a dictionary of metrics across all workers, using the optional
|
|
`count` as unormalized weight.
|
|
"""
|
|
if not is_distributed():
|
|
return metrics
|
|
keys, values = zip(*metrics.items())
|
|
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
tensor = torch.tensor(list(values) + [1], device=device, dtype=torch.float32)
|
|
tensor *= count
|
|
all_reduce(tensor)
|
|
averaged = (tensor[:-1] / tensor[-1]).cpu().tolist()
|
|
return dict(zip(keys, averaged))
|