diff --git a/GPT_SoVITS/AR/data/bucket_sampler.py b/GPT_SoVITS/AR/data/bucket_sampler.py index d8457334..5fff25c4 100644 --- a/GPT_SoVITS/AR/data/bucket_sampler.py +++ b/GPT_SoVITS/AR/data/bucket_sampler.py @@ -39,12 +39,13 @@ class DistributedBucketSampler(Sampler[T_co]): if num_replicas is None: if not dist.is_available(): raise RuntimeError("Requires distributed package to be available") - num_replicas = dist.get_world_size() if torch.cuda.is_available() else 1 + # CUDA/HIP availability does not imply that a process group exists. + num_replicas = dist.get_world_size() if dist.is_initialized() else 1 if rank is None: if not dist.is_available(): raise RuntimeError("Requires distributed package to be available") - rank = dist.get_rank() if torch.cuda.is_available() else 0 - if torch.cuda.is_available(): + rank = dist.get_rank() if dist.is_initialized() else 0 + if dist.is_initialized() and torch.cuda.is_available(): torch.cuda.set_device(rank) if rank >= num_replicas or rank < 0: raise ValueError("Invalid rank {}, rank should be in the interval [0, {}]".format(rank, num_replicas - 1)) diff --git a/GPT_SoVITS/s1_train.py b/GPT_SoVITS/s1_train.py index 1176f0bc..93805ebf 100644 --- a/GPT_SoVITS/s1_train.py +++ b/GPT_SoVITS/s1_train.py @@ -84,6 +84,7 @@ class my_model_ckpt(ModelCheckpoint): def main(args): config = load_yaml_config(args.config_file) + gpu_count = torch.cuda.device_count() output_dir = Path(config["output_dir"]) output_dir.mkdir(parents=True, exist_ok=True) @@ -117,8 +118,10 @@ def main(args): devices=-1 if torch.cuda.is_available() else 1, benchmark=False, fast_dev_run=False, + # Avoid unnecessary DDP/NCCL synchronization on a single GPU. It can + # hang indefinitely with ROCm under WSL; multi-GPU training still uses DDP. strategy=DDPStrategy(process_group_backend="nccl" if platform.system() != "Windows" else "gloo") - if torch.cuda.is_available() + if gpu_count > 1 else "auto", precision=config["train"]["precision"], logger=logger, diff --git a/install.sh b/install.sh index da0fb839..c9130145 100644 --- a/install.sh +++ b/install.sh @@ -332,8 +332,22 @@ if [ "$USE_CUDA" = true ] && [ "$WORKFLOW" = false ]; then run_pip_quiet torch torchcodec --index-url "https://download.pytorch.org/whl/cu126" fi elif [ "$USE_ROCM" = true ] && [ "$WORKFLOW" = false ]; then - echo -e "${INFO}Installing PyTorch For ROCm 6.2..." - run_pip_quiet torch torchcodec --index-url "https://download.pytorch.org/whl/rocm6.2" + PYTHON_TAG=$(python -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")') + case "$PYTHON_TAG" in + cp310 | cp311 | cp312 | cp313) ;; + *) + echo -e "${ERROR}ROCm 7.2 wheels require Python 3.10-3.13 (detected: $PYTHON_TAG)" + exit 1 + ;; + esac + + echo -e "${INFO}Installing PyTorch 2.8 For ROCm 7.2..." + ROCM_WHEEL_ROOT="https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2" + run_pip_quiet \ + "$ROCM_WHEEL_ROOT/torch-2.8.0%2Brocm7.2.0.lw.gitbf943426-${PYTHON_TAG}-${PYTHON_TAG}-linux_x86_64.whl" \ + "$ROCM_WHEEL_ROOT/torchaudio-2.8.0%2Brocm7.2.0.git6e1c7fe9-${PYTHON_TAG}-${PYTHON_TAG}-linux_x86_64.whl" \ + "$ROCM_WHEEL_ROOT/torchvision-0.23.0%2Brocm7.2.0.git824e8c87-${PYTHON_TAG}-${PYTHON_TAG}-linux_x86_64.whl" \ + "$ROCM_WHEEL_ROOT/triton-3.4.0%2Brocm7.2.0.git0cace8d2-${PYTHON_TAG}-${PYTHON_TAG}-linux_x86_64.whl" elif [ "$USE_CPU" = true ] && [ "$WORKFLOW" = false ]; then echo -e "${INFO}Installing PyTorch For CPU..." run_pip_quiet torch torchcodec --index-url "https://download.pytorch.org/whl/cpu" @@ -349,7 +363,15 @@ hash -r run_pip_quiet -r extra-req.txt --no-deps -run_pip_quiet -r requirements.txt +if [ "$USE_ROCM" = true ] && [ "$WORKFLOW" = false ]; then + ROCM_REQUIREMENTS=$(mktemp) + grep -vE '^(torchaudio|onnxruntime-gpu)([<>=;[:space:]]|$)' requirements.txt >"$ROCM_REQUIREMENTS" + run_pip_quiet -r "$ROCM_REQUIREMENTS" + rm -f "$ROCM_REQUIREMENTS" + run_pip_quiet onnxruntime-migraphx==1.23.2 -f "https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2.1/" +else + run_pip_quiet -r requirements.txt +fi echo -e "${SUCCESS}Python Dependencies Installed" diff --git a/requirements.txt b/requirements.txt index d1f6dfec..79e8aac9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ g2pk2 ko_pron opencc python_mecab_ko; sys_platform != 'win32' -fastapi[standard]>=0.115.2 +fastapi[standard]==0.115.2 x_transformers torchmetrics<=1.5 pydantic<=2.10.6