Merge c922dfbcd067852e3802020616391ab826b2c43b into 48b1a0169a28582a8984402f82cf438d3bfa6aca

GPT_SoVITS/AR/data/bucket_sampler.py

@@ -39,12 +39,13 @@ class DistributedBucketSampler(Sampler[T_co]):
         if num_replicas is None:
             if not dist.is_available():
                 raise RuntimeError("Requires distributed package to be available")
-            num_replicas = dist.get_world_size() if torch.cuda.is_available() else 1
+            # CUDA/HIP availability does not imply that a process group exists.
+            num_replicas = dist.get_world_size() if dist.is_initialized() else 1
         if rank is None:
             if not dist.is_available():
                 raise RuntimeError("Requires distributed package to be available")
-            rank = dist.get_rank() if torch.cuda.is_available() else 0
-            if torch.cuda.is_available():
+            rank = dist.get_rank() if dist.is_initialized() else 0
+            if dist.is_initialized() and torch.cuda.is_available():
                 torch.cuda.set_device(rank)
         if rank >= num_replicas or rank < 0:
             raise ValueError("Invalid rank {}, rank should be in the interval [0, {}]".format(rank, num_replicas - 1))

GPT_SoVITS/s1_train.py

@@ -84,6 +84,7 @@ class my_model_ckpt(ModelCheckpoint):
 
 def main(args):
     config = load_yaml_config(args.config_file)
+    gpu_count = torch.cuda.device_count()
 
     output_dir = Path(config["output_dir"])
     output_dir.mkdir(parents=True, exist_ok=True)
@@ -117,8 +118,10 @@ def main(args):
         devices=-1 if torch.cuda.is_available() else 1,
         benchmark=False,
         fast_dev_run=False,
+        # Avoid unnecessary DDP/NCCL synchronization on a single GPU. It can
+        # hang indefinitely with ROCm under WSL; multi-GPU training still uses DDP.
         strategy=DDPStrategy(process_group_backend="nccl" if platform.system() != "Windows" else "gloo")
-        if torch.cuda.is_available()
+        if gpu_count > 1
         else "auto",
         precision=config["train"]["precision"],
         logger=logger,

install.sh

@@ -332,8 +332,22 @@ if [ "$USE_CUDA" = true ] && [ "$WORKFLOW" = false ]; then
         run_pip_quiet torch torchcodec --index-url "https://download.pytorch.org/whl/cu126"
     fi
 elif [ "$USE_ROCM" = true ] && [ "$WORKFLOW" = false ]; then
-    echo -e "${INFO}Installing PyTorch For ROCm 6.2..."
-    run_pip_quiet torch torchcodec --index-url "https://download.pytorch.org/whl/rocm6.2"
+    PYTHON_TAG=$(python -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")')
+    case "$PYTHON_TAG" in
+    cp310 | cp311 | cp312 | cp313) ;;
+    *)
+        echo -e "${ERROR}ROCm 7.2 wheels require Python 3.10-3.13 (detected: $PYTHON_TAG)"
+        exit 1
+        ;;
+    esac
+
+    echo -e "${INFO}Installing PyTorch 2.8 For ROCm 7.2..."
+    ROCM_WHEEL_ROOT="https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2"
+    run_pip_quiet \
+        "$ROCM_WHEEL_ROOT/torch-2.8.0%2Brocm7.2.0.lw.gitbf943426-${PYTHON_TAG}-${PYTHON_TAG}-linux_x86_64.whl" \
+        "$ROCM_WHEEL_ROOT/torchaudio-2.8.0%2Brocm7.2.0.git6e1c7fe9-${PYTHON_TAG}-${PYTHON_TAG}-linux_x86_64.whl" \
+        "$ROCM_WHEEL_ROOT/torchvision-0.23.0%2Brocm7.2.0.git824e8c87-${PYTHON_TAG}-${PYTHON_TAG}-linux_x86_64.whl" \
+        "$ROCM_WHEEL_ROOT/triton-3.4.0%2Brocm7.2.0.git0cace8d2-${PYTHON_TAG}-${PYTHON_TAG}-linux_x86_64.whl"
 elif [ "$USE_CPU" = true ] && [ "$WORKFLOW" = false ]; then
     echo -e "${INFO}Installing PyTorch For CPU..."
     run_pip_quiet torch torchcodec --index-url "https://download.pytorch.org/whl/cpu"
@@ -349,7 +363,15 @@ hash -r
 
 run_pip_quiet -r extra-req.txt --no-deps
 
-run_pip_quiet -r requirements.txt
+if [ "$USE_ROCM" = true ] && [ "$WORKFLOW" = false ]; then
+    ROCM_REQUIREMENTS=$(mktemp)
+    grep -vE '^(torchaudio|onnxruntime-gpu)([<>=;[:space:]]|$)' requirements.txt >"$ROCM_REQUIREMENTS"
+    run_pip_quiet -r "$ROCM_REQUIREMENTS"
+    rm -f "$ROCM_REQUIREMENTS"
+    run_pip_quiet onnxruntime-migraphx==1.23.2 -f "https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2.1/"
+else
+    run_pip_quiet -r requirements.txt
+fi
 
 echo -e "${SUCCESS}Python Dependencies Installed"
 

requirements.txt

@@ -34,7 +34,7 @@ g2pk2
 ko_pron
 opencc
 python_mecab_ko; sys_platform != 'win32'
-fastapi[standard]>=0.115.2
+fastapi[standard]==0.115.2
 x_transformers
 torchmetrics<=1.5
 pydantic<=2.10.6