修复resume epoch数识别错,每次resume都要都训一轮的问题

修复resume epoch数识别错,每次resume都要都训一轮的问题
This commit is contained in:
RVC-Boss 2025-02-23 15:09:22 +08:00 committed by GitHub
parent 514fb692db
commit aa07216bba
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 13 additions and 4 deletions

View File

@ -205,6 +205,7 @@ def run(rank, n_gpus, hps):
net_g,
optim_g,
)
epoch_str+=1
global_step = (epoch_str - 1) * len(train_loader)
# epoch_str = 1
# global_step = 0
@ -215,7 +216,7 @@ def run(rank, n_gpus, hps):
if hps.train.pretrained_s2G != ""and hps.train.pretrained_s2G != None and os.path.exists(hps.train.pretrained_s2G):
if rank == 0:
logger.info("loaded pretrained %s" % hps.train.pretrained_s2G)
print(
print("loaded pretrained %s" % hps.train.pretrained_s2G,
net_g.module.load_state_dict(
torch.load(hps.train.pretrained_s2G, map_location="cpu")["weight"],
strict=False,
@ -227,7 +228,7 @@ def run(rank, n_gpus, hps):
if hps.train.pretrained_s2D != ""and hps.train.pretrained_s2D != None and os.path.exists(hps.train.pretrained_s2D):
if rank == 0:
logger.info("loaded pretrained %s" % hps.train.pretrained_s2D)
print(
print("loaded pretrained %s" % hps.train.pretrained_s2D,
net_d.module.load_state_dict(
torch.load(hps.train.pretrained_s2D, map_location="cpu")["weight"]
) if torch.cuda.is_available() else net_d.load_state_dict(
@ -251,6 +252,7 @@ def run(rank, n_gpus, hps):
scaler = GradScaler(enabled=hps.train.fp16_run)
for epoch in range(epoch_str, hps.train.epochs + 1):
print("start training from epoch %s"%epoch)
if rank == 0:
train_and_evaluate(
rank,
@ -280,6 +282,7 @@ def run(rank, n_gpus, hps):
)
scheduler_g.step()
scheduler_d.step()
print("training done")
def train_and_evaluate(

View File

@ -178,6 +178,7 @@ def run(rank, n_gpus, hps):
net_g,
optim_g,
)
epoch_str+=1
global_step = (epoch_str - 1) * len(train_loader)
# epoch_str = 1
# global_step = 0
@ -188,7 +189,7 @@ def run(rank, n_gpus, hps):
if hps.train.pretrained_s2G != ""and hps.train.pretrained_s2G != None and os.path.exists(hps.train.pretrained_s2G):
if rank == 0:
logger.info("loaded pretrained %s" % hps.train.pretrained_s2G)
print(
print("loaded pretrained %s" % hps.train.pretrained_s2G,
net_g.module.load_state_dict(
torch.load(hps.train.pretrained_s2G, map_location="cpu")["weight"],
strict=False,
@ -225,6 +226,7 @@ def run(rank, n_gpus, hps):
net_d=optim_d=scheduler_d=None
for epoch in range(epoch_str, hps.train.epochs + 1):
print("start training from epoch %s"%epoch)
if rank == 0:
train_and_evaluate(
rank,
@ -254,6 +256,7 @@ def run(rank, n_gpus, hps):
)
scheduler_g.step()
# scheduler_d.step()
print("training done")
def train_and_evaluate(

View File

@ -161,6 +161,7 @@ def run(rank, n_gpus, hps):
net_g,
optim_g,
)
epoch_str+=1
global_step = (epoch_str - 1) * len(train_loader)
except: # 如果首次不能加载加载pretrain
# traceback.print_exc()
@ -170,7 +171,7 @@ def run(rank, n_gpus, hps):
if hps.train.pretrained_s2G != ""and hps.train.pretrained_s2G != None and os.path.exists(hps.train.pretrained_s2G):
if rank == 0:
logger.info("loaded pretrained %s" % hps.train.pretrained_s2G)
print(
print("loaded pretrained %s" % hps.train.pretrained_s2G,
net_g.load_state_dict(
torch.load(hps.train.pretrained_s2G, map_location="cpu")["weight"],
strict=False,
@ -198,6 +199,7 @@ def run(rank, n_gpus, hps):
net_d=optim_d=scheduler_d=None
for epoch in range(epoch_str, hps.train.epochs + 1):
print("start training from epoch %s"%epoch)
if rank == 0:
train_and_evaluate(
rank,
@ -226,6 +228,7 @@ def run(rank, n_gpus, hps):
None,
)
scheduler_g.step()
print("training done")
def train_and_evaluate(
rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers