DDP Multi-GPU --resume bug fix (#1810)
This commit is contained in:
1 parent
73cf75faa8
commit
7180b22e2d
1 file changed
+2
-1
@@ -472,9 +472,10 @@ if __name__ == '__main__':
|
|||||||
if opt.resume: # resume an interrupted run
|
if opt.resume: # resume an interrupted run
|
||||||
ckpt = opt.resume if isinstance(opt.resume, str) else get_latest_run() # specified or most recent path
|
ckpt = opt.resume if isinstance(opt.resume, str) else get_latest_run() # specified or most recent path
|
||||||
assert os.path.isfile(ckpt), 'ERROR: --resume checkpoint does not exist'
|
assert os.path.isfile(ckpt), 'ERROR: --resume checkpoint does not exist'
|
||||||
|
apriori = opt.global_rank, opt.local_rank
|
||||||
with open(Path(ckpt).parent.parent / 'opt.yaml') as f:
|
with open(Path(ckpt).parent.parent / 'opt.yaml') as f:
|
||||||
opt = argparse.Namespace(**yaml.load(f, Loader=yaml.FullLoader)) # replace
|
opt = argparse.Namespace(**yaml.load(f, Loader=yaml.FullLoader)) # replace
|
||||||
opt.cfg, opt.weights, opt.resume = '', ckpt, True
|
opt.cfg, opt.weights, opt.resume, opt.global_rank, opt.local_rank = '', ckpt, True, *apriori # reinstate
|
||||||
logger.info('Resuming training from %s' % ckpt)
|
logger.info('Resuming training from %s' % ckpt)
|
||||||
else:
|
else:
|
||||||
# opt.hyp = opt.hyp or ('hyp.finetune.yaml' if opt.weights else 'hyp.scratch.yaml')
|
# opt.hyp = opt.hyp or ('hyp.finetune.yaml' if opt.weights else 'hyp.scratch.yaml')
|
||||||
|
|||||||
Reference in new issue
Block a user