Remove hook torch.nan_to_num(x) (#8826)

* Remove hook `torch.nan_to_num(x)`

Observed erratic training behavior (green line) with the nan_to_num hook in classifier branch. I'm going to remove it from master.

* Update train.py
This commit is contained in:
Glenn Jocher authored and GitHub committed 2022-08-01 21:39:04 +02:00
1 parent 59578f2782
commit f3c78a387e
1 file changed
+1 -1
+1 -1
View File
@@ -131,7 +131,7 @@ def train(hyp, opt, device, callbacks): # hyp is path/to/hyp.yaml or hyp dictio
freeze = [f'model.{x}.' for x in (freeze if len(freeze) > 1 else range(freeze[0]))] # layers to freeze
for k, v in model.named_parameters():
v.requires_grad = True # train all layers
v.register_hook(lambda x: torch.nan_to_num(x)) # NaN to 0.0
# v.register_hook(lambda x: torch.nan_to_num(x)) # NaN to 0 (commented for erratic training results)
if any(x in k for x in freeze):
LOGGER.info(f'freezing {k}')
v.requires_grad = False