Single-source training (#680)

* Single-source training

* Extract hyperparameters into seperate files

* weight decay scientific notation yaml reader bug fix

* remove import glob

* intersect_dicts() implementation

* 'or' bug fix

* .to(device) bug fix

utils/torch_utils.py

@@ -55,10 +55,14 @@ def time_synchronized():
 
 
 def is_parallel(model):
-    # is model is parallel with DP or DDP
     return type(model) in (nn.parallel.DataParallel, nn.parallel.DistributedDataParallel)
 
 
+def intersect_dicts(da, db, exclude=()):
+    # Dictionary intersection of matching keys and shapes, omitting 'exclude' keys, using da values
+    return {k: v for k, v in da.items() if k in db and not any(x in k for x in exclude) and v.shape == db[k].shape}
+
+
 def initialize_weights(model):
     for m in model.modules():
         t = type(m)
@@ -72,7 +76,7 @@ def initialize_weights(model):
 
 
 def find_modules(model, mclass=nn.Conv2d):
-    # finds layer indices matching module class 'mclass'
+    # Finds layer indices matching module class 'mclass'
     return [i for i, m in enumerate(model.module_list) if isinstance(m, mclass)]