From 5e1a9553fbed73995c9b81e63ba41cc70fdf89de Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Fri, 16 Sep 2022 21:46:07 +0200 Subject: [PATCH] AutoBatch `cudnn.benchmark=True` fix (#9448) * AutoBatch `cudnn.benchmark=True` fix May resolve https://github.com/ultralytics/yolov5/issues/9287 Signed-off-by: Glenn Jocher * Update autobatch.py Signed-off-by: Glenn Jocher * Update autobatch.py Signed-off-by: Glenn Jocher * Update general.py Signed-off-by: Glenn Jocher Signed-off-by: Glenn Jocher --- utils/autobatch.py | 3 +++ utils/general.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/autobatch.py b/utils/autobatch.py index 641b055..3204fd2 100644 --- a/utils/autobatch.py +++ b/utils/autobatch.py @@ -33,6 +33,9 @@ def autobatch(model, imgsz=640, fraction=0.8, batch_size=16): if device.type == 'cpu': LOGGER.info(f'{prefix}CUDA not detected, using default CPU batch-size {batch_size}') return batch_size + if torch.backends.cudnn.benchmark: + LOGGER.info(f'{prefix} ⚠️ Requires torch.backends.cudnn.benchmark=False, using default batch-size {batch_size}') + return batch_size # Inspect CUDA memory gb = 1 << 30 # bytes to GiB (1024 ** 3) diff --git a/utils/general.py b/utils/general.py index 8633511..af95b3d 100644 --- a/utils/general.py +++ b/utils/general.py @@ -223,7 +223,7 @@ def init_seeds(seed=0, deterministic=False): torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.cuda.manual_seed_all(seed) # for Multi-GPU, exception safe - torch.backends.cudnn.benchmark = True # for faster training + # torch.backends.cudnn.benchmark = True # AutoBatch problem https://github.com/ultralytics/yolov5/issues/9287 if deterministic and check_version(torch.__version__, '1.12.0'): # https://github.com/ultralytics/yolov5/pull/8213 torch.use_deterministic_algorithms(True) torch.backends.cudnn.deterministic = True