Ankle/yolov5_ZWgra
1
0 Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

Update Actions with Lychee and GitHub Token (#12592)

Browse Source 
* Add Ultralytics Actions

* Auto-format by Ultralytics actions

* Update format.yml

Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com>

---------

Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com>
Co-authored-by: UltralyticsAssistant <web@ultralytics.com>
Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>
This commit is contained in:
Paula DerrengerUltralyticsAssistantGlenn Jocher
 authored and GitHub committed 2024-01-08 01:29:14 +01:00
1 parent f56355d321
commit ad97dc6a6d
47 files changed
+4217 -3462
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

utils/loggers/__init__.py
+103 -92
View File
@@ -1,7 +1,5 @@
# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license
"""
Logging utils
"""
"""Logging utils."""
import json
import os
import warnings
@@ -16,8 +14,8 @@ from utils.loggers.wandb.wandb_utils import WandbLogger
from utils.plots import plot_images, plot_labels, plot_results
from utils.torch_utils import de_parallel
LOGGERS = ('csv', 'tb', 'wandb', 'clearml', 'comet') # *.csv, TensorBoard, Weights & Biases, ClearML
RANK = int(os.getenv('RANK', -1))
LOGGERS = ("csv", "tb", "wandb", "clearml", "comet") # *.csv, TensorBoard, Weights & Biases, ClearML
RANK = int(os.getenv("RANK", -1))
try:
from torch.utils.tensorboard import SummaryWriter
@@ -27,8 +25,8 @@ except ImportError:
try:
import wandb
assert hasattr(wandb, '__version__') # verify package import not local dir
if pkg.parse_version(wandb.__version__) >= pkg.parse_version('0.12.2') and RANK in {0, -1}:
assert hasattr(wandb, "__version__") # verify package import not local dir
if pkg.parse_version(wandb.__version__) >= pkg.parse_version("0.12.2") and RANK in {0, -1}:
try:
wandb_login_success = wandb.login(timeout=30)
except wandb.errors.UsageError: # known non-TTY terminal issue
@@ -41,7 +39,7 @@ except (ImportError, AssertionError):
try:
import clearml
assert hasattr(clearml, '__version__') # verify package import not local dir
assert hasattr(clearml, "__version__") # verify package import not local dir
except (ImportError, AssertionError):
clearml = None
@@ -49,7 +47,7 @@ try:
if RANK in {0, -1}:
import comet_ml
assert hasattr(comet_ml, '__version__') # verify package import not local dir
assert hasattr(comet_ml, "__version__") # verify package import not local dir
from utils.loggers.comet import CometLogger
else:
@@ -59,7 +57,11 @@ except (ImportError, AssertionError):
def _json_default(value):
"""Format `value` for JSON serialization (e.g. unwrap tensors). Fall back to strings."""
"""
Format `value` for JSON serialization (e.g. unwrap tensors).
Fall back to strings.
"""
if isinstance(value, torch.Tensor):
try:
value = value.item()
@@ -70,7 +72,7 @@ def _json_default(value):
return str(value)
class Loggers():
class Loggers:
# YOLOv5 Loggers class
def __init__(self, save_dir=None, weights=None, opt=None, hyp=None, logger=None, include=LOGGERS):
self.save_dir = save_dir
@@ -81,62 +83,65 @@ class Loggers():
self.logger = logger # for printing results to console
self.include = include
self.keys = [
'train/box_loss',
'train/obj_loss',
'train/cls_loss', # train loss
'metrics/precision',
'metrics/recall',
'metrics/mAP_0.5',
'metrics/mAP_0.5:0.95', # metrics
'val/box_loss',
'val/obj_loss',
'val/cls_loss', # val loss
'x/lr0',
'x/lr1',
'x/lr2'] # params
self.best_keys = ['best/epoch', 'best/precision', 'best/recall', 'best/mAP_0.5', 'best/mAP_0.5:0.95']
"train/box_loss",
"train/obj_loss",
"train/cls_loss", # train loss
"metrics/precision",
"metrics/recall",
"metrics/mAP_0.5",
"metrics/mAP_0.5:0.95", # metrics
"val/box_loss",
"val/obj_loss",
"val/cls_loss", # val loss
"x/lr0",
"x/lr1",
"x/lr2",
] # params
self.best_keys = ["best/epoch", "best/precision", "best/recall", "best/mAP_0.5", "best/mAP_0.5:0.95"]
for k in LOGGERS:
setattr(self, k, None) # init empty logger dictionary
self.csv = True # always log to csv
self.ndjson_console = ('ndjson_console' in self.include) # log ndjson to console
self.ndjson_file = ('ndjson_file' in self.include) # log ndjson to file
self.ndjson_console = "ndjson_console" in self.include # log ndjson to console
self.ndjson_file = "ndjson_file" in self.include # log ndjson to file
# Messages
if not comet_ml:
prefix = colorstr('Comet: ')
prefix = colorstr("Comet: ")
s = f"{prefix}run 'pip install comet_ml' to automatically track and visualize YOLOv5 🚀 runs in Comet"
self.logger.info(s)
# TensorBoard
s = self.save_dir
if 'tb' in self.include and not self.opt.evolve:
prefix = colorstr('TensorBoard: ')
if "tb" in self.include and not self.opt.evolve:
prefix = colorstr("TensorBoard: ")
self.logger.info(f"{prefix}Start with 'tensorboard --logdir {s.parent}', view at http://localhost:6006/")
self.tb = SummaryWriter(str(s))
# W&B
if wandb and 'wandb' in self.include:
if wandb and "wandb" in self.include:
self.opt.hyp = self.hyp # add hyperparameters
self.wandb = WandbLogger(self.opt)
else:
self.wandb = None
# ClearML
if clearml and 'clearml' in self.include:
if clearml and "clearml" in self.include:
try:
self.clearml = ClearmlLogger(self.opt, self.hyp)
except Exception:
self.clearml = None
prefix = colorstr('ClearML: ')
LOGGER.warning(f'{prefix}WARNING ⚠️ ClearML is installed but not configured, skipping ClearML logging.'
f' See https://docs.ultralytics.com/yolov5/tutorials/clearml_logging_integration#readme')
prefix = colorstr("ClearML: ")
LOGGER.warning(
f"{prefix}WARNING ⚠️ ClearML is installed but not configured, skipping ClearML logging."
f" See https://docs.ultralytics.com/yolov5/tutorials/clearml_logging_integration#readme"
)
else:
self.clearml = None
# Comet
if comet_ml and 'comet' in self.include:
if isinstance(self.opt.resume, str) and self.opt.resume.startswith('comet://'):
run_id = self.opt.resume.split('/')[-1]
if comet_ml and "comet" in self.include:
if isinstance(self.opt.resume, str) and self.opt.resume.startswith("comet://"):
run_id = self.opt.resume.split("/")[-1]
self.comet_logger = CometLogger(self.opt, self.hyp, run_id=run_id)
else:
@@ -170,9 +175,9 @@ class Loggers():
# Callback runs on pre-train routine end
if self.plots:
plot_labels(labels, names, self.save_dir)
paths = self.save_dir.glob('*labels*.jpg') # training labels
paths = self.save_dir.glob("*labels*.jpg") # training labels
if self.wandb:
self.wandb.log({'Labels': [wandb.Image(str(x), caption=x.name) for x in paths]})
self.wandb.log({"Labels": [wandb.Image(str(x), caption=x.name) for x in paths]})
if self.comet_logger:
self.comet_logger.on_pretrain_routine_end(paths)
if self.clearml:
@@ -185,16 +190,16 @@ class Loggers():
# ni: number integrated batches (since train start)
if self.plots:
if ni < 3:
f = self.save_dir / f'train_batch{ni}.jpg' # filename
f = self.save_dir / f"train_batch{ni}.jpg" # filename
plot_images(imgs, targets, paths, f)
if ni == 0 and self.tb and not self.opt.sync_bn:
log_tensorboard_graph(self.tb, model, imgsz=(self.opt.imgsz, self.opt.imgsz))
if ni == 10 and (self.wandb or self.clearml):
files = sorted(self.save_dir.glob('train*.jpg'))
files = sorted(self.save_dir.glob("train*.jpg"))
if self.wandb:
self.wandb.log({'Mosaics': [wandb.Image(str(f), caption=f.name) for f in files if f.exists()]})
self.wandb.log({"Mosaics": [wandb.Image(str(f), caption=f.name) for f in files if f.exists()]})
if self.clearml:
self.clearml.log_debug_samples(files, title='Mosaics')
self.clearml.log_debug_samples(files, title="Mosaics")
if self.comet_logger:
self.comet_logger.on_train_batch_end(log_dict, step=ni)
@@ -225,11 +230,11 @@ class Loggers():
def on_val_end(self, nt, tp, fp, p, r, f1, ap, ap50, ap_class, confusion_matrix):
# Callback runs on val end
if self.wandb or self.clearml:
files = sorted(self.save_dir.glob('val*.jpg'))
files = sorted(self.save_dir.glob("val*.jpg"))
if self.wandb:
self.wandb.log({'Validation': [wandb.Image(str(f), caption=f.name) for f in files]})
self.wandb.log({"Validation": [wandb.Image(str(f), caption=f.name) for f in files]})
if self.clearml:
self.clearml.log_debug_samples(files, title='Validation')
self.clearml.log_debug_samples(files, title="Validation")
if self.comet_logger:
self.comet_logger.on_val_end(nt, tp, fp, p, r, f1, ap, ap50, ap_class, confusion_matrix)
@@ -238,18 +243,18 @@ class Loggers():
# Callback runs at the end of each fit (train+val) epoch
x = dict(zip(self.keys, vals))
if self.csv:
file = self.save_dir / 'results.csv'
file = self.save_dir / "results.csv"
n = len(x) + 1 # number of cols
s = '' if file.exists() else (('%20s,' * n % tuple(['epoch'] + self.keys)).rstrip(',') + '\n') # add header
with open(file, 'a') as f:
f.write(s + ('%20.5g,' * n % tuple([epoch] + vals)).rstrip(',') + '\n')
s = "" if file.exists() else (("%20s," * n % tuple(["epoch"] + self.keys)).rstrip(",") + "\n") # add header
with open(file, "a") as f:
f.write(s + ("%20.5g," * n % tuple([epoch] + vals)).rstrip(",") + "\n")
if self.ndjson_console or self.ndjson_file:
json_data = json.dumps(dict(epoch=epoch, **x), default=_json_default)
if self.ndjson_console:
print(json_data)
if self.ndjson_file:
file = self.save_dir / 'results.ndjson'
with open(file, 'a') as f:
file = self.save_dir / "results.ndjson"
with open(file, "a") as f:
print(json_data, file=f)
if self.tb:
@@ -279,9 +284,9 @@ class Loggers():
if self.wandb:
self.wandb.log_model(last.parent, self.opt, epoch, fi, best_model=best_fitness == fi)
if self.clearml:
self.clearml.task.update_output_model(model_path=str(last),
model_name='Latest Model',
auto_delete_file=False)
self.clearml.task.update_output_model(
model_path=str(last), model_name="Latest Model", auto_delete_file=False
)
if self.comet_logger:
self.comet_logger.on_model_save(last, epoch, final_epoch, best_fitness, fi)
@@ -289,31 +294,34 @@ class Loggers():
def on_train_end(self, last, best, epoch, results):
# Callback runs on training end, i.e. saving best model
if self.plots:
plot_results(file=self.save_dir / 'results.csv') # save results.png
files = ['results.png', 'confusion_matrix.png', *(f'{x}_curve.png' for x in ('F1', 'PR', 'P', 'R'))]
plot_results(file=self.save_dir / "results.csv") # save results.png
files = ["results.png", "confusion_matrix.png", *(f"{x}_curve.png" for x in ("F1", "PR", "P", "R"))]
files = [(self.save_dir / f) for f in files if (self.save_dir / f).exists()] # filter
self.logger.info(f"Results saved to {colorstr('bold', self.save_dir)}")
if self.tb and not self.clearml: # These images are already captured by ClearML by now, we don't want doubles
for f in files:
self.tb.add_image(f.stem, cv2.imread(str(f))[..., ::-1], epoch, dataformats='HWC')
self.tb.add_image(f.stem, cv2.imread(str(f))[..., ::-1], epoch, dataformats="HWC")
if self.wandb:
self.wandb.log(dict(zip(self.keys[3:10], results)))
self.wandb.log({'Results': [wandb.Image(str(f), caption=f.name) for f in files]})
self.wandb.log({"Results": [wandb.Image(str(f), caption=f.name) for f in files]})
# Calling wandb.log. TODO: Refactor this into WandbLogger.log_model
if not self.opt.evolve:
wandb.log_artifact(str(best if best.exists() else last),
type='model',
name=f'run_{self.wandb.wandb_run.id}_model',
aliases=['latest', 'best', 'stripped'])
wandb.log_artifact(
str(best if best.exists() else last),
type="model",
name=f"run_{self.wandb.wandb_run.id}_model",
aliases=["latest", "best", "stripped"],
)
self.wandb.finish_run()
if self.clearml and not self.opt.evolve:
self.clearml.log_summary(dict(zip(self.keys[3:10], results)))
[self.clearml.log_plot(title=f.stem, plot_path=f) for f in files]
self.clearml.log_model(str(best if best.exists() else last),
"Best Model" if best.exists() else "Last Model", epoch)
self.clearml.log_model(
str(best if best.exists() else last), "Best Model" if best.exists() else "Last Model", epoch
)
if self.comet_logger:
final_results = dict(zip(self.keys[3:10], results))
@@ -339,38 +347,41 @@ class GenericLogger:
include: loggers to include
"""
def __init__(self, opt, console_logger, include=('tb', 'wandb', 'clearml')):
def __init__(self, opt, console_logger, include=("tb", "wandb", "clearml")):
# init default loggers
self.save_dir = Path(opt.save_dir)
self.include = include
self.console_logger = console_logger
self.csv = self.save_dir / 'results.csv' # CSV logger
if 'tb' in self.include:
prefix = colorstr('TensorBoard: ')
self.csv = self.save_dir / "results.csv" # CSV logger
if "tb" in self.include:
prefix = colorstr("TensorBoard: ")
self.console_logger.info(
f"{prefix}Start with 'tensorboard --logdir {self.save_dir.parent}', view at http://localhost:6006/")
f"{prefix}Start with 'tensorboard --logdir {self.save_dir.parent}', view at http://localhost:6006/"
)
self.tb = SummaryWriter(str(self.save_dir))
if wandb and 'wandb' in self.include:
self.wandb = wandb.init(project=web_project_name(str(opt.project)),
name=None if opt.name == 'exp' else opt.name,
config=opt)
if wandb and "wandb" in self.include:
self.wandb = wandb.init(
project=web_project_name(str(opt.project)), name=None if opt.name == "exp" else opt.name, config=opt
)
else:
self.wandb = None
if clearml and 'clearml' in self.include:
if clearml and "clearml" in self.include:
try:
# Hyp is not available in classification mode
if 'hyp' not in opt:
if "hyp" not in opt:
hyp = {}
else:
hyp = opt.hyp
self.clearml = ClearmlLogger(opt, hyp)
except Exception:
self.clearml = None
prefix = colorstr('ClearML: ')
LOGGER.warning(f'{prefix}WARNING ⚠️ ClearML is installed but not configured, skipping ClearML logging.'
f' See https://github.com/ultralytics/yolov5/tree/master/utils/loggers/clearml#readme')
prefix = colorstr("ClearML: ")
LOGGER.warning(
f"{prefix}WARNING ⚠️ ClearML is installed but not configured, skipping ClearML logging."
f" See https://github.com/ultralytics/yolov5/tree/master/utils/loggers/clearml#readme"
)
else:
self.clearml = None
@@ -379,9 +390,9 @@ class GenericLogger:
if self.csv:
keys, vals = list(metrics.keys()), list(metrics.values())
n = len(metrics) + 1 # number of cols
s = '' if self.csv.exists() else (('%23s,' * n % tuple(['epoch'] + keys)).rstrip(',') + '\n') # header
with open(self.csv, 'a') as f:
f.write(s + ('%23.5g,' * n % tuple([epoch] + vals)).rstrip(',') + '\n')
s = "" if self.csv.exists() else (("%23s," * n % tuple(["epoch"] + keys)).rstrip(",") + "\n") # header
with open(self.csv, "a") as f:
f.write(s + ("%23.5g," * n % tuple([epoch] + vals)).rstrip(",") + "\n")
if self.tb:
for k, v in metrics.items():
@@ -393,20 +404,20 @@ class GenericLogger:
if self.clearml:
self.clearml.log_scalars(metrics, epoch)
def log_images(self, files, name='Images', epoch=0):
def log_images(self, files, name="Images", epoch=0):
# Log images to all loggers
files = [Path(f) for f in (files if isinstance(files, (tuple, list)) else [files])] # to Path
files = [f for f in files if f.exists()] # filter by exists
if self.tb:
for f in files:
self.tb.add_image(f.stem, cv2.imread(str(f))[..., ::-1], epoch, dataformats='HWC')
self.tb.add_image(f.stem, cv2.imread(str(f))[..., ::-1], epoch, dataformats="HWC")
if self.wandb:
self.wandb.log({name: [wandb.Image(str(f), caption=f.name) for f in files]}, step=epoch)
if self.clearml:
if name == 'Results':
if name == "Results":
[self.clearml.log_plot(f.stem, f) for f in files]
else:
self.clearml.log_debug_samples(files, title=name)
@@ -419,7 +430,7 @@ class GenericLogger:
def log_model(self, model_path, epoch=0, metadata={}):
# Log model to all loggers
if self.wandb:
art = wandb.Artifact(name=f'run_{wandb.run.id}_model', type='model', metadata=metadata)
art = wandb.Artifact(name=f"run_{wandb.run.id}_model", type="model", metadata=metadata)
art.add_file(str(model_path))
wandb.log_artifact(art)
if self.clearml:
@@ -440,15 +451,15 @@ def log_tensorboard_graph(tb, model, imgsz=(640, 640)):
imgsz = (imgsz, imgsz) if isinstance(imgsz, int) else imgsz # expand
im = torch.zeros((1, 3, *imgsz)).to(p.device).type_as(p) # input image (WARNING: must be zeros, not empty)
with warnings.catch_warnings():
warnings.simplefilter('ignore') # suppress jit trace warning
warnings.simplefilter("ignore") # suppress jit trace warning
tb.add_graph(torch.jit.trace(de_parallel(model), im, strict=False), [])
except Exception as e:
LOGGER.warning(f'WARNING ⚠️ TensorBoard graph visualization failure {e}')
LOGGER.warning(f"WARNING ⚠️ TensorBoard graph visualization failure {e}")
def web_project_name(project):
# Convert local project name to web project name
if not project.startswith('runs/train'):
if not project.startswith("runs/train"):
return project
suffix = '-Classify' if project.endswith('-cls') else '-Segment' if project.endswith('-seg') else ''
return f'YOLOv5{suffix}'
suffix = "-Classify" if project.endswith("-cls") else "-Segment" if project.endswith("-seg") else ""
return f"YOLOv5{suffix}"
utils/loggers/clearml/clearml_utils.py
+64 -59
View File
@@ -13,55 +13,63 @@ try:
import clearml
from clearml import Dataset, Task
assert hasattr(clearml, '__version__') # verify package import not local dir
assert hasattr(clearml, "__version__") # verify package import not local dir
except (ImportError, AssertionError):
clearml = None
def construct_dataset(clearml_info_string):
"""Load in a clearml dataset and fill the internal data_dict with its contents.
"""
dataset_id = clearml_info_string.replace('clearml://', '')
"""Load in a clearml dataset and fill the internal data_dict with its contents."""
dataset_id = clearml_info_string.replace("clearml://", "")
dataset = Dataset.get(dataset_id=dataset_id)
dataset_root_path = Path(dataset.get_local_copy())
# We'll search for the yaml file definition in the dataset
yaml_filenames = list(glob.glob(str(dataset_root_path / '*.yaml')) + glob.glob(str(dataset_root_path / '*.yml')))
yaml_filenames = list(glob.glob(str(dataset_root_path / "*.yaml")) + glob.glob(str(dataset_root_path / "*.yml")))
if len(yaml_filenames) > 1:
raise ValueError('More than one yaml file was found in the dataset root, cannot determine which one contains '
'the dataset definition this way.')
raise ValueError(
"More than one yaml file was found in the dataset root, cannot determine which one contains "
"the dataset definition this way."
)
elif len(yaml_filenames) == 0:
raise ValueError('No yaml definition found in dataset root path, check that there is a correct yaml file '
'inside the dataset root path.')
raise ValueError(
"No yaml definition found in dataset root path, check that there is a correct yaml file "
"inside the dataset root path."
)
with open(yaml_filenames[0]) as f:
dataset_definition = yaml.safe_load(f)
assert set(dataset_definition.keys()).issuperset(
{'train', 'test', 'val', 'nc', 'names'}
assert set(
dataset_definition.keys()
).issuperset(
{"train", "test", "val", "nc", "names"}
), "The right keys were not found in the yaml file, make sure it at least has the following keys: ('train', 'test', 'val', 'nc', 'names')"
data_dict = dict()
data_dict['train'] = str(
(dataset_root_path / dataset_definition['train']).resolve()) if dataset_definition['train'] else None
data_dict['test'] = str(
(dataset_root_path / dataset_definition['test']).resolve()) if dataset_definition['test'] else None
data_dict['val'] = str(
(dataset_root_path / dataset_definition['val']).resolve()) if dataset_definition['val'] else None
data_dict['nc'] = dataset_definition['nc']
data_dict['names'] = dataset_definition['names']
data_dict["train"] = (
str((dataset_root_path / dataset_definition["train"]).resolve()) if dataset_definition["train"] else None
)
data_dict["test"] = (
str((dataset_root_path / dataset_definition["test"]).resolve()) if dataset_definition["test"] else None
)
data_dict["val"] = (
str((dataset_root_path / dataset_definition["val"]).resolve()) if dataset_definition["val"] else None
)
data_dict["nc"] = dataset_definition["nc"]
data_dict["names"] = dataset_definition["names"]
return data_dict
class ClearmlLogger:
"""Log training runs, datasets, models, and predictions to ClearML.
"""
Log training runs, datasets, models, and predictions to ClearML.
This logger sends information to ClearML at app.clear.ml or to your own hosted server. By default,
this information includes hyperparameters, system configuration and metrics, model metrics, code information and
basic data metrics and analyses.
This logger sends information to ClearML at app.clear.ml or to your own hosted server. By default, this information
includes hyperparameters, system configuration and metrics, model metrics, code information and basic data metrics
and analyses.
By providing additional command line arguments to train.py, datasets,
models and predictions can also be logged.
By providing additional command line arguments to train.py, datasets, models and predictions can also be logged.
"""
def __init__(self, opt, hyp):
@@ -81,36 +89,36 @@ class ClearmlLogger:
self.max_imgs_to_log_per_epoch = 16
# Get the interval of epochs when bounding box images should be logged
# Only for detection task though!
if 'bbox_interval' in opt:
if "bbox_interval" in opt:
self.bbox_interval = opt.bbox_interval
self.clearml = clearml
self.task = None
self.data_dict = None
if self.clearml:
self.task = Task.init(
project_name=opt.project if not str(opt.project).startswith('runs/') else 'YOLOv5',
task_name=opt.name if opt.name != 'exp' else 'Training',
tags=['YOLOv5'],
project_name=opt.project if not str(opt.project).startswith("runs/") else "YOLOv5",
task_name=opt.name if opt.name != "exp" else "Training",
tags=["YOLOv5"],
output_uri=True,
reuse_last_task_id=opt.exist_ok,
auto_connect_frameworks={
'pytorch': False,
'matplotlib': False}
auto_connect_frameworks={"pytorch": False, "matplotlib": False},
# We disconnect pytorch auto-detection, because we added manual model save points in the code
)
# ClearML's hooks will already grab all general parameters
# Only the hyperparameters coming from the yaml config file
# will have to be added manually!
self.task.connect(hyp, name='Hyperparameters')
self.task.connect(opt, name='Args')
self.task.connect(hyp, name="Hyperparameters")
self.task.connect(opt, name="Args")
# Make sure the code is easily remotely runnable by setting the docker image to use by the remote agent
self.task.set_base_docker('ultralytics/yolov5:latest',
docker_arguments='--ipc=host -e="CLEARML_AGENT_SKIP_PYTHON_ENV_INSTALL=1"',
docker_setup_bash_script='pip install clearml')
self.task.set_base_docker(
"ultralytics/yolov5:latest",
docker_arguments='--ipc=host -e="CLEARML_AGENT_SKIP_PYTHON_ENV_INSTALL=1"',
docker_setup_bash_script="pip install clearml",
)
# Get ClearML Dataset Version if requested
if opt.data.startswith('clearml://'):
if opt.data.startswith("clearml://"):
# data_dict should have the following keys:
# names, nc (number of classes), test, train, val (all three relative paths to ../datasets)
self.data_dict = construct_dataset(opt.data)
@@ -120,33 +128,32 @@ class ClearmlLogger:
def log_scalars(self, metrics, epoch):
"""
Log scalars/metrics to ClearML
Log scalars/metrics to ClearML.
arguments:
metrics (dict) Metrics in dict format: {"metrics/mAP": 0.8, ...}
epoch (int) iteration number for the current set of metrics
"""
for k, v in metrics.items():
title, series = k.split('/')
title, series = k.split("/")
self.task.get_logger().report_scalar(title, series, v, epoch)
def log_model(self, model_path, model_name, epoch=0):
"""
Log model weights to ClearML
Log model weights to ClearML.
arguments:
model_path (PosixPath or str) Path to the model weights
model_name (str) Name of the model visible in ClearML
epoch (int) Iteration / epoch of the model weights
"""
self.task.update_output_model(model_path=str(model_path),
name=model_name,
iteration=epoch,
auto_delete_file=False)
self.task.update_output_model(
model_path=str(model_path), name=model_name, iteration=epoch, auto_delete_file=False
)
def log_summary(self, metrics):
"""
Log final metrics to a summary table
Log final metrics to a summary table.
arguments:
metrics (dict) Metrics in dict format: {"metrics/mAP": 0.8, ...}
@@ -156,7 +163,7 @@ class ClearmlLogger:
def log_plot(self, title, plot_path):
"""
Log image as plot in the plot section of ClearML
Log image as plot in the plot section of ClearML.
arguments:
title (str) Title of the plot
@@ -164,12 +171,12 @@ class ClearmlLogger:
"""
img = mpimg.imread(plot_path)
fig = plt.figure()
ax = fig.add_axes([0, 0, 1, 1], frameon=False, aspect='auto', xticks=[], yticks=[]) # no ticks
ax = fig.add_axes([0, 0, 1, 1], frameon=False, aspect="auto", xticks=[], yticks=[]) # no ticks
ax.imshow(img)
self.task.get_logger().report_matplotlib_figure(title, "", figure=fig, report_interactive=False)
def log_debug_samples(self, files, title='Debug Samples'):
def log_debug_samples(self, files, title="Debug Samples"):
"""
Log files (images) as debug samples in the ClearML task.
@@ -179,12 +186,11 @@ class ClearmlLogger:
"""
for f in files:
if f.exists():
it = re.search(r'_batch(\d+)', f.name)
it = re.search(r"_batch(\d+)", f.name)
iteration = int(it.groups()[0]) if it else 0
self.task.get_logger().report_image(title=title,
series=f.name.replace(f"_batch{iteration}", ''),
local_path=str(f),
iteration=iteration)
self.task.get_logger().report_image(
title=title, series=f.name.replace(f"_batch{iteration}", ""), local_path=str(f), iteration=iteration
)
def log_image_with_boxes(self, image_path, boxes, class_names, image, conf_threshold=0.25):
"""
@@ -206,15 +212,14 @@ class ClearmlLogger:
class_name = class_names[int(class_nr)]
confidence_percentage = round(float(conf) * 100, 2)
label = f'{class_name}: {confidence_percentage}%'
label = f"{class_name}: {confidence_percentage}%"
if conf > conf_threshold:
annotator.rectangle(box.cpu().numpy(), outline=color)
annotator.box_label(box.cpu().numpy(), label=label, color=color)
annotated_image = annotator.result()
self.task.get_logger().report_image(title='Bounding Boxes',
series=image_path.name,
iteration=self.current_epoch,
image=annotated_image)
self.task.get_logger().report_image(
title="Bounding Boxes", series=image_path.name, iteration=self.current_epoch, image=annotated_image
)
self.current_epoch_logged_images.add(image_path)
utils/loggers/clearml/hpo.py
+41 -37
View File
@@ -1,18 +1,21 @@
from clearml import Task
# Connecting ClearML with the current process,
# from here on everything is logged automatically
from clearml.automation import HyperParameterOptimizer, UniformParameterRange
from clearml.automation.optuna import OptimizerOptuna
task = Task.init(project_name='Hyper-Parameter Optimization',
task_name='YOLOv5',
task_type=Task.TaskTypes.optimizer,
reuse_last_task_id=False)
task = Task.init(
project_name="Hyper-Parameter Optimization",
task_name="YOLOv5",
task_type=Task.TaskTypes.optimizer,
reuse_last_task_id=False,
)
# Example use case:
optimizer = HyperParameterOptimizer(
# This is the experiment we want to optimize
base_task_id='<your_template_task_id>',
base_task_id="<your_template_task_id>",
# here we define the hyper-parameters to optimize
# Notice: The parameter name should exactly match what you see in the UI: <section_name>/<parameter>
# For Example, here we see in the base experiment a section Named: "General"
@@ -20,39 +23,40 @@ optimizer = HyperParameterOptimizer(
# If you have `argparse` for example, then arguments will appear under the "Args" section,
# and you should instead pass "Args/batch_size"
hyper_parameters=[
UniformParameterRange('Hyperparameters/lr0', min_value=1e-5, max_value=1e-1),
UniformParameterRange('Hyperparameters/lrf', min_value=0.01, max_value=1.0),
UniformParameterRange('Hyperparameters/momentum', min_value=0.6, max_value=0.98),
UniformParameterRange('Hyperparameters/weight_decay', min_value=0.0, max_value=0.001),
UniformParameterRange('Hyperparameters/warmup_epochs', min_value=0.0, max_value=5.0),
UniformParameterRange('Hyperparameters/warmup_momentum', min_value=0.0, max_value=0.95),
UniformParameterRange('Hyperparameters/warmup_bias_lr', min_value=0.0, max_value=0.2),
UniformParameterRange('Hyperparameters/box', min_value=0.02, max_value=0.2),
UniformParameterRange('Hyperparameters/cls', min_value=0.2, max_value=4.0),
UniformParameterRange('Hyperparameters/cls_pw', min_value=0.5, max_value=2.0),
UniformParameterRange('Hyperparameters/obj', min_value=0.2, max_value=4.0),
UniformParameterRange('Hyperparameters/obj_pw', min_value=0.5, max_value=2.0),
UniformParameterRange('Hyperparameters/iou_t', min_value=0.1, max_value=0.7),
UniformParameterRange('Hyperparameters/anchor_t', min_value=2.0, max_value=8.0),
UniformParameterRange('Hyperparameters/fl_gamma', min_value=0.0, max_value=4.0),
UniformParameterRange('Hyperparameters/hsv_h', min_value=0.0, max_value=0.1),
UniformParameterRange('Hyperparameters/hsv_s', min_value=0.0, max_value=0.9),
UniformParameterRange('Hyperparameters/hsv_v', min_value=0.0, max_value=0.9),
UniformParameterRange('Hyperparameters/degrees', min_value=0.0, max_value=45.0),
UniformParameterRange('Hyperparameters/translate', min_value=0.0, max_value=0.9),
UniformParameterRange('Hyperparameters/scale', min_value=0.0, max_value=0.9),
UniformParameterRange('Hyperparameters/shear', min_value=0.0, max_value=10.0),
UniformParameterRange('Hyperparameters/perspective', min_value=0.0, max_value=0.001),
UniformParameterRange('Hyperparameters/flipud', min_value=0.0, max_value=1.0),
UniformParameterRange('Hyperparameters/fliplr', min_value=0.0, max_value=1.0),
UniformParameterRange('Hyperparameters/mosaic', min_value=0.0, max_value=1.0),
UniformParameterRange('Hyperparameters/mixup', min_value=0.0, max_value=1.0),
UniformParameterRange('Hyperparameters/copy_paste', min_value=0.0, max_value=1.0)],
UniformParameterRange("Hyperparameters/lr0", min_value=1e-5, max_value=1e-1),
UniformParameterRange("Hyperparameters/lrf", min_value=0.01, max_value=1.0),
UniformParameterRange("Hyperparameters/momentum", min_value=0.6, max_value=0.98),
UniformParameterRange("Hyperparameters/weight_decay", min_value=0.0, max_value=0.001),
UniformParameterRange("Hyperparameters/warmup_epochs", min_value=0.0, max_value=5.0),
UniformParameterRange("Hyperparameters/warmup_momentum", min_value=0.0, max_value=0.95),
UniformParameterRange("Hyperparameters/warmup_bias_lr", min_value=0.0, max_value=0.2),
UniformParameterRange("Hyperparameters/box", min_value=0.02, max_value=0.2),
UniformParameterRange("Hyperparameters/cls", min_value=0.2, max_value=4.0),
UniformParameterRange("Hyperparameters/cls_pw", min_value=0.5, max_value=2.0),
UniformParameterRange("Hyperparameters/obj", min_value=0.2, max_value=4.0),
UniformParameterRange("Hyperparameters/obj_pw", min_value=0.5, max_value=2.0),
UniformParameterRange("Hyperparameters/iou_t", min_value=0.1, max_value=0.7),
UniformParameterRange("Hyperparameters/anchor_t", min_value=2.0, max_value=8.0),
UniformParameterRange("Hyperparameters/fl_gamma", min_value=0.0, max_value=4.0),
UniformParameterRange("Hyperparameters/hsv_h", min_value=0.0, max_value=0.1),
UniformParameterRange("Hyperparameters/hsv_s", min_value=0.0, max_value=0.9),
UniformParameterRange("Hyperparameters/hsv_v", min_value=0.0, max_value=0.9),
UniformParameterRange("Hyperparameters/degrees", min_value=0.0, max_value=45.0),
UniformParameterRange("Hyperparameters/translate", min_value=0.0, max_value=0.9),
UniformParameterRange("Hyperparameters/scale", min_value=0.0, max_value=0.9),
UniformParameterRange("Hyperparameters/shear", min_value=0.0, max_value=10.0),
UniformParameterRange("Hyperparameters/perspective", min_value=0.0, max_value=0.001),
UniformParameterRange("Hyperparameters/flipud", min_value=0.0, max_value=1.0),
UniformParameterRange("Hyperparameters/fliplr", min_value=0.0, max_value=1.0),
UniformParameterRange("Hyperparameters/mosaic", min_value=0.0, max_value=1.0),
UniformParameterRange("Hyperparameters/mixup", min_value=0.0, max_value=1.0),
UniformParameterRange("Hyperparameters/copy_paste", min_value=0.0, max_value=1.0),
],
# this is the objective metric we want to maximize/minimize
objective_metric_title='metrics',
objective_metric_series='mAP_0.5',
objective_metric_title="metrics",
objective_metric_series="mAP_0.5",
# now we decide if we want to maximize it or minimize it (accuracy we maximize)
objective_metric_sign='max',
objective_metric_sign="max",
# let us limit the number of concurrent experiments,
# this in turn will make sure we do dont bombard the scheduler with experiments.
# if we have an auto-scaler connected, this, by proxy, will limit the number of machine
@@ -81,4 +85,4 @@ optimizer.wait()
# make sure background optimization stopped
optimizer.stop()
print('We are done, good bye')
print("We are done, good bye")
utils/loggers/comet/__init__.py
+125 -118
View File
@@ -17,7 +17,7 @@ try:
# Project Configuration
config = comet_ml.config.get_config()
COMET_PROJECT_NAME = config.get_string(os.getenv('COMET_PROJECT_NAME'), 'comet.project_name', default='yolov5')
COMET_PROJECT_NAME = config.get_string(os.getenv("COMET_PROJECT_NAME"), "comet.project_name", default="yolov5")
except ImportError:
comet_ml = None
COMET_PROJECT_NAME = None
@@ -31,42 +31,40 @@ from utils.dataloaders import img2label_paths
from utils.general import check_dataset, scale_boxes, xywh2xyxy
from utils.metrics import box_iou
COMET_PREFIX = 'comet://'
COMET_PREFIX = "comet://"
COMET_MODE = os.getenv('COMET_MODE', 'online')
COMET_MODE = os.getenv("COMET_MODE", "online")
# Model Saving Settings
COMET_MODEL_NAME = os.getenv('COMET_MODEL_NAME', 'yolov5')
COMET_MODEL_NAME = os.getenv("COMET_MODEL_NAME", "yolov5")
# Dataset Artifact Settings
COMET_UPLOAD_DATASET = os.getenv('COMET_UPLOAD_DATASET', 'false').lower() == 'true'
COMET_UPLOAD_DATASET = os.getenv("COMET_UPLOAD_DATASET", "false").lower() == "true"
# Evaluation Settings
COMET_LOG_CONFUSION_MATRIX = (os.getenv('COMET_LOG_CONFUSION_MATRIX', 'true').lower() == 'true')
COMET_LOG_PREDICTIONS = os.getenv('COMET_LOG_PREDICTIONS', 'true').lower() == 'true'
COMET_MAX_IMAGE_UPLOADS = int(os.getenv('COMET_MAX_IMAGE_UPLOADS', 100))
COMET_LOG_CONFUSION_MATRIX = os.getenv("COMET_LOG_CONFUSION_MATRIX", "true").lower() == "true"
COMET_LOG_PREDICTIONS = os.getenv("COMET_LOG_PREDICTIONS", "true").lower() == "true"
COMET_MAX_IMAGE_UPLOADS = int(os.getenv("COMET_MAX_IMAGE_UPLOADS", 100))
# Confusion Matrix Settings
CONF_THRES = float(os.getenv('CONF_THRES', 0.001))
IOU_THRES = float(os.getenv('IOU_THRES', 0.6))
CONF_THRES = float(os.getenv("CONF_THRES", 0.001))
IOU_THRES = float(os.getenv("IOU_THRES", 0.6))
# Batch Logging Settings
COMET_LOG_BATCH_METRICS = (os.getenv('COMET_LOG_BATCH_METRICS', 'false').lower() == 'true')
COMET_BATCH_LOGGING_INTERVAL = os.getenv('COMET_BATCH_LOGGING_INTERVAL', 1)
COMET_PREDICTION_LOGGING_INTERVAL = os.getenv('COMET_PREDICTION_LOGGING_INTERVAL', 1)
COMET_LOG_PER_CLASS_METRICS = (os.getenv('COMET_LOG_PER_CLASS_METRICS', 'false').lower() == 'true')
COMET_LOG_BATCH_METRICS = os.getenv("COMET_LOG_BATCH_METRICS", "false").lower() == "true"
COMET_BATCH_LOGGING_INTERVAL = os.getenv("COMET_BATCH_LOGGING_INTERVAL", 1)
COMET_PREDICTION_LOGGING_INTERVAL = os.getenv("COMET_PREDICTION_LOGGING_INTERVAL", 1)
COMET_LOG_PER_CLASS_METRICS = os.getenv("COMET_LOG_PER_CLASS_METRICS", "false").lower() == "true"
RANK = int(os.getenv('RANK', -1))
RANK = int(os.getenv("RANK", -1))
to_pil = T.ToPILImage()
class CometLogger:
"""Log metrics, parameters, source code, models and much more
with Comet
"""
"""Log metrics, parameters, source code, models and much more with Comet."""
def __init__(self, opt, hyp, run_id=None, job_type='Training', **experiment_kwargs) -> None:
def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwargs) -> None:
self.job_type = job_type
self.opt = opt
self.hyp = hyp
@@ -87,57 +85,58 @@ class CometLogger:
# Default parameters to pass to Experiment objects
self.default_experiment_kwargs = {
'log_code': False,
'log_env_gpu': True,
'log_env_cpu': True,
'project_name': COMET_PROJECT_NAME, }
"log_code": False,
"log_env_gpu": True,
"log_env_cpu": True,
"project_name": COMET_PROJECT_NAME,
}
self.default_experiment_kwargs.update(experiment_kwargs)
self.experiment = self._get_experiment(self.comet_mode, run_id)
self.experiment.set_name(self.opt.name)
self.data_dict = self.check_dataset(self.opt.data)
self.class_names = self.data_dict['names']
self.num_classes = self.data_dict['nc']
self.class_names = self.data_dict["names"]
self.num_classes = self.data_dict["nc"]
self.logged_images_count = 0
self.max_images = COMET_MAX_IMAGE_UPLOADS
if run_id is None:
self.experiment.log_other('Created from', 'YOLOv5')
self.experiment.log_other("Created from", "YOLOv5")
if not isinstance(self.experiment, comet_ml.OfflineExperiment):
workspace, project_name, experiment_id = self.experiment.url.split('/')[-3:]
workspace, project_name, experiment_id = self.experiment.url.split("/")[-3:]
self.experiment.log_other(
'Run Path',
f'{workspace}/{project_name}/{experiment_id}',
"Run Path",
f"{workspace}/{project_name}/{experiment_id}",
)
self.log_parameters(vars(opt))
self.log_parameters(self.opt.hyp)
self.log_asset_data(
self.opt.hyp,
name='hyperparameters.json',
metadata={'type': 'hyp-config-file'},
name="hyperparameters.json",
metadata={"type": "hyp-config-file"},
)
self.log_asset(
f'{self.opt.save_dir}/opt.yaml',
metadata={'type': 'opt-config-file'},
f"{self.opt.save_dir}/opt.yaml",
metadata={"type": "opt-config-file"},
)
self.comet_log_confusion_matrix = COMET_LOG_CONFUSION_MATRIX
if hasattr(self.opt, 'conf_thres'):
if hasattr(self.opt, "conf_thres"):
self.conf_thres = self.opt.conf_thres
else:
self.conf_thres = CONF_THRES
if hasattr(self.opt, 'iou_thres'):
if hasattr(self.opt, "iou_thres"):
self.iou_thres = self.opt.iou_thres
else:
self.iou_thres = IOU_THRES
self.log_parameters({'val_iou_threshold': self.iou_thres, 'val_conf_threshold': self.conf_thres})
self.log_parameters({"val_iou_threshold": self.iou_thres, "val_conf_threshold": self.conf_thres})
self.comet_log_predictions = COMET_LOG_PREDICTIONS
if self.opt.bbox_interval == -1:
self.comet_log_prediction_interval = (1 if self.opt.epochs < 10 else self.opt.epochs // 10)
self.comet_log_prediction_interval = 1 if self.opt.epochs < 10 else self.opt.epochs // 10
else:
self.comet_log_prediction_interval = self.opt.bbox_interval
@@ -147,30 +146,35 @@ class CometLogger:
self.comet_log_per_class_metrics = COMET_LOG_PER_CLASS_METRICS
self.experiment.log_others({
'comet_mode': COMET_MODE,
'comet_max_image_uploads': COMET_MAX_IMAGE_UPLOADS,
'comet_log_per_class_metrics': COMET_LOG_PER_CLASS_METRICS,
'comet_log_batch_metrics': COMET_LOG_BATCH_METRICS,
'comet_log_confusion_matrix': COMET_LOG_CONFUSION_MATRIX,
'comet_model_name': COMET_MODEL_NAME, })
self.experiment.log_others(
{
"comet_mode": COMET_MODE,
"comet_max_image_uploads": COMET_MAX_IMAGE_UPLOADS,
"comet_log_per_class_metrics": COMET_LOG_PER_CLASS_METRICS,
"comet_log_batch_metrics": COMET_LOG_BATCH_METRICS,
"comet_log_confusion_matrix": COMET_LOG_CONFUSION_MATRIX,
"comet_model_name": COMET_MODEL_NAME,
}
)
# Check if running the Experiment with the Comet Optimizer
if hasattr(self.opt, 'comet_optimizer_id'):
self.experiment.log_other('optimizer_id', self.opt.comet_optimizer_id)
self.experiment.log_other('optimizer_objective', self.opt.comet_optimizer_objective)
self.experiment.log_other('optimizer_metric', self.opt.comet_optimizer_metric)
self.experiment.log_other('optimizer_parameters', json.dumps(self.hyp))
if hasattr(self.opt, "comet_optimizer_id"):
self.experiment.log_other("optimizer_id", self.opt.comet_optimizer_id)
self.experiment.log_other("optimizer_objective", self.opt.comet_optimizer_objective)
self.experiment.log_other("optimizer_metric", self.opt.comet_optimizer_metric)
self.experiment.log_other("optimizer_parameters", json.dumps(self.hyp))
def _get_experiment(self, mode, experiment_id=None):
if mode == 'offline':
if mode == "offline":
if experiment_id is not None:
return comet_ml.ExistingOfflineExperiment(
previous_experiment=experiment_id,
**self.default_experiment_kwargs,
)
return comet_ml.OfflineExperiment(**self.default_experiment_kwargs, )
return comet_ml.OfflineExperiment(
**self.default_experiment_kwargs,
)
else:
try:
@@ -183,11 +187,13 @@ class CometLogger:
return comet_ml.Experiment(**self.default_experiment_kwargs)
except ValueError:
logger.warning('COMET WARNING: '
'Comet credentials have not been set. '
'Comet will default to offline logging. '
'Please set your credentials to enable online logging.')
return self._get_experiment('offline', experiment_id)
logger.warning(
"COMET WARNING: "
"Comet credentials have not been set. "
"Comet will default to offline logging. "
"Please set your credentials to enable online logging."
)
return self._get_experiment("offline", experiment_id)
return
@@ -211,12 +217,13 @@ class CometLogger:
return
model_metadata = {
'fitness_score': fitness_score[-1],
'epochs_trained': epoch + 1,
'save_period': opt.save_period,
'total_epochs': opt.epochs, }
"fitness_score": fitness_score[-1],
"epochs_trained": epoch + 1,
"save_period": opt.save_period,
"total_epochs": opt.epochs,
}
model_files = glob.glob(f'{path}/*.pt')
model_files = glob.glob(f"{path}/*.pt")
for model_path in model_files:
name = Path(model_path).name
@@ -232,14 +239,14 @@ class CometLogger:
with open(data_file) as f:
data_config = yaml.safe_load(f)
path = data_config.get('path')
path = data_config.get("path")
if path and path.startswith(COMET_PREFIX):
path = data_config['path'].replace(COMET_PREFIX, '')
path = data_config["path"].replace(COMET_PREFIX, "")
data_dict = self.download_dataset_artifact(path)
return data_dict
self.log_asset(self.opt.data, metadata={'type': 'data-config-file'})
self.log_asset(self.opt.data, metadata={"type": "data-config-file"})
return check_dataset(data_file)
@@ -255,8 +262,8 @@ class CometLogger:
filtered_detections = detections[mask]
filtered_labels = labelsn[mask]
image_id = path.split('/')[-1].split('.')[0]
image_name = f'{image_id}_curr_epoch_{self.experiment.curr_epoch}'
image_id = path.split("/")[-1].split(".")[0]
image_name = f"{image_id}_curr_epoch_{self.experiment.curr_epoch}"
if image_name not in self.logged_image_names:
native_scale_image = PIL.Image.open(path)
self.log_image(native_scale_image, name=image_name)
@@ -264,23 +271,21 @@ class CometLogger:
metadata = []
for cls, *xyxy in filtered_labels.tolist():
metadata.append({
'label': f'{self.class_names[int(cls)]}-gt',
'score': 100,
'box': {
'x': xyxy[0],
'y': xyxy[1],
'x2': xyxy[2],
'y2': xyxy[3]}, })
metadata.append(
{
"label": f"{self.class_names[int(cls)]}-gt",
"score": 100,
"box": {"x": xyxy[0], "y": xyxy[1], "x2": xyxy[2], "y2": xyxy[3]},
}
)
for *xyxy, conf, cls in filtered_detections.tolist():
metadata.append({
'label': f'{self.class_names[int(cls)]}',
'score': conf * 100,
'box': {
'x': xyxy[0],
'y': xyxy[1],
'x2': xyxy[2],
'y2': xyxy[3]}, })
metadata.append(
{
"label": f"{self.class_names[int(cls)]}",
"score": conf * 100,
"box": {"x": xyxy[0], "y": xyxy[1], "x2": xyxy[2], "y2": xyxy[3]},
}
)
self.metadata_dict[image_name] = metadata
self.logged_images_count += 1
@@ -307,7 +312,7 @@ class CometLogger:
return predn, labelsn
def add_assets_to_artifact(self, artifact, path, asset_path, split):
img_paths = sorted(glob.glob(f'{asset_path}/*'))
img_paths = sorted(glob.glob(f"{asset_path}/*"))
label_paths = img2label_paths(img_paths)
for image_file, label_file in zip(img_paths, label_paths):
@@ -317,33 +322,33 @@ class CometLogger:
artifact.add(
image_file,
logical_path=image_logical_path,
metadata={'split': split},
metadata={"split": split},
)
artifact.add(
label_file,
logical_path=label_logical_path,
metadata={'split': split},
metadata={"split": split},
)
except ValueError as e:
logger.error('COMET ERROR: Error adding file to Artifact. Skipping file.')
logger.error(f'COMET ERROR: {e}')
logger.error("COMET ERROR: Error adding file to Artifact. Skipping file.")
logger.error(f"COMET ERROR: {e}")
continue
return artifact
def upload_dataset_artifact(self):
dataset_name = self.data_dict.get('dataset_name', 'yolov5-dataset')
path = str((ROOT / Path(self.data_dict['path'])).resolve())
dataset_name = self.data_dict.get("dataset_name", "yolov5-dataset")
path = str((ROOT / Path(self.data_dict["path"])).resolve())
metadata = self.data_dict.copy()
for key in ['train', 'val', 'test']:
for key in ["train", "val", "test"]:
split_path = metadata.get(key)
if split_path is not None:
metadata[key] = split_path.replace(path, '')
metadata[key] = split_path.replace(path, "")
artifact = comet_ml.Artifact(name=dataset_name, artifact_type='dataset', metadata=metadata)
artifact = comet_ml.Artifact(name=dataset_name, artifact_type="dataset", metadata=metadata)
for key in metadata.keys():
if key in ['train', 'val', 'test']:
if key in ["train", "val", "test"]:
if isinstance(self.upload_dataset, str) and (key != self.upload_dataset):
continue
@@ -362,26 +367,27 @@ class CometLogger:
metadata = logged_artifact.metadata
data_dict = metadata.copy()
data_dict['path'] = artifact_save_dir
data_dict["path"] = artifact_save_dir
metadata_names = metadata.get('names')
metadata_names = metadata.get("names")
if isinstance(metadata_names, dict):
data_dict['names'] = {int(k): v for k, v in metadata.get('names').items()}
data_dict["names"] = {int(k): v for k, v in metadata.get("names").items()}
elif isinstance(metadata_names, list):
data_dict['names'] = {int(k): v for k, v in zip(range(len(metadata_names)), metadata_names)}
data_dict["names"] = {int(k): v for k, v in zip(range(len(metadata_names)), metadata_names)}
else:
raise "Invalid 'names' field in dataset yaml file. Please use a list or dictionary"
return self.update_data_paths(data_dict)
def update_data_paths(self, data_dict):
path = data_dict.get('path', '')
path = data_dict.get("path", "")
for split in ['train', 'val', 'test']:
for split in ["train", "val", "test"]:
if data_dict.get(split):
split_path = data_dict.get(split)
data_dict[split] = (f'{path}/{split_path}' if isinstance(split, str) else [
f'{path}/{x}' for x in split_path])
data_dict[split] = (
f"{path}/{split_path}" if isinstance(split, str) else [f"{path}/{x}" for x in split_path]
)
return data_dict
@@ -422,11 +428,11 @@ class CometLogger:
def on_train_end(self, files, save_dir, last, best, epoch, results):
if self.comet_log_predictions:
curr_epoch = self.experiment.curr_epoch
self.experiment.log_asset_data(self.metadata_dict, 'image-metadata.json', epoch=curr_epoch)
self.experiment.log_asset_data(self.metadata_dict, "image-metadata.json", epoch=curr_epoch)
for f in files:
self.log_asset(f, metadata={'epoch': epoch})
self.log_asset(f'{save_dir}/results.csv', metadata={'epoch': epoch})
self.log_asset(f, metadata={"epoch": epoch})
self.log_asset(f"{save_dir}/results.csv", metadata={"epoch": epoch})
if not self.opt.evolve:
model_path = str(best if best.exists() else last)
@@ -440,9 +446,9 @@ class CometLogger:
)
# Check if running Experiment with Comet Optimizer
if hasattr(self.opt, 'comet_optimizer_id'):
if hasattr(self.opt, "comet_optimizer_id"):
metric = results.get(self.opt.comet_optimizer_metric)
self.experiment.log_other('optimizer_metric_value', metric)
self.experiment.log_other("optimizer_metric_value", metric)
self.finish_run()
@@ -477,21 +483,22 @@ class CometLogger:
class_name = self.class_names[c]
self.experiment.log_metrics(
{
'mAP@.5': ap50[i],
'mAP@.5:.95': ap[i],
'precision': p[i],
'recall': r[i],
'f1': f1[i],
'true_positives': tp[i],
'false_positives': fp[i],
'support': nt[c], },
"mAP@.5": ap50[i],
"mAP@.5:.95": ap[i],
"precision": p[i],
"recall": r[i],
"f1": f1[i],
"true_positives": tp[i],
"false_positives": fp[i],
"support": nt[c],
},
prefix=class_name,
)
if self.comet_log_confusion_matrix:
epoch = self.experiment.curr_epoch
class_names = list(self.class_names.values())
class_names.append('background')
class_names.append("background")
num_classes = len(class_names)
self.experiment.log_confusion_matrix(
@@ -499,9 +506,9 @@ class CometLogger:
max_categories=num_classes,
labels=class_names,
epoch=epoch,
column_label='Actual Category',
row_label='Predicted Category',
file_name=f'confusion-matrix-epoch-{epoch}.json',
column_label="Actual Category",
row_label="Predicted Category",
file_name=f"confusion-matrix-epoch-{epoch}.json",
)
def on_fit_epoch_end(self, result, epoch):
utils/loggers/comet/comet_utils.py
+27 -27
View File
@@ -11,28 +11,28 @@ import yaml
logger = logging.getLogger(__name__)
COMET_PREFIX = 'comet://'
COMET_MODEL_NAME = os.getenv('COMET_MODEL_NAME', 'yolov5')
COMET_DEFAULT_CHECKPOINT_FILENAME = os.getenv('COMET_DEFAULT_CHECKPOINT_FILENAME', 'last.pt')
COMET_PREFIX = "comet://"
COMET_MODEL_NAME = os.getenv("COMET_MODEL_NAME", "yolov5")
COMET_DEFAULT_CHECKPOINT_FILENAME = os.getenv("COMET_DEFAULT_CHECKPOINT_FILENAME", "last.pt")
def download_model_checkpoint(opt, experiment):
model_dir = f'{opt.project}/{experiment.name}'
model_dir = f"{opt.project}/{experiment.name}"
os.makedirs(model_dir, exist_ok=True)
model_name = COMET_MODEL_NAME
model_asset_list = experiment.get_model_asset_list(model_name)
if len(model_asset_list) == 0:
logger.error(f'COMET ERROR: No checkpoints found for model name : {model_name}')
logger.error(f"COMET ERROR: No checkpoints found for model name : {model_name}")
return
model_asset_list = sorted(
model_asset_list,
key=lambda x: x['step'],
key=lambda x: x["step"],
reverse=True,
)
logged_checkpoint_map = {asset['fileName']: asset['assetId'] for asset in model_asset_list}
logged_checkpoint_map = {asset["fileName"]: asset["assetId"] for asset in model_asset_list}
resource_url = urlparse(opt.weights)
checkpoint_filename = resource_url.query
@@ -44,28 +44,28 @@ def download_model_checkpoint(opt, experiment):
checkpoint_filename = COMET_DEFAULT_CHECKPOINT_FILENAME
if asset_id is None:
logger.error(f'COMET ERROR: Checkpoint {checkpoint_filename} not found in the given Experiment')
logger.error(f"COMET ERROR: Checkpoint {checkpoint_filename} not found in the given Experiment")
return
try:
logger.info(f'COMET INFO: Downloading checkpoint {checkpoint_filename}')
logger.info(f"COMET INFO: Downloading checkpoint {checkpoint_filename}")
asset_filename = checkpoint_filename
model_binary = experiment.get_asset(asset_id, return_type='binary', stream=False)
model_download_path = f'{model_dir}/{asset_filename}'
with open(model_download_path, 'wb') as f:
model_binary = experiment.get_asset(asset_id, return_type="binary", stream=False)
model_download_path = f"{model_dir}/{asset_filename}"
with open(model_download_path, "wb") as f:
f.write(model_binary)
opt.weights = model_download_path
except Exception as e:
logger.warning('COMET WARNING: Unable to download checkpoint from Comet')
logger.warning("COMET WARNING: Unable to download checkpoint from Comet")
logger.exception(e)
def set_opt_parameters(opt, experiment):
"""Update the opts Namespace with parameters
from Comet's ExistingExperiment when resuming a run
"""
Update the opts Namespace with parameters from Comet's ExistingExperiment when resuming a run.
Args:
opt (argparse.Namespace): Namespace of command line options
@@ -75,9 +75,9 @@ def set_opt_parameters(opt, experiment):
resume_string = opt.resume
for asset in asset_list:
if asset['fileName'] == 'opt.yaml':
asset_id = asset['assetId']
asset_binary = experiment.get_asset(asset_id, return_type='binary', stream=False)
if asset["fileName"] == "opt.yaml":
asset_id = asset["assetId"]
asset_binary = experiment.get_asset(asset_id, return_type="binary", stream=False)
opt_dict = yaml.safe_load(asset_binary)
for key, value in opt_dict.items():
setattr(opt, key, value)
@@ -85,18 +85,18 @@ def set_opt_parameters(opt, experiment):
# Save hyperparameters to YAML file
# Necessary to pass checks in training script
save_dir = f'{opt.project}/{experiment.name}'
save_dir = f"{opt.project}/{experiment.name}"
os.makedirs(save_dir, exist_ok=True)
hyp_yaml_path = f'{save_dir}/hyp.yaml'
with open(hyp_yaml_path, 'w') as f:
hyp_yaml_path = f"{save_dir}/hyp.yaml"
with open(hyp_yaml_path, "w") as f:
yaml.dump(opt.hyp, f)
opt.hyp = hyp_yaml_path
def check_comet_weights(opt):
"""Downloads model weights from Comet and updates the
weights path to point to saved weights location
"""
Downloads model weights from Comet and updates the weights path to point to saved weights location.
Args:
opt (argparse.Namespace): Command Line arguments passed
@@ -113,7 +113,7 @@ def check_comet_weights(opt):
if opt.weights.startswith(COMET_PREFIX):
api = comet_ml.API()
resource = urlparse(opt.weights)
experiment_path = f'{resource.netloc}{resource.path}'
experiment_path = f"{resource.netloc}{resource.path}"
experiment = api.get(experiment_path)
download_model_checkpoint(opt, experiment)
return True
@@ -122,8 +122,8 @@ def check_comet_weights(opt):
def check_comet_resume(opt):
"""Restores run parameters to its original state based on the model checkpoint
and logged Experiment parameters.
"""
Restores run parameters to its original state based on the model checkpoint and logged Experiment parameters.
Args:
opt (argparse.Namespace): Command Line arguments passed
@@ -140,7 +140,7 @@ def check_comet_resume(opt):
if opt.resume.startswith(COMET_PREFIX):
api = comet_ml.API()
resource = urlparse(opt.resume)
experiment_path = f'{resource.netloc}{resource.path}'
experiment_path = f"{resource.netloc}{resource.path}"
experiment = api.get(experiment_path)
set_opt_parameters(opt, experiment)
download_model_checkpoint(opt, experiment)
utils/loggers/comet/hpo.py
+58 -56
View File
@@ -21,77 +21,79 @@ from utils.torch_utils import select_device
# Project Configuration
config = comet_ml.config.get_config()
COMET_PROJECT_NAME = config.get_string(os.getenv('COMET_PROJECT_NAME'), 'comet.project_name', default='yolov5')
COMET_PROJECT_NAME = config.get_string(os.getenv("COMET_PROJECT_NAME"), "comet.project_name", default="yolov5")
def get_args(known=False):
parser = argparse.ArgumentParser()
parser.add_argument('--weights', type=str, default=ROOT / 'yolov5s.pt', help='initial weights path')
parser.add_argument('--cfg', type=str, default='', help='model.yaml path')
parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='dataset.yaml path')
parser.add_argument('--hyp', type=str, default=ROOT / 'data/hyps/hyp.scratch-low.yaml', help='hyperparameters path')
parser.add_argument('--epochs', type=int, default=300, help='total training epochs')
parser.add_argument('--batch-size', type=int, default=16, help='total batch size for all GPUs, -1 for autobatch')
parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=640, help='train, val image size (pixels)')
parser.add_argument('--rect', action='store_true', help='rectangular training')
parser.add_argument('--resume', nargs='?', const=True, default=False, help='resume most recent training')
parser.add_argument('--nosave', action='store_true', help='only save final checkpoint')
parser.add_argument('--noval', action='store_true', help='only validate final epoch')
parser.add_argument('--noautoanchor', action='store_true', help='disable AutoAnchor')
parser.add_argument('--noplots', action='store_true', help='save no plot files')
parser.add_argument('--evolve', type=int, nargs='?', const=300, help='evolve hyperparameters for x generations')
parser.add_argument('--bucket', type=str, default='', help='gsutil bucket')
parser.add_argument('--cache', type=str, nargs='?', const='ram', help='--cache images in "ram" (default) or "disk"')
parser.add_argument('--image-weights', action='store_true', help='use weighted image selection for training')
parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
parser.add_argument('--multi-scale', action='store_true', help='vary img-size +/- 50%%')
parser.add_argument('--single-cls', action='store_true', help='train multi-class data as single-class')
parser.add_argument('--optimizer', type=str, choices=['SGD', 'Adam', 'AdamW'], default='SGD', help='optimizer')
parser.add_argument('--sync-bn', action='store_true', help='use SyncBatchNorm, only available in DDP mode')
parser.add_argument('--workers', type=int, default=8, help='max dataloader workers (per RANK in DDP mode)')
parser.add_argument('--project', default=ROOT / 'runs/train', help='save to project/name')
parser.add_argument('--name', default='exp', help='save to project/name')
parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment')
parser.add_argument('--quad', action='store_true', help='quad dataloader')
parser.add_argument('--cos-lr', action='store_true', help='cosine LR scheduler')
parser.add_argument('--label-smoothing', type=float, default=0.0, help='Label smoothing epsilon')
parser.add_argument('--patience', type=int, default=100, help='EarlyStopping patience (epochs without improvement)')
parser.add_argument('--freeze', nargs='+', type=int, default=[0], help='Freeze layers: backbone=10, first3=0 1 2')
parser.add_argument('--save-period', type=int, default=-1, help='Save checkpoint every x epochs (disabled if < 1)')
parser.add_argument('--seed', type=int, default=0, help='Global training seed')
parser.add_argument('--local_rank', type=int, default=-1, help='Automatic DDP Multi-GPU argument, do not modify')
parser.add_argument("--weights", type=str, default=ROOT / "yolov5s.pt", help="initial weights path")
parser.add_argument("--cfg", type=str, default="", help="model.yaml path")
parser.add_argument("--data", type=str, default=ROOT / "data/coco128.yaml", help="dataset.yaml path")
parser.add_argument("--hyp", type=str, default=ROOT / "data/hyps/hyp.scratch-low.yaml", help="hyperparameters path")
parser.add_argument("--epochs", type=int, default=300, help="total training epochs")
parser.add_argument("--batch-size", type=int, default=16, help="total batch size for all GPUs, -1 for autobatch")
parser.add_argument("--imgsz", "--img", "--img-size", type=int, default=640, help="train, val image size (pixels)")
parser.add_argument("--rect", action="store_true", help="rectangular training")
parser.add_argument("--resume", nargs="?", const=True, default=False, help="resume most recent training")
parser.add_argument("--nosave", action="store_true", help="only save final checkpoint")
parser.add_argument("--noval", action="store_true", help="only validate final epoch")
parser.add_argument("--noautoanchor", action="store_true", help="disable AutoAnchor")
parser.add_argument("--noplots", action="store_true", help="save no plot files")
parser.add_argument("--evolve", type=int, nargs="?", const=300, help="evolve hyperparameters for x generations")
parser.add_argument("--bucket", type=str, default="", help="gsutil bucket")
parser.add_argument("--cache", type=str, nargs="?", const="ram", help='--cache images in "ram" (default) or "disk"')
parser.add_argument("--image-weights", action="store_true", help="use weighted image selection for training")
parser.add_argument("--device", default="", help="cuda device, i.e. 0 or 0,1,2,3 or cpu")
parser.add_argument("--multi-scale", action="store_true", help="vary img-size +/- 50%%")
parser.add_argument("--single-cls", action="store_true", help="train multi-class data as single-class")
parser.add_argument("--optimizer", type=str, choices=["SGD", "Adam", "AdamW"], default="SGD", help="optimizer")
parser.add_argument("--sync-bn", action="store_true", help="use SyncBatchNorm, only available in DDP mode")
parser.add_argument("--workers", type=int, default=8, help="max dataloader workers (per RANK in DDP mode)")
parser.add_argument("--project", default=ROOT / "runs/train", help="save to project/name")
parser.add_argument("--name", default="exp", help="save to project/name")
parser.add_argument("--exist-ok", action="store_true", help="existing project/name ok, do not increment")
parser.add_argument("--quad", action="store_true", help="quad dataloader")
parser.add_argument("--cos-lr", action="store_true", help="cosine LR scheduler")
parser.add_argument("--label-smoothing", type=float, default=0.0, help="Label smoothing epsilon")
parser.add_argument("--patience", type=int, default=100, help="EarlyStopping patience (epochs without improvement)")
parser.add_argument("--freeze", nargs="+", type=int, default=[0], help="Freeze layers: backbone=10, first3=0 1 2")
parser.add_argument("--save-period", type=int, default=-1, help="Save checkpoint every x epochs (disabled if < 1)")
parser.add_argument("--seed", type=int, default=0, help="Global training seed")
parser.add_argument("--local_rank", type=int, default=-1, help="Automatic DDP Multi-GPU argument, do not modify")
# Weights & Biases arguments
parser.add_argument('--entity', default=None, help='W&B: Entity')
parser.add_argument('--upload_dataset', nargs='?', const=True, default=False, help='W&B: Upload data, "val" option')
parser.add_argument('--bbox_interval', type=int, default=-1, help='W&B: Set bounding-box image logging interval')
parser.add_argument('--artifact_alias', type=str, default='latest', help='W&B: Version of dataset artifact to use')
parser.add_argument("--entity", default=None, help="W&B: Entity")
parser.add_argument("--upload_dataset", nargs="?", const=True, default=False, help='W&B: Upload data, "val" option')
parser.add_argument("--bbox_interval", type=int, default=-1, help="W&B: Set bounding-box image logging interval")
parser.add_argument("--artifact_alias", type=str, default="latest", help="W&B: Version of dataset artifact to use")
# Comet Arguments
parser.add_argument('--comet_optimizer_config', type=str, help='Comet: Path to a Comet Optimizer Config File.')
parser.add_argument('--comet_optimizer_id', type=str, help='Comet: ID of the Comet Optimizer sweep.')
parser.add_argument('--comet_optimizer_objective', type=str, help="Comet: Set to 'minimize' or 'maximize'.")
parser.add_argument('--comet_optimizer_metric', type=str, help='Comet: Metric to Optimize.')
parser.add_argument('--comet_optimizer_workers',
type=int,
default=1,
help='Comet: Number of Parallel Workers to use with the Comet Optimizer.')
parser.add_argument("--comet_optimizer_config", type=str, help="Comet: Path to a Comet Optimizer Config File.")
parser.add_argument("--comet_optimizer_id", type=str, help="Comet: ID of the Comet Optimizer sweep.")
parser.add_argument("--comet_optimizer_objective", type=str, help="Comet: Set to 'minimize' or 'maximize'.")
parser.add_argument("--comet_optimizer_metric", type=str, help="Comet: Metric to Optimize.")
parser.add_argument(
"--comet_optimizer_workers",
type=int,
default=1,
help="Comet: Number of Parallel Workers to use with the Comet Optimizer.",
)
return parser.parse_known_args()[0] if known else parser.parse_args()
def run(parameters, opt):
hyp_dict = {k: v for k, v in parameters.items() if k not in ['epochs', 'batch_size']}
hyp_dict = {k: v for k, v in parameters.items() if k not in ["epochs", "batch_size"]}
opt.save_dir = str(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok or opt.evolve))
opt.batch_size = parameters.get('batch_size')
opt.epochs = parameters.get('epochs')
opt.batch_size = parameters.get("batch_size")
opt.epochs = parameters.get("epochs")
device = select_device(opt.device, batch_size=opt.batch_size)
train(hyp_dict, opt, device, callbacks=Callbacks())
if __name__ == '__main__':
if __name__ == "__main__":
opt = get_args(known=True)
opt.weights = str(opt.weights)
@@ -99,7 +101,7 @@ if __name__ == '__main__':
opt.data = str(opt.data)
opt.project = str(opt.project)
optimizer_id = os.getenv('COMET_OPTIMIZER_ID')
optimizer_id = os.getenv("COMET_OPTIMIZER_ID")
if optimizer_id is None:
with open(opt.comet_optimizer_config) as f:
optimizer_config = json.load(f)
@@ -110,9 +112,9 @@ if __name__ == '__main__':
opt.comet_optimizer_id = optimizer.id
status = optimizer.status()
opt.comet_optimizer_objective = status['spec']['objective']
opt.comet_optimizer_metric = status['spec']['metric']
opt.comet_optimizer_objective = status["spec"]["objective"]
opt.comet_optimizer_metric = status["spec"]["metric"]
logger.info('COMET INFO: Starting Hyperparameter Sweep')
logger.info("COMET INFO: Starting Hyperparameter Sweep")
for parameter in optimizer.get_parameters():
run(parameter['parameters'], opt)
run(parameter["parameters"], opt)
utils/loggers/wandb/wandb_utils.py
+62 -47
View File
@@ -15,34 +15,35 @@ FILE = Path(__file__).resolve()
ROOT = FILE.parents[3] # YOLOv5 root directory
if str(ROOT) not in sys.path:
sys.path.append(str(ROOT)) # add ROOT to PATH
RANK = int(os.getenv('RANK', -1))
DEPRECATION_WARNING = f"{colorstr('wandb')}: WARNING ⚠️ wandb is deprecated and will be removed in a future release. " \
f'See supported integrations at https://github.com/ultralytics/yolov5#integrations.'
RANK = int(os.getenv("RANK", -1))
DEPRECATION_WARNING = (
f"{colorstr('wandb')}: WARNING ⚠️ wandb is deprecated and will be removed in a future release. "
f'See supported integrations at https://github.com/ultralytics/yolov5#integrations.'
)
try:
import wandb
assert hasattr(wandb, '__version__') # verify package import not local dir
assert hasattr(wandb, "__version__") # verify package import not local dir
LOGGER.warning(DEPRECATION_WARNING)
except (ImportError, AssertionError):
wandb = None
class WandbLogger():
"""Log training runs, datasets, models, and predictions to Weights & Biases.
class WandbLogger:
"""
Log training runs, datasets, models, and predictions to Weights & Biases.
This logger sends information to W&B at wandb.ai. By default, this information
includes hyperparameters, system configuration and metrics, model metrics,
and basic data metrics and analyses.
This logger sends information to W&B at wandb.ai. By default, this information includes hyperparameters, system
configuration and metrics, model metrics, and basic data metrics and analyses.
By providing additional command line arguments to train.py, datasets,
models and predictions can also be logged.
By providing additional command line arguments to train.py, datasets, models and predictions can also be logged.
For more on how this logger is used, see the Weights & Biases documentation:
https://docs.wandb.com/guides/integrations/yolov5
"""
def __init__(self, opt, run_id=None, job_type='Training'):
def __init__(self, opt, run_id=None, job_type="Training"):
"""
- Initialize WandbLogger instance
- Upload dataset if opt.upload_dataset is True
@@ -53,7 +54,7 @@ class WandbLogger():
run_id (str) -- Run ID of W&B run to be resumed
job_type (str) -- To set the job_type for this run
"""
"""
# Pre-training routine --
self.job_type = job_type
self.wandb, self.wandb_run = wandb, wandb.run if wandb else None
@@ -64,17 +65,23 @@ class WandbLogger():
self.max_imgs_to_log = 16
self.data_dict = None
if self.wandb:
self.wandb_run = wandb.init(config=opt,
resume='allow',
project='YOLOv5' if opt.project == 'runs/train' else Path(opt.project).stem,
entity=opt.entity,
name=opt.name if opt.name != 'exp' else None,
job_type=job_type,
id=run_id,
allow_val_change=True) if not wandb.run else wandb.run
self.wandb_run = (
wandb.init(
config=opt,
resume="allow",
project="YOLOv5" if opt.project == "runs/train" else Path(opt.project).stem,
entity=opt.entity,
name=opt.name if opt.name != "exp" else None,
job_type=job_type,
id=run_id,
allow_val_change=True,
)
if not wandb.run
else wandb.run
)
if self.wandb_run:
if self.job_type == 'Training':
if self.job_type == "Training":
if isinstance(opt.data, dict):
# This means another dataset manager has already processed the dataset info (e.g. ClearML)
# and they will have stored the already processed dict in opt.data
@@ -97,11 +104,17 @@ class WandbLogger():
if isinstance(opt.resume, str):
model_dir, _ = self.download_model_artifact(opt)
if model_dir:
self.weights = Path(model_dir) / 'last.pt'
self.weights = Path(model_dir) / "last.pt"
config = self.wandb_run.config
opt.weights, opt.save_period, opt.batch_size, opt.bbox_interval, opt.epochs, opt.hyp, opt.imgsz = str(
self.weights), config.save_period, config.batch_size, config.bbox_interval, config.epochs, \
config.hyp, config.imgsz
opt.weights, opt.save_period, opt.batch_size, opt.bbox_interval, opt.epochs, opt.hyp, opt.imgsz = (
str(self.weights),
config.save_period,
config.batch_size,
config.bbox_interval,
config.epochs,
config.hyp,
config.imgsz,
)
if opt.bbox_interval == -1:
self.bbox_interval = opt.bbox_interval = (opt.epochs // 10) if opt.epochs > 10 else 1
@@ -110,7 +123,7 @@ class WandbLogger():
def log_model(self, path, opt, epoch, fitness_score, best_model=False):
"""
Log the model checkpoint as W&B artifact
Log the model checkpoint as W&B artifact.
arguments:
path (Path) -- Path of directory containing the checkpoints
@@ -119,26 +132,30 @@ class WandbLogger():
fitness_score (float) -- fitness score for current epoch
best_model (boolean) -- Boolean representing if the current checkpoint is the best yet.
"""
model_artifact = wandb.Artifact('run_' + wandb.run.id + '_model',
type='model',
metadata={
'original_url': str(path),
'epochs_trained': epoch + 1,
'save period': opt.save_period,
'project': opt.project,
'total_epochs': opt.epochs,
'fitness_score': fitness_score})
model_artifact.add_file(str(path / 'last.pt'), name='last.pt')
wandb.log_artifact(model_artifact,
aliases=['latest', 'last', 'epoch ' + str(self.current_epoch), 'best' if best_model else ''])
LOGGER.info(f'Saving model artifact on epoch {epoch + 1}')
model_artifact = wandb.Artifact(
"run_" + wandb.run.id + "_model",
type="model",
metadata={
"original_url": str(path),
"epochs_trained": epoch + 1,
"save period": opt.save_period,
"project": opt.project,
"total_epochs": opt.epochs,
"fitness_score": fitness_score,
},
)
model_artifact.add_file(str(path / "last.pt"), name="last.pt")
wandb.log_artifact(
model_artifact, aliases=["latest", "last", "epoch " + str(self.current_epoch), "best" if best_model else ""]
)
LOGGER.info(f"Saving model artifact on epoch {epoch + 1}")
def val_one_image(self, pred, predn, path, names, im):
pass
def log(self, log_dict):
"""
save the metrics to the logging dictionary
Save the metrics to the logging dictionary.
arguments:
log_dict (Dict) -- metrics/media to be logged in current step
@@ -149,7 +166,7 @@ class WandbLogger():
def end_epoch(self):
"""
commit the log_dict, model artifacts and Tables to W&B and flush the log_dict.
Commit the log_dict, model artifacts and Tables to W&B and flush the log_dict.
arguments:
best_result (boolean): Boolean representing if the result of this evaluation is best or not
@@ -160,16 +177,14 @@ class WandbLogger():
wandb.log(self.log_dict)
except BaseException as e:
LOGGER.info(
f'An error occurred in wandb logger. The training will proceed without interruption. More info\n{e}'
f"An error occurred in wandb logger. The training will proceed without interruption. More info\n{e}"
)
self.wandb_run.finish()
self.wandb_run = None
self.log_dict = {}
def finish_run(self):
"""
Log metrics if any and finish the current W&B run
"""
"""Log metrics if any and finish the current W&B run."""
if self.wandb_run:
if self.log_dict:
with all_logging_disabled():
@@ -180,7 +195,7 @@ class WandbLogger():
@contextmanager
def all_logging_disabled(highest_level=logging.CRITICAL):
""" source - https://gist.github.com/simon-weber/7853144
"""source - https://gist.github.com/simon-weber/7853144
A context manager that will prevent any logging messages triggered during the body from being processed.
:param highest_level: the maximum logging level in use.
This would only need to be changed if a custom level greater than CRITICAL is defined.