Ankle/yolov5_birdproject
1
0 Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Update Actions with Lychee and GitHub Token (#12592)

Browse Source 
* Add Ultralytics Actions

* Auto-format by Ultralytics actions

* Update format.yml

Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com>

---------

Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com>
Co-authored-by: UltralyticsAssistant <web@ultralytics.com>
Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>
This commit is contained in:
Paula DerrengerUltralyticsAssistantGlenn Jocher
 authored and GitHub committed 2024-01-08 01:29:14 +01:00
1 parent f56355d321
commit ad97dc6a6d
47 files changed
+4217 -3462
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

utils/loggers/clearml/clearml_utils.py
+64 -59
View File
@@ -13,55 +13,63 @@ try:
import clearml
from clearml import Dataset, Task
assert hasattr(clearml, '__version__') # verify package import not local dir
assert hasattr(clearml, "__version__") # verify package import not local dir
except (ImportError, AssertionError):
clearml = None
def construct_dataset(clearml_info_string):
"""Load in a clearml dataset and fill the internal data_dict with its contents.
"""
dataset_id = clearml_info_string.replace('clearml://', '')
"""Load in a clearml dataset and fill the internal data_dict with its contents."""
dataset_id = clearml_info_string.replace("clearml://", "")
dataset = Dataset.get(dataset_id=dataset_id)
dataset_root_path = Path(dataset.get_local_copy())
# We'll search for the yaml file definition in the dataset
yaml_filenames = list(glob.glob(str(dataset_root_path / '*.yaml')) + glob.glob(str(dataset_root_path / '*.yml')))
yaml_filenames = list(glob.glob(str(dataset_root_path / "*.yaml")) + glob.glob(str(dataset_root_path / "*.yml")))
if len(yaml_filenames) > 1:
raise ValueError('More than one yaml file was found in the dataset root, cannot determine which one contains '
'the dataset definition this way.')
raise ValueError(
"More than one yaml file was found in the dataset root, cannot determine which one contains "
"the dataset definition this way."
)
elif len(yaml_filenames) == 0:
raise ValueError('No yaml definition found in dataset root path, check that there is a correct yaml file '
'inside the dataset root path.')
raise ValueError(
"No yaml definition found in dataset root path, check that there is a correct yaml file "
"inside the dataset root path."
)
with open(yaml_filenames[0]) as f:
dataset_definition = yaml.safe_load(f)
assert set(dataset_definition.keys()).issuperset(
{'train', 'test', 'val', 'nc', 'names'}
assert set(
dataset_definition.keys()
).issuperset(
{"train", "test", "val", "nc", "names"}
), "The right keys were not found in the yaml file, make sure it at least has the following keys: ('train', 'test', 'val', 'nc', 'names')"
data_dict = dict()
data_dict['train'] = str(
(dataset_root_path / dataset_definition['train']).resolve()) if dataset_definition['train'] else None
data_dict['test'] = str(
(dataset_root_path / dataset_definition['test']).resolve()) if dataset_definition['test'] else None
data_dict['val'] = str(
(dataset_root_path / dataset_definition['val']).resolve()) if dataset_definition['val'] else None
data_dict['nc'] = dataset_definition['nc']
data_dict['names'] = dataset_definition['names']
data_dict["train"] = (
str((dataset_root_path / dataset_definition["train"]).resolve()) if dataset_definition["train"] else None
)
data_dict["test"] = (
str((dataset_root_path / dataset_definition["test"]).resolve()) if dataset_definition["test"] else None
)
data_dict["val"] = (
str((dataset_root_path / dataset_definition["val"]).resolve()) if dataset_definition["val"] else None
)
data_dict["nc"] = dataset_definition["nc"]
data_dict["names"] = dataset_definition["names"]
return data_dict
class ClearmlLogger:
"""Log training runs, datasets, models, and predictions to ClearML.
"""
Log training runs, datasets, models, and predictions to ClearML.
This logger sends information to ClearML at app.clear.ml or to your own hosted server. By default,
this information includes hyperparameters, system configuration and metrics, model metrics, code information and
basic data metrics and analyses.
This logger sends information to ClearML at app.clear.ml or to your own hosted server. By default, this information
includes hyperparameters, system configuration and metrics, model metrics, code information and basic data metrics
and analyses.
By providing additional command line arguments to train.py, datasets,
models and predictions can also be logged.
By providing additional command line arguments to train.py, datasets, models and predictions can also be logged.
"""
def __init__(self, opt, hyp):
@@ -81,36 +89,36 @@ class ClearmlLogger:
self.max_imgs_to_log_per_epoch = 16
# Get the interval of epochs when bounding box images should be logged
# Only for detection task though!
if 'bbox_interval' in opt:
if "bbox_interval" in opt:
self.bbox_interval = opt.bbox_interval
self.clearml = clearml
self.task = None
self.data_dict = None
if self.clearml:
self.task = Task.init(
project_name=opt.project if not str(opt.project).startswith('runs/') else 'YOLOv5',
task_name=opt.name if opt.name != 'exp' else 'Training',
tags=['YOLOv5'],
project_name=opt.project if not str(opt.project).startswith("runs/") else "YOLOv5",
task_name=opt.name if opt.name != "exp" else "Training",
tags=["YOLOv5"],
output_uri=True,
reuse_last_task_id=opt.exist_ok,
auto_connect_frameworks={
'pytorch': False,
'matplotlib': False}
auto_connect_frameworks={"pytorch": False, "matplotlib": False},
# We disconnect pytorch auto-detection, because we added manual model save points in the code
)
# ClearML's hooks will already grab all general parameters
# Only the hyperparameters coming from the yaml config file
# will have to be added manually!
self.task.connect(hyp, name='Hyperparameters')
self.task.connect(opt, name='Args')
self.task.connect(hyp, name="Hyperparameters")
self.task.connect(opt, name="Args")
# Make sure the code is easily remotely runnable by setting the docker image to use by the remote agent
self.task.set_base_docker('ultralytics/yolov5:latest',
docker_arguments='--ipc=host -e="CLEARML_AGENT_SKIP_PYTHON_ENV_INSTALL=1"',
docker_setup_bash_script='pip install clearml')
self.task.set_base_docker(
"ultralytics/yolov5:latest",
docker_arguments='--ipc=host -e="CLEARML_AGENT_SKIP_PYTHON_ENV_INSTALL=1"',
docker_setup_bash_script="pip install clearml",
)
# Get ClearML Dataset Version if requested
if opt.data.startswith('clearml://'):
if opt.data.startswith("clearml://"):
# data_dict should have the following keys:
# names, nc (number of classes), test, train, val (all three relative paths to ../datasets)
self.data_dict = construct_dataset(opt.data)
@@ -120,33 +128,32 @@ class ClearmlLogger:
def log_scalars(self, metrics, epoch):
"""
Log scalars/metrics to ClearML
Log scalars/metrics to ClearML.
arguments:
metrics (dict) Metrics in dict format: {"metrics/mAP": 0.8, ...}
epoch (int) iteration number for the current set of metrics
"""
for k, v in metrics.items():
title, series = k.split('/')
title, series = k.split("/")
self.task.get_logger().report_scalar(title, series, v, epoch)
def log_model(self, model_path, model_name, epoch=0):
"""
Log model weights to ClearML
Log model weights to ClearML.
arguments:
model_path (PosixPath or str) Path to the model weights
model_name (str) Name of the model visible in ClearML
epoch (int) Iteration / epoch of the model weights
"""
self.task.update_output_model(model_path=str(model_path),
name=model_name,
iteration=epoch,
auto_delete_file=False)
self.task.update_output_model(
model_path=str(model_path), name=model_name, iteration=epoch, auto_delete_file=False
)
def log_summary(self, metrics):
"""
Log final metrics to a summary table
Log final metrics to a summary table.
arguments:
metrics (dict) Metrics in dict format: {"metrics/mAP": 0.8, ...}
@@ -156,7 +163,7 @@ class ClearmlLogger:
def log_plot(self, title, plot_path):
"""
Log image as plot in the plot section of ClearML
Log image as plot in the plot section of ClearML.
arguments:
title (str) Title of the plot
@@ -164,12 +171,12 @@ class ClearmlLogger:
"""
img = mpimg.imread(plot_path)
fig = plt.figure()
ax = fig.add_axes([0, 0, 1, 1], frameon=False, aspect='auto', xticks=[], yticks=[]) # no ticks
ax = fig.add_axes([0, 0, 1, 1], frameon=False, aspect="auto", xticks=[], yticks=[]) # no ticks
ax.imshow(img)
self.task.get_logger().report_matplotlib_figure(title, "", figure=fig, report_interactive=False)
def log_debug_samples(self, files, title='Debug Samples'):
def log_debug_samples(self, files, title="Debug Samples"):
"""
Log files (images) as debug samples in the ClearML task.
@@ -179,12 +186,11 @@ class ClearmlLogger:
"""
for f in files:
if f.exists():
it = re.search(r'_batch(\d+)', f.name)
it = re.search(r"_batch(\d+)", f.name)
iteration = int(it.groups()[0]) if it else 0
self.task.get_logger().report_image(title=title,
series=f.name.replace(f"_batch{iteration}", ''),
local_path=str(f),
iteration=iteration)
self.task.get_logger().report_image(
title=title, series=f.name.replace(f"_batch{iteration}", ""), local_path=str(f), iteration=iteration
)
def log_image_with_boxes(self, image_path, boxes, class_names, image, conf_threshold=0.25):
"""
@@ -206,15 +212,14 @@ class ClearmlLogger:
class_name = class_names[int(class_nr)]
confidence_percentage = round(float(conf) * 100, 2)
label = f'{class_name}: {confidence_percentage}%'
label = f"{class_name}: {confidence_percentage}%"
if conf > conf_threshold:
annotator.rectangle(box.cpu().numpy(), outline=color)
annotator.box_label(box.cpu().numpy(), label=label, color=color)
annotated_image = annotator.result()
self.task.get_logger().report_image(title='Bounding Boxes',
series=image_path.name,
iteration=self.current_epoch,
image=annotated_image)
self.task.get_logger().report_image(
title="Bounding Boxes", series=image_path.name, iteration=self.current_epoch, image=annotated_image
)
self.current_epoch_logged_images.add(image_path)