# Ultralytics YOLO 🚀, AGPL-3.0 license import numpy as np import math from ..utils import LOGGER from ..utils.ops import xywh2ltwh, xywh2xyxy from .basetrack import BaseTrack, TrackState from .utils import matching from .utils.kalman_filter import KalmanFilterXYAH class STrack(BaseTrack): """ Single object tracking representation that uses Kalman filtering for state estimation. This class is responsible for storing all the information regarding individual tracklets and performs state updates and predictions based on Kalman filter. Attributes: shared_kalman (KalmanFilterXYAH): Shared Kalman filter that is used across all STrack instances for prediction. _tlwh (np.ndarray): Private attribute to store top-left corner coordinates and width and height of bounding box. kalman_filter (KalmanFilterXYAH): Instance of Kalman filter used for this particular object track. mean (np.ndarray): Mean state estimate vector. covariance (np.ndarray): Covariance of state estimate. is_activated (bool): Boolean flag indicating if the track has been activated. score (float): Confidence score of the track. tracklet_len (int): Length of the tracklet. cls (any): Class label for the object. idx (int): Index or identifier for the object. frame_id (int): Current frame ID. start_frame (int): Frame where the object was first detected. Methods: predict(): Predict the next state of the object using Kalman filter. multi_predict(stracks): Predict the next states for multiple tracks. multi_gmc(stracks, H): Update multiple track states using a homography matrix. activate(kalman_filter, frame_id): Activate a new tracklet. re_activate(new_track, frame_id, new_id): Reactivate a previously lost tracklet. update(new_track, frame_id): Update the state of a matched track. convert_coords(tlwh): Convert bounding box to x-y-aspect-height format. tlwh_to_xyah(tlwh): Convert tlwh bounding box to xyah format. """ shared_kalman = KalmanFilterXYAH() def __init__(self, xywh, score, cls): """Initialize new STrack instance.""" super().__init__() # xywh+idx or xywha+idx assert len(xywh) in {5, 6}, f"expected 5 or 6 values but got {len(xywh)}" self._tlwh = np.asarray(xywh2ltwh(xywh[:4]), dtype=np.float32) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False self.score = score self.tracklet_len = 0 self.cls = cls self.idx = xywh[-1] self.angle = xywh[4] if len(xywh) == 6 else None self.det = np.asarray(xywh2xyxy(xywh[:4]), dtype=np.float32) self.recent_scores = [] # 用于存储最近10次的score self.recent_scores.append(self.score) self.tracklet_len_score = self.score def predict(self): """Predicts mean and covariance using Kalman filter.""" mean_state = self.mean.copy() if self.state != TrackState.Tracked: mean_state[7] = 0 self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance) def prior_predict(self): """Predicts mean and covariance using Kalman filter.""" mean_state = self.mean.copy() if self.state != TrackState.Tracked: return None mean, covariance = self.kalman_filter.predict(mean_state, self.covariance) return mean @staticmethod def multi_predict(stracks): """Perform multi-object predictive tracking using Kalman filter for given stracks.""" if len(stracks) <= 0: return multi_mean = np.asarray([st.mean.copy() for st in stracks]) multi_covariance = np.asarray([st.covariance for st in stracks]) for i, st in enumerate(stracks): if st.state != TrackState.Tracked: multi_mean[i][7] = 0 multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(multi_mean, multi_covariance) for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): stracks[i].mean = mean stracks[i].covariance = cov @staticmethod def multi_gmc(stracks, H=np.eye(2, 3)): """Update state tracks positions and covariances using a homography matrix.""" if len(stracks) > 0: multi_mean = np.asarray([st.mean.copy() for st in stracks]) multi_covariance = np.asarray([st.covariance for st in stracks]) R = H[:2, :2] R8x8 = np.kron(np.eye(4, dtype=float), R) t = H[:2, 2] for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): mean = R8x8.dot(mean) mean[:2] += t cov = R8x8.dot(cov).dot(R8x8.transpose()) stracks[i].mean = mean stracks[i].covariance = cov def activate(self, kalman_filter, frame_id): """Start a new tracklet.""" self.kalman_filter = kalman_filter self.track_id = self.next_id() self.mean, self.covariance = self.kalman_filter.initiate(self.convert_coords(self._tlwh)) self.tracklet_len = 0 self.state = TrackState.Tracked if frame_id == 1: self.is_activated = True self.frame_id = frame_id self.start_frame = frame_id def re_activate(self, new_track, frame_id, new_id=False): """Reactivates a previously lost track with a new detection.""" self.mean, self.covariance = self.kalman_filter.update( self.mean, self.covariance, self.convert_coords(new_track.tlwh) ) self.tracklet_len = 0 self.state = TrackState.Tracked self.is_activated = True self.frame_id = frame_id if new_id: self.track_id = self.next_id() self.score = new_track.score self.cls = new_track.cls self.angle = new_track.angle self.idx = new_track.idx self.det = new_track.xyxy self.recent_scores = [] # re clear self.recent_scores.append(self.score) self.tracklet_len_score = self.score def update(self, new_track, frame_id): """ Update the state of a matched track. Args: new_track (STrack): The new track containing updated information. frame_id (int): The ID of the current frame. """ self.frame_id = frame_id self.tracklet_len += 1 new_tlwh = new_track.tlwh self.mean, self.covariance = self.kalman_filter.update( self.mean, self.covariance, self.convert_coords(new_tlwh) ) self.state = TrackState.Tracked self.is_activated = True self.score = new_track.score self.cls = new_track.cls self.angle = new_track.angle self.idx = new_track.idx self.det = new_track.xyxy self.recent_scores.append(self.score) if len(self.recent_scores) > 10: self.recent_scores.pop(0) self.tracklet_len_score = sum(self.recent_scores) def update_simi(self, new_track, frame_id): """ Update the state of a matched track. Args: new_track (STrack): The new track containing updated information. frame_id (int): The ID of the current frame. """ self.frame_id = frame_id self.tracklet_len += 1 # 认为存在突变 new_tlwh = new_track.tlwh self.mean, self.covariance = self.kalman_filter.initiate(self.convert_coords(new_tlwh)) self.state = TrackState.Tracked self.is_activated = True self.score = new_track.score self.cls = new_track.cls self.angle = new_track.angle self.idx = new_track.idx self.det = new_track.xyxy self.recent_scores.append(self.score) if len(self.recent_scores) > 10: self.recent_scores.pop(0) self.tracklet_len_score = sum(self.recent_scores) def convert_coords(self, tlwh): """Convert a bounding box's top-left-width-height format to its x-y-aspect-height equivalent.""" return self.tlwh_to_xyah(tlwh) @property def tlwh(self): """Get current position in bounding box format (top left x, top left y, width, height).""" if self.mean is None: return self._tlwh.copy() ret = self.mean[:4].copy() ret[2] *= ret[3] ret[:2] -= ret[2:] / 2 return ret @property def xyxy(self): """Convert bounding box to format (min x, min y, max x, max y), i.e., (top left, bottom right).""" ret = self.tlwh.copy() ret[2:] += ret[:2] return ret @staticmethod def tlwh_to_xyah(tlwh): """Convert bounding box to format (center x, center y, aspect ratio, height), where the aspect ratio is width / height. """ ret = np.asarray(tlwh).copy() ret[:2] += ret[2:] / 2 ret[2] /= ret[3] return ret @property def xywh(self): """Get current position in bounding box format (center x, center y, width, height).""" ret = np.asarray(self.tlwh).copy() ret[:2] += ret[2:] / 2 return ret @property def xywha(self): """Get current position in bounding box format (center x, center y, width, height, angle).""" if self.angle is None: LOGGER.warning("WARNING ⚠️ `angle` attr not found, returning `xywh` instead.") return self.xywh return np.concatenate([self.xywh, self.angle[None]]) @property def result(self): """Get current tracking results.""" coords = self.xyxy if self.angle is None else self.xywha return coords.tolist() + [self.track_id, self.score, self.cls, self.idx] def __repr__(self): """Return a string representation of the SOTTracker object with start and end frames and track ID.""" return f"OT_{self.track_id}_({self.start_frame}-{self.end_frame})" class SOTTracker: """ SOTTracker: A tracking algorithm built on top of YOLOv8 for object detection and tracking. The class is responsible for initializing, updating, and managing the tracks for detected objects in a video sequence. It maintains the state of tracked, lost, and removed tracks over frames, utilizes Kalman filtering for predicting the new object locations, and performs data association. Attributes: tracked_stracks (list[STrack]): List of successfully activated tracks. lost_stracks (list[STrack]): List of lost tracks. removed_stracks (list[STrack]): List of removed tracks. frame_id (int): The current frame ID. args (namespace): Command-line arguments. max_time_lost (int): The maximum frames for a track to be considered as 'lost'. kalman_filter (object): Kalman Filter object. Methods: update(results, img=None): Updates object tracker with new detections. get_kalmanfilter(): Returns a Kalman filter object for tracking bounding boxes. init_track(dets, scores, cls, img=None): Initialize object tracking with detections. get_dists(tracks, detections): Calculates the distance between tracks and detections. multi_predict(tracks): Predicts the location of tracks. reset_id(): Resets the ID counter of STrack. joint_stracks(tlista, tlistb): Combines two lists of stracks. sub_stracks(tlista, tlistb): Filters out the stracks present in the second list from the first list. remove_duplicate_stracks(stracksa, stracksb): Removes duplicate stracks based on IoU. """ def __init__(self, args, frame_rate=30): """Initialize a YOLOv8 object to track objects with given arguments and frame rate.""" self.tracked_stracks = [] # type: list[STrack] self.lost_stracks = [] # type: list[STrack] self.removed_stracks = [] # type: list[STrack] self.frame_id = 0 self.args = args self.max_time_lost = int(frame_rate / 30.0 * args.track_buffer) self.kalman_filter = self.get_kalmanfilter() self.reset_id() self.prior_strack = None def update(self, results, img=None): """Updates object tracker with new detections and returns tracked object bounding boxes.""" self.frame_id += 1 self.prior_strack = None activated_stracks = [] refind_stracks = [] lost_stracks = [] removed_stracks = [] scores = results.conf bboxes = results.xywhr if hasattr(results, "xywhr") else results.xywh # Add index bboxes = np.concatenate([bboxes, np.arange(len(bboxes)).reshape(-1, 1)], axis=-1) cls = results.cls # if scores[0] < 0.1: # note:当box为0时IOU为1 # return np.asarray([]) # Add newly detected tracklets to tracked_stracks unconfirmed = [] tracked_stracks = [] # type: list[STrack] for track in self.tracked_stracks: if not track.is_activated: unconfirmed.append(track) else: tracked_stracks.append(track) strack_pool = self.joint_stracks(tracked_stracks, self.lost_stracks) # Predict the current location with KF self.multi_predict(strack_pool) # important for correct state if hasattr(self, "gmc") and img is not None: warp = self.gmc.apply(img, dets) STrack.multi_gmc(strack_pool, warp) STrack.multi_gmc(unconfirmed, warp) # step 000 for sot prior_strack = None prior_update_id = -1 candidate_det = None len_max = 0 # 寻找优势目标 # 优先在激活目标中寻找 for track in self.tracked_stracks: if track.tracklet_len_score > max(len_max, 5): prior_strack = track len_max = track.tracklet_len_score # 没有则在短时丢失目标中寻找 if prior_strack is None: for track in self.lost_stracks: if track.tracklet_len_score > max(len_max, 5): prior_strack = track len_max = track.tracklet_len_score # 存在优势目标后进行匹配 if prior_strack is not None: sot_detections = self.init_track(bboxes, scores, cls, img) simi_dists = self.get_simi_dists([prior_strack], sot_detections) iou_dists = matching.iou_distance([prior_strack], sot_detections) # idet = np.argmax(iou_dists) # if iou_dists[0, idet] > 0.5: # note:当box为0时IOU为1 # prior_strack.update(sot_detections[idet], self.frame_id) # activated_stracks.append(prior_strack) # prior_update_id = idet # elif simi_dists.max() > 0.6: # IOU计算替换: # # 置信度较高则直接更新 # if sot_detections[idet].score > 0.3: # prior_strack.update_simi(sot_detections[idet], self.frame_id) # activated_stracks.append(prior_strack) # prior_update_id = idet # # 置信度较低则采用更严格匹配 # else: # candidate_det = sot_detections[idet] # track_id # candidate_det.track_id = prior_strack.track_id idet = np.argmax(simi_dists) if simi_dists.max() > 0.6: # IOU计算替换 # 置信度较高则直接更新 if iou_dists[0, idet] > 0.5: prior_strack.update(sot_detections[idet], self.frame_id) activated_stracks.append(prior_strack) prior_update_id = idet else: if sot_detections[idet].score > 0.2: # 0.3 prior_strack.update_simi(sot_detections[idet], self.frame_id) activated_stracks.append(prior_strack) prior_update_id = idet # 置信度较低则采用更严格匹配 else: candidate_det = sot_detections[idet] # track_id candidate_det.track_id = prior_strack.track_id if prior_update_id != -1: scores[idet] = 0 # for sot remain_inds = scores >= self.args.track_high_thresh inds_low = scores > self.args.track_low_thresh inds_high = scores < self.args.track_high_thresh inds_second = inds_low & inds_high dets_second = bboxes[inds_second] dets = bboxes[remain_inds] scores_keep = scores[remain_inds] scores_second = scores[inds_second] cls_keep = cls[remain_inds] cls_second = cls[inds_second] # Step 2: First association, with high score detection boxes detections = self.init_track(dets, scores_keep, cls_keep, img) if prior_update_id != -1: strack_pool = self.sub_stracks(strack_pool, [prior_strack]) # for sot dists = self.get_dists(strack_pool, detections) matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.args.match_thresh) for itracked, idet in matches: track = strack_pool[itracked] det = detections[idet] if track.state == TrackState.Tracked: track.update(det, self.frame_id) activated_stracks.append(track) else: track.re_activate(det, self.frame_id, new_id=False) refind_stracks.append(track) # Step 3: Second association, with low score detection boxes association the untrack to the low score detections detections_second = self.init_track(dets_second, scores_second, cls_second, img) r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked] # TODO dists = matching.iou_distance(r_tracked_stracks, detections_second) matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.5) for itracked, idet in matches: track = r_tracked_stracks[itracked] det = detections_second[idet] if track.state == TrackState.Tracked: track.update(det, self.frame_id) activated_stracks.append(track) else: track.re_activate(det, self.frame_id, new_id=False) refind_stracks.append(track) for it in u_track: track = r_tracked_stracks[it] if track.state != TrackState.Lost: track.mark_lost() lost_stracks.append(track) # Deal with unconfirmed tracks, usually tracks with only one beginning frame detections = [detections[i] for i in u_detection] dists = self.get_dists(unconfirmed, detections) matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7) for itracked, idet in matches: unconfirmed[itracked].update(detections[idet], self.frame_id) activated_stracks.append(unconfirmed[itracked]) for it in u_unconfirmed: track = unconfirmed[it] track.mark_removed() removed_stracks.append(track) # Step 4: Init new stracks for inew in u_detection: track = detections[inew] if track.score < self.args.new_track_thresh: continue track.activate(self.kalman_filter, self.frame_id) activated_stracks.append(track) # Step 5: Update state for track in self.lost_stracks: if self.frame_id - track.end_frame > self.max_time_lost: track.mark_removed() removed_stracks.append(track) self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked] self.tracked_stracks = self.joint_stracks(self.tracked_stracks, activated_stracks) self.tracked_stracks = self.joint_stracks(self.tracked_stracks, refind_stracks) self.lost_stracks = self.sub_stracks(self.lost_stracks, self.tracked_stracks) self.lost_stracks.extend(lost_stracks) self.lost_stracks = self.sub_stracks(self.lost_stracks, self.removed_stracks) self.tracked_stracks, self.lost_stracks = self.remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks) self.removed_stracks.extend(removed_stracks) if len(self.removed_stracks) > 1000: self.removed_stracks = self.removed_stracks[-999:] # clip remove stracks to 1000 maximum # 当多个目标时:只输出优先目标 # 当没有可靠目标但有检测时:输出simi最相似的目标 prior_strack = None len_max = -1 for track in self.tracked_stracks: if track.state == TrackState.Tracked and track.tracklet_len_score > len_max: prior_strack = track len_max = track.tracklet_len_score # 初始化点置信度将为0.25,按轨迹长度进行置信度分级,并采用轨迹加权以减轻一直误检影响 score_max = 0 if prior_strack is not None and prior_strack.tracklet_len_score < 1: for track in self.tracked_stracks: #选出置信度最高 if track.state == TrackState.Tracked and track.score > score_max: prior_strack = track score_max = track.score if prior_strack.score < 0.6: prior_strack = None if prior_strack is not None: self.prior_strack = prior_strack return np.asarray([prior_strack.det.tolist()+prior_strack.result[-4:]], dtype=np.float32) # result为滤波调整后 elif candidate_det is not None: return np.asarray([candidate_det.result], dtype=np.float32) else: # valid_indices = np.where(scores > 0)[0] # if valid_indices.size > 0: # max_idx = valid_indices[np.argmax(scores[valid_indices])] # out_det = STrack(bboxes[max_idx], scores[max_idx], cls[max_idx]) # return np.asarray([out_det.result], dtype=np.float32) return np.asarray([]) # return np.asarray([x.result for x in self.tracked_stracks if x.is_activated], dtype=np.float32) def prior_predict(self): """Returns a Kalman filter object for tracking bounding boxes.""" if self.prior_strack is not None: return self.prior_strack.prior_predict() return None def multi_gmc(self, warp): STrack.multi_gmc(self.tracked_stracks, warp) STrack.multi_gmc(self.lost_stracks, warp) def get_kalmanfilter(self): """Returns a Kalman filter object for tracking bounding boxes.""" return KalmanFilterXYAH() def init_track(self, dets, scores, cls, img=None): """Initialize object tracking with detections and scores using STrack algorithm.""" return [STrack(xyxy, s, c) for (xyxy, s, c) in zip(dets, scores, cls)] if len(dets) else [] # detections def get_dists(self, tracks, detections): """Calculates the distance between tracks and detections using IoU and fuses scores.""" dists = matching.iou_distance(tracks, detections) # TODO: mot20 # if not self.args.mot20: dists = matching.fuse_score(dists, detections) return dists def get_simi_dists(self, tracks, detections): """Calculates the distance between tracks and detections using IoU and fuses scores.""" dists = self.simi_distance(tracks, detections) return dists def multi_predict(self, tracks): """Returns the predicted tracks using the YOLOv8 network.""" STrack.multi_predict(tracks) @staticmethod def reset_id(): """Resets the ID counter of STrack.""" STrack.reset_id() def reset(self): """Reset tracker.""" self.tracked_stracks = [] # type: list[STrack] self.lost_stracks = [] # type: list[STrack] self.removed_stracks = [] # type: list[STrack] self.frame_id = 0 self.kalman_filter = self.get_kalmanfilter() self.reset_id() @staticmethod def joint_stracks(tlista, tlistb): """Combine two lists of stracks into a single one.""" exists = {} res = [] for t in tlista: exists[t.track_id] = 1 res.append(t) for t in tlistb: tid = t.track_id if not exists.get(tid, 0): exists[tid] = 1 res.append(t) return res @staticmethod def sub_stracks(tlista, tlistb): """DEPRECATED CODE in https://github.com/ultralytics/ultralytics/pull/1890/ stracks = {t.track_id: t for t in tlista} for t in tlistb: tid = t.track_id if stracks.get(tid, 0): del stracks[tid] return list(stracks.values()) """ track_ids_b = {t.track_id for t in tlistb} return [t for t in tlista if t.track_id not in track_ids_b] @staticmethod def remove_duplicate_stracks(stracksa, stracksb): """Remove duplicate stracks with non-maximum IoU distance.""" pdist = matching.iou_distance(stracksa, stracksb) pairs = np.where(pdist < 0.15) dupa, dupb = [], [] for p, q in zip(*pairs): timep = stracksa[p].frame_id - stracksa[p].start_frame timeq = stracksb[q].frame_id - stracksb[q].start_frame if timep > timeq: dupb.append(q) else: dupa.append(p) resa = [t for i, t in enumerate(stracksa) if i not in dupa] resb = [t for i, t in enumerate(stracksb) if i not in dupb] return resa, resb @staticmethod def simi_distance(atracks: list, btracks: list) -> np.ndarray: """ Compute cost based on Intersection over Union (IoU) between tracks. Args: atracks (list[STrack] | list[np.ndarray]): List of tracks 'a' or bounding boxes. btracks (list[STrack] | list[np.ndarray]): List of tracks 'b' or bounding boxes. Returns: (np.ndarray): Cost matrix computed based on IoU. """ if atracks and isinstance(atracks[0], np.ndarray) or btracks and isinstance(btracks[0], np.ndarray): atlbrs = atracks btlbrs = btracks else: atlbrs = [track.xywha if track.angle is not None else track.xyxy for track in atracks] btlbrs = [track.xywha if track.angle is not None else track.xyxy for track in btracks] simi = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float32) if len(atlbrs) and len(btlbrs): simi = SOTTracker.bbox_simi( np.ascontiguousarray(atlbrs, dtype=np.float32), np.ascontiguousarray(btlbrs, dtype=np.float32), ) return 1 - simi # cost matrix @staticmethod def bbox_simi(box1, box2, eps=1e-7): """ Calculate the intersection over box2 area given box1 and box2. Boxes are in x1y1x2y2 format. Args: box1 (np.ndarray): A numpy array of shape (n, 4) representing n bounding boxes. box2 (np.ndarray): A numpy array of shape (m, 4) representing m bounding boxes. iou (bool): Calculate the standard IoU if True else return inter_area/box2_area. eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. Returns: (np.ndarray): A numpy array of shape (n, m) representing the intersection over box2 area. """ # Get the coordinates of bounding boxes b1_x1, b1_y1, b1_x2, b1_y2 = box1.T b2_x1, b2_y1, b2_x2, b2_y2 = box2.T cx1 = (b1_x1 + b1_x2) / 2 cy1 = (b1_y1 + b1_y2) / 2 cx2 = (b2_x1 + b2_x2) / 2 cy2 = (b2_y1 + b2_y2) / 2 w1 = b1_x2 - b1_x1 h1 = b1_y2 - b1_y1 w2 = b2_x2 - b2_x1 h2 = b2_y2 - b2_y1 cw = np.maximum(b1_x2,b2_x2) - np.minimum(b1_x1,b2_x1) # convex (smallest enclosing box) width ch = np.maximum(b1_y2,b2_y2) - np.minimum(b1_y1,b2_y1) # convex height # shape = abs(w1*h1-w2*h2) + abs(w1*h2-w2*h1) # dis = (cx1 - cx2)**2 + (cy1 - cy2)**2 shape = 0.5*np.abs(w1*h1-w2*h2)/(np.maximum(w1*h1,w2*h2)+eps) + 0.5*np.abs(w1*h2-w2*h1)/(np.maximum(w1*h2,w2*h1)+eps) dis = np.sqrt((cx1 - cx2)**2 + (cy1 - cy2)**2) / (np.sqrt(cw**2 + ch**2) + eps) simi = shape * 0.8 + dis * 0.2 # if shape.shape[0] > 1: # print(simi) return simi