add evaluation code

README.md

@@ -79,6 +79,22 @@ MOT16-13
  45.7 74.4 33.0| 40.5  91.4  0.53|  380  48  172  160|  1915 30072   192   594|  36.3  75.9  36.7
 ```
 
+### Evaluate
+You can use official [matlab eval devkit](https://bitbucket.org/amilan/motchallenge-devkit/) to evaluate the outputs.
+Or directly use the python version [motmetrics](https://github.com/cheind/py-motmetrics).
+I already added the python evaluation method in the `eval_mot.py` script.
+The results are slightly different from the official devkit since the ignoring method is not identical.
+Results from python evaluation:
+```
+          IDF1   IDP   IDR  Rcll  Prcn  GT MT  PT  ML   FP    FN IDs   FM  MOTA  MOTP
+MOT16-02 37.1% 75.6% 24.6% 30.2% 93.0%  54  7  21  26  406 12440  47  146 27.7% 0.247
+MOT16-05 53.7% 83.0% 39.7% 44.6% 93.1% 125 13  68  44  224  3779  35  130 40.8% 0.242
+MOT16-09 61.1% 75.8% 51.1% 63.6% 94.3%  25  8  15   2  202  1913  28   64 59.2% 0.247
+MOT16-11 54.9% 72.2% 44.3% 58.1% 94.7%  69 12  28  29  301  3840  27   70 54.6% 0.208
+MOT16-13 38.2% 71.6% 26.0% 29.7% 81.6% 107 11  38  58  766  8051  46  178 22.6% 0.276
+OVERALL  46.1% 75.1% 33.3% 40.6% 91.5% 380 51 170 159 1899 30023 183  588 36.5% 0.241
+```
+
 ### Resources
 
 Paper: Real-time Multiple People Tracking with Deeply Learned Candidate Selection and Person Re-identification ([researchgate](https://www.researchgate.net/publication/326224594_Real-time_Multiple_People_Tracking_with_Deeply_Learned_Candidate_Selection_and_Person_Re-identification), [arxiv](https://arxiv.org/abs/1809.04427))

datasets/__init__.py

@@ -0,0 +1,17 @@
+from .mot_seq import MOTSeq
+
+
+__factory = {
+    # 'kitti': KITTISeq,
+    'mot': MOTSeq,
+}
+
+
+def get_names():
+    return tuple(__factory.keys())
+
+
+def init_dataset(name, *args, **kwargs):
+    if name not in get_names():
+        raise KeyError("Unknown dataset: {}".format(name))
+    return __factory[name](*args, **kwargs)

datasets/mot_seq.py

@@ -3,6 +3,8 @@
 import torch.utils.data as data
 from scipy.misc import imread
 
+from utils.io import read_mot_results, unzip_objs
+
 
 """
 labels={'ped', ...			% 1
@@ -22,28 +24,28 @@
 """
 
 
-def read_mot_results(filename, is_gt=False):
-    labels = {1, 7, -1}
-    targets = dict()
-    if os.path.isfile(filename):
-        with open(filename, 'r') as f:
-            for line in f.readlines():
-                linelist = line.split(',')
-                if len(linelist) < 7:
-                    continue
-                fid = int(linelist[0])
-                targets.setdefault(fid, list())
-
-                if is_gt and ('MOT16-' in filename or 'MOT17-' in filename):
-                    label = int(float(linelist[-2])) if len(linelist) > 7 else -1
-                    if label not in labels:
-                        continue
-                tlwh = tuple(map(float, linelist[2:7]))
-                target_id = int(linelist[1])
-
-                targets[fid].append((tlwh, target_id))
-
-    return targets
+# def read_mot_results(filename, is_gt=False):
+#     labels = {1, -1}
+#     targets = dict()
+#     if os.path.isfile(filename):
+#         with open(filename, 'r') as f:
+#             for line in f.readlines():
+#                 linelist = line.split(',')
+#                 if len(linelist) < 7:
+#                     continue
+#                 fid = int(linelist[0])
+#                 targets.setdefault(fid, list())
+#
+#                 if is_gt and ('MOT16-' in filename or 'MOT17-' in filename):
+#                     label = int(float(linelist[-2])) if len(linelist) > 7 else -1
+#                     if label not in labels:
+#                         continue
+#                 tlwh = tuple(map(float, linelist[2:7]))
+#                 target_id = int(linelist[1])
+#
+#                 targets[fid].append((tlwh, target_id))
+#
+#     return targets
 
 
 class MOTSeq(data.Dataset):
@@ -60,11 +62,11 @@ def __init__(self, root, det_root, seq_name, min_height, min_det_score):
             self.det_file = os.path.join(self.root, self.seq_name, 'det', 'det.txt')
         else:
             self.det_file = os.path.join(det_root, '{}.txt'.format(self.seq_name))
-        self.dets = read_mot_results(self.det_file, is_gt=False)
+        self.dets = read_mot_results(self.det_file, is_gt=False, is_ignore=False)
 
         self.gt_file = os.path.join(self.root, self.seq_name, 'gt', 'gt.txt')
         if os.path.isfile(self.gt_file):
-            self.gts = read_mot_results(self.gt_file, is_gt=True)
+            self.gts = read_mot_results(self.gt_file, is_gt=True, is_ignore=False)
         else:
             self.gts = None
 
@@ -77,24 +79,18 @@ def __getitem__(self, i):
         im = imread(im_name)  # rgb
         im = im[:, :, ::-1]  # bgr
 
-
         frame = i + 1
         dets = self.dets.get(frame, [])
-        dets, track_ids = zip(*self.dets[frame]) if len(dets) > 0 else (np.empty([0, 5]), np.empty([0, 1]))
-        dets = np.asarray(dets)
-        tlwhs = dets[:, 0:4]
-        scores = dets[:, 4]
+        tlwhs, _, scores = unzip_objs(dets)
+        scores = np.asarray(scores)
 
         keep = (tlwhs[:, 3] >= self.min_height) & (scores > self.min_det_score)
         tlwhs = tlwhs[keep]
         scores = scores[keep]
-        track_ids = np.asarray(track_ids, dtype=np.int)[keep]
 
         if self.gts is not None:
             gts = self.gts.get(frame, [])
-            gt_tlwhs, gt_ids = zip(*self.gts[frame]) if len(gts) > 0 else (np.empty([0, 5]), np.empty([0, 1]))
-            gt_tlwhs = np.asarray(gt_tlwhs)
-            gt_tlwhs = gt_tlwhs[:, 0:4]
+            gt_tlwhs, gt_ids, _ = unzip_objs(gts)
         else:
             gt_tlwhs, gt_ids = None, None
 

eval_mot.py

@@ -1,12 +1,14 @@
 import os
 import cv2
 import logging
+import motmetrics as mm
 from tracker.mot_tracker import OnlineTracker
 
 from datasets.mot_seq import get_loader
 from utils import visualization as vis
 from utils.log import logger
 from utils.timer import Timer
+from utils.evaluation import Evaluator
 
 
 def mkdirs(path):
@@ -92,14 +94,35 @@ def main(data_root='/data/MOT16/train', det_root=None,
     data_type = 'mot'
 
     # run tracking
+    accs = []
     for seq in seqs:
         output_dir = os.path.join(data_root, 'outputs', seq) if save_image else None
 
         logger.info('start seq: {}'.format(seq))
         loader = get_loader(data_root, det_root, seq)
-        eval_seq(loader, data_type, os.path.join(result_root, '{}.txt'.format(seq)),
+        result_filename = os.path.join(result_root, '{}.txt'.format(seq))
+        eval_seq(loader, data_type, result_filename,
                  save_dir=output_dir, show_image=show_image)
 
+        # eval
+        logger.info('Evaluate seq: {}'.format(seq))
+        evaluator = Evaluator(data_root, seq, data_type)
+        accs.append(evaluator.eval_file(result_filename))
+
+    # get summary
+    # metrics = ['mota', 'num_switches', 'idp', 'idr', 'idf1', 'precision', 'recall']
+    metrics = mm.metrics.motchallenge_metrics
+    # metrics = None
+    mh = mm.metrics.create()
+    summary = Evaluator.get_summary(accs, seqs, metrics)
+    strsummary = mm.io.render_summary(
+        summary,
+        formatters=mh.formatters,
+        namemap=mm.io.motchallenge_metric_names
+    )
+    print(strsummary)
+    Evaluator.save_summary(summary, os.path.join(result_root, f'summary_{exp_name}.xlsx'))
+
     # # eval
     # try:
     #     import matlab.engine as matlab_engine
@@ -119,17 +142,17 @@ def main(data_root='/data/MOT16/train', det_root=None,
 
 
 if __name__ == '__main__':
-    import fire
-    fire.Fire(main)
-
-    # seqs_str = '''MOT16-02
-    #             MOT16-05
-    #             MOT16-09
-    #             MOT16-11
-    #             MOT16-13'''
-    # seqs = [seq.strip() for seq in seqs_str.split()]
-    #
-    # main(data_root='/data/MOT16/train',
-    #      seqs=seqs,
-    #      exp_name='mot16_val',
-    #      show_image=True)
+    # import fire
+    # fire.Fire(main)
+
+    seqs_str = '''MOT16-02
+                MOT16-05
+                MOT16-09
+                MOT16-11
+                MOT16-13'''
+    seqs = [seq.strip() for seq in seqs_str.split()]
+
+    main(data_root='/data/MOT16/train',
+         seqs=seqs,
+         exp_name='mot16_val',
+         show_image=False)

requirements.txt

@@ -8,4 +8,5 @@ cffi
 sklearn
 numba
 torchvision
-fire
+fire
+motmetrics

utils/evaluation.py

@@ -0,0 +1,101 @@
+import os
+import numpy as np
+import copy
+import motmetrics as mm
+
+from utils.io import read_results, unzip_objs
+
+
+class Evaluator(object):
+
+    def __init__(self, data_root, seq_name, data_type):
+        self.data_root = data_root
+        self.seq_name = seq_name
+        self.data_type = data_type
+
+        self.load_annotations()
+        self.reset_accumulator()
+
+    def load_annotations(self):
+        assert self.data_type == 'mot'
+
+        gt_filename = os.path.join(self.data_root, self.seq_name, 'gt', 'gt.txt')
+        self.gt_frame_dict = read_results(gt_filename, self.data_type, is_gt=True)
+        self.gt_ignore_frame_dict = read_results(gt_filename, self.data_type, is_ignore=True)
+
+    def reset_accumulator(self):
+        self.acc = mm.MOTAccumulator(auto_id=True)
+
+    def eval_frame(self, frame_id, trk_tlwhs, trk_ids, rtn_events=False):
+        # results
+        trk_tlwhs = np.copy(trk_tlwhs)
+        trk_ids = np.copy(trk_ids)
+
+        # gts
+        gt_objs = self.gt_frame_dict.get(frame_id, [])
+        gt_tlwhs, gt_ids = unzip_objs(gt_objs)[:2]
+
+        # ignore boxes
+        ignore_objs = self.gt_ignore_frame_dict.get(frame_id, [])
+        ignore_tlwhs = unzip_objs(ignore_objs)[0]
+
+        # remove ignored results
+        keep = np.ones(len(trk_tlwhs), dtype=bool)
+        iou_distance = mm.distances.iou_matrix(ignore_tlwhs, trk_tlwhs, max_iou=0.5)
+        match_is, match_js = mm.lap.linear_sum_assignment(iou_distance)
+        match_is, match_js = map(lambda a: np.asarray(a, dtype=int), [match_is, match_js])
+        match_ious = iou_distance[match_is, match_js]
+
+        match_js = np.asarray(match_js, dtype=int)
+        match_js = match_js[np.logical_not(np.isnan(match_ious))]
+        keep[match_js] = False
+        trk_tlwhs = trk_tlwhs[keep]
+        trk_ids = trk_ids[keep]
+
+        # get distance matrix
+        iou_distance = mm.distances.iou_matrix(gt_tlwhs, trk_tlwhs, max_iou=0.5)
+
+        # acc
+        self.acc.update(gt_ids, trk_ids, iou_distance)
+
+        if rtn_events and iou_distance.size > 0 and hasattr(self.acc, 'last_mot_events'):
+            events = self.acc.last_mot_events  # only supported by https://github.com/longcw/py-motmetrics
+        else:
+            events = None
+        return events
+
+    def eval_file(self, filename):
+        self.reset_accumulator()
+
+        result_frame_dict = read_results(filename, self.data_type, is_gt=False)
+        frames = sorted(list(set(self.gt_frame_dict.keys()) | set(result_frame_dict.keys())))
+        for frame_id in frames:
+            trk_objs = result_frame_dict.get(frame_id, [])
+            trk_tlwhs, trk_ids = unzip_objs(trk_objs)[:2]
+            self.eval_frame(frame_id, trk_tlwhs, trk_ids, rtn_events=False)
+
+        return self.acc
+
+    @staticmethod
+    def get_summary(accs, names, metrics=('mota', 'num_switches', 'idp', 'idr', 'idf1', 'precision', 'recall')):
+        names = copy.deepcopy(names)
+        if metrics is None:
+            metrics = mm.metrics.motchallenge_metrics
+        metrics = copy.deepcopy(metrics)
+
+        mh = mm.metrics.create()
+        summary = mh.compute_many(
+            accs,
+            metrics=metrics,
+            names=names,
+            generate_overall=True
+        )
+
+        return summary
+
+    @staticmethod
+    def save_summary(summary, filename):
+        import pandas as pd
+        writer = pd.ExcelWriter(filename)
+        summary.to_excel(writer)
+        writer.save()