Merge branch 'release/1.0.1'

Release.md

@@ -13,4 +13,9 @@
     - python setup.py bdist_wheel
     - twine upload dist\*
     - git add, commit
-    - git flow release finish <VERSION>
+    - git flow release finish <VERSION>
+    - git push
+    - git push --tags
+    - git checkout master
+    - git push
+    - git checkout develop

motmetrics/__init__.py

@@ -7,4 +7,4 @@
 
 
 # Needs to be last line
-__version__ = '1.0.0'
+__version__ = '1.0.1'

motmetrics/apps/eval_motchallenge.py

@@ -10,6 +10,7 @@
 import logging
 import motmetrics as mm
 import pandas as pd
+from collections import OrderedDict
 from pathlib import Path
 
 def parse_args():
@@ -48,14 +49,13 @@ def parse_args():
     parser.add_argument('--fmt', type=str, help='Data format', default='mot15-2D')
     return parser.parse_args()
 
-def compare_dataframes(gt, ts):
-
+def compare_dataframes(gts, ts):
     accs = []
     names = []
     for k, tsacc in ts.items():
-        if k in gt:            
+        if k in gts:            
             logging.info('Comparing {}...'.format(k))
-            accs.append(mm.utils.compare_to_groundtruth(gt[k], tsacc, 'iou', distth=0.5))
+            accs.append(mm.utils.compare_to_groundtruth(gts[k], tsacc, 'iou', distth=0.5))
             names.append(k)
         else:
             logging.warning('No ground truth for {}, skipping.'.format(k))
@@ -77,11 +77,10 @@ def compare_dataframes(gt, ts):
     logging.info('Found {} groundtruths and {} test files.'.format(len(gtfiles), len(tsfiles)))
     logging.info('Loading files.')
 
-    gt = dict([(Path(f).parts[-3], mm.io.loadtxt(f, fmt='mot15-2D')) for f in gtfiles])
-    ts = dict([(os.path.splitext(Path(f).parts[-1])[0], mm.io.loadtxt(f, fmt='mot15-2D')) for f in tsfiles])
-
-
-    mh = mm.metrics.create()
+    gt = OrderedDict([(Path(f).parts[-3], mm.io.loadtxt(f, fmt=args.fmt, min_confidence=1)) for f in gtfiles])
+    ts = OrderedDict([(os.path.splitext(Path(f).parts[-1])[0], mm.io.loadtxt(f, fmt=args.fmt)) for f in tsfiles])    
+
+    mh = mm.metrics.create()    
     accs, names = compare_dataframes(gt, ts)
 
     logging.info('Running metrics')

motmetrics/io.py

@@ -32,22 +32,33 @@ def load_motchallenge(fname, **kwargs):
     fname : str
         Filename to load data from
 
+    Kwargs
+    ------
+    sep : str
+        Allowed field separators, defaults to '\s+|\t+|,'
+    min_confidence : float
+        Rows with confidence less than this threshold are removed.
+        Defaults to -1. You should set this to 1 when loading 
+        ground truth MOTChallenge data, so that invalid rectangles in
+        the ground truth are not considered during matching.
+
     Returns
     ------
     df : pandas.DataFrame 
         The returned dataframe has the following columns
-            'X', 'Y', 'Width', 'Height', 'Score', 'ClassId', 'Visibility'
+            'X', 'Y', 'Width', 'Height', 'Confidence', 'ClassId', 'Visibility'
         The dataframe is indexed by ('FrameId', 'Id')    
     """
 
     sep = kwargs.pop('sep', '\s+|\t+|,')
+    min_confidence = kwargs.pop('min_confidence', -1)
     df = pd.read_csv(
         fname, 
         sep=sep, 
         index_col=[0,1], 
         skipinitialspace=True, 
         header=None,
-        names=['FrameId', 'Id', 'X', 'Y', 'Width', 'Height', 'Score', 'ClassId', 'Visibility', 'unused'],
+        names=['FrameId', 'Id', 'X', 'Y', 'Width', 'Height', 'Confidence', 'ClassId', 'Visibility', 'unused'],
         engine='python'
     )
 
@@ -57,7 +68,8 @@ def load_motchallenge(fname, **kwargs):
     # Removed trailing column
     del df['unused']
 
-    return df
+    # Remove all rows without sufficient confidence
+    return df[df['Confidence'] >= min_confidence]
 
 def load_vatictxt(fname, **kwargs):
     """Load Vatic text format.

motmetrics/metrics.py

@@ -219,8 +219,8 @@ def num_objects(df):
 
 def track_ratios(df, obj_frequencies):
     """Ratio of assigned to total appearance count per unique object id."""   
-    tracked = df[df.Type !='MISS']['OId'].value_counts()   
-    return tracked.div(obj_frequencies).fillna(1.)
+    tracked = df[df.Type != 'MISS']['OId'].value_counts()
+    return tracked.div(obj_frequencies).fillna(0.)
 
 def mostly_tracked(df, track_ratios):
     """Number of objects tracked for at least 80 percent of lifespan."""

motmetrics/utils.py

@@ -36,23 +36,37 @@ def compare_to_groundtruth(gt, dt, dist='iou', distfields=['X', 'Y', 'Width', 'H
         Maximum tolerable distance. Pairs exceeding this threshold are marked 'do-not-pair'.
     """
 
+    def compute_iou(a, b):
+        return iou_matrix(a, b, max_iou=distth)
+
+    def compute_euc(a, b):
+        return norm2squared_matrix(a, b, max_d2=distth)
+
+    compute_dist = compute_iou if dist.upper() == 'IOU' else compute_euc
+
     acc = MOTAccumulator()
 
-    for frameid, fgt in gt.groupby(level=0):
-        fgt = fgt.loc[frameid] 
-        oids = fgt.index.values
-
-        hids = []
-        dists = []
+    # We need to account for all frames reported either by ground truth or
+    # detector. In case a frame is missing in GT this will lead to FPs, in 
+    # case a frame is missing in detector results this will lead to FNs.
+    allframeids = gt.index.union(dt.index).levels[0]
+
+    for fid in allframeids:
+        oids = np.empty(0)
+        hids = np.empty(0)
+        dists = np.empty((0,0))
 
-        if frameid in dt.index:
-            fdt = dt.loc[frameid]
-            hids = fdt.index.values
-            if dist == 'iou':                
-                dists = iou_matrix(fgt[distfields].values, fdt[distfields].values, max_iou=distth)
-            else:
-                dists = norm2squared_matrix(fgt[distfields].values, fdt[distfields].values, max_d2=distth)
+        if fid in gt.index:
+            fgt = gt.loc[fid] 
+            oids = fgt.index.values
 
-        acc.update(oids, hids, dists, frameid=frameid)
+        if fid in dt.index:
+            fdt = dt.loc[fid]
+            hids = fdt.index.values
 
+        if oids.shape[0] > 0 and hids.shape[0] > 0:
+            dists = compute_dist(fgt[distfields].values, fdt[distfields].values)
+
+        acc.update(oids, hids, dists, frameid=fid)
+
     return acc