add visualization of fixed camera positions

.gitignore

@@ -2,6 +2,10 @@
 output/
 *data/
 experiment
+
+# plots
+*.png
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

src/calib.py

@@ -1,4 +1,7 @@
 import matplotlib.pyplot as plt
+from matplotlib.patches import Polygon
+from matplotlib.collections import PatchCollection
+from matplotlib import cm
 import cv2
 import numpy as np
 import os
@@ -174,7 +177,7 @@ def backproject_pitch(P, x, C_cam):
     p = np.dot(X.T, plane_normal) * C_homo - np.dot(C_homo.T, plane_normal) * X
 
     p /= p[-1]
-    return p
+    return p.ravel()
 
 
 def display_soccer_pitch_ground(points, imgname, C_cam):
@@ -192,8 +195,56 @@ def display_soccer_pitch_ground(points, imgname, C_cam):
         plt.scatter(p[0], p[1])
         plt.annotate(p[2], (p[0], p[1]))
 
-    # plt.show()
     plt.savefig(imgname)
+    plt.show()
+
+def display_cameras_on_pitch(bboxs, cams, keys, imgname):
+    fig, ax = plt.subplots(1,1,figsize=(16, 9))
+
+    for pts in pitch:
+        pts[[1, 2]] = pts[[2, 1]]
+        px, py = pts[0], pts[2]
+        plt.plot(SCALE * px, SCALE * py, 'r-')
+
+    patches = []
+    # colors = iter([plt.cm.Paired(i) for i in range(len(cams))])
+    colors = []
+    for bbox, cam, key in zip(bboxs, cams, keys):
+        # clr = next(colors)
+
+        if int(key) % 4 == 0:
+            clr = 'red'
+        elif int(key) % 4 == 1:
+            clr = 'gray'
+        elif int(key) % 4 == 2:
+            clr = 'red'
+        else:
+            clr = 'gray'
+        plt.scatter(cam[0], cam[1], c=clr)
+        plt.annotate('Camera '+key, (cam[0], cam[1]))
+
+        # plot the lines
+        for coord in bbox:
+            # line = np.vstack([cam, coord])
+            plt.plot([cam[0], coord[0]], [cam[1], coord[1]], color=clr, linestyle=':', alpha=0.8)
+
+        # plot field of view
+        polygon = Polygon(bbox, True)
+        patches.append(polygon)
+        colors.append(clr)
+
+    # p = PatchCollection(patches, alpha=0.4, cmap=plt.cm.Paired)
+    # p.set_array(np.arange(len(cams)))
+    p = PatchCollection(patches, alpha=0.1, color='gray')
+    # p.set_array(np.arange(len(cams)))
+    ax.add_collection(p)
+    p.set_color(colors)
+    p.set_alpha(list(map(lambda x: 0.2 if x == 'gray' else 0.1, colors)))
+    ax.set_xbound(-85, 85)
+    ax.set_ybound(-60,60)
+
+    plt.savefig(imgname)
+    plt.show()
 
 
 def visualize_tracks_on_pitch(tracks):
@@ -246,9 +297,16 @@ def visualize_tracks_on_pitch(tracks):
     opt = a.parse_args()
 
     img = cv2.imread(
-        "/scratch2/wuti/Others/3DVision/0125-0135/ULSAN HYUNDAI FC vs AL DUHAIL SC 16m RIGHT/img/image0001.jpg"
+        "/scratch2/wuti/Others/3DVision/fixed_cameras/FIXED-0125-0135/cam1_img/image0001.png"
     )
 
+    H, W, _ = img.shape
+    print(W, H)
+
+    corners = [[0, 0, 1], [0, H, 1], [W, H, 1], [W, 0, 1]]
+    # cam_list = ["1","2","3","4","5","6","7","8"]
+    cam_list = ["2","5","6"]
+
     # result_file = '/scratch2/wuti/Others/3DVision/test_result_filtered_team/16m_right_filtered_team.txt'
     result_file = opt.res_path
 
@@ -325,10 +383,36 @@ def visualize_tracks_on_pitch(tracks):
         else:
             tracks_pitch.append([track[0], track[1], tx, ty])
 
+    # print(points)
     # display_soccer_pitch_ground(points, 'test.png', C_cam)
 
     np.savetxt(outname, np.array(tracks_pitch), delimiter=',')
 
+    if calib_file.endswith('.json'):
+        # visualize cameras
+        if opt.viz:
+            cams = []
+            bboxs = []
+            keys = []
+            for key, calib_cam in calib.items():
+                if key not in cam_list:
+                    continue
+                # print(key)
+                keys.append(key)
+                K = np.array(calib_cam["K"]).reshape(3,3)
+                R = np.array(calib_cam["R"]).reshape(3,3)
+                T = np.array(calib_cam["T"]).reshape(3,1)
+                P = K @ np.hstack([R, T])
+                cam = -R.T.dot(T).ravel()
+                cams.append(cam)
+                bbox = []
+                for corner in corners:
+                    tx, ty, tz, _ = backproject_pitch(P,np.array(corner).reshape(-1,1),cam)
+                    bbox.append([tx, ty])
+                bboxs.append(np.array(bbox))
+
+            display_cameras_on_pitch(bboxs, cams, keys, 'fixed_cameras_on_pitch_256.png')
+
     # # visualize tracking result
     # # result_file = '/scratch2/wuti/Others/3DVision/cam1_result_filtered_team/cam1_right_team.txt'
     # result_file = '/scratch2/wuti/Others/3DVision/ground_truth/ground_truth_3d/gt_pitch.txt'

src/load_tracab.py

@@ -5,8 +5,9 @@
 import matplotlib.pyplot as plt
 from glob import glob
 import math
-
+import pandas as pd
 from calib import computeP
+from scipy import interpolate
 
 FPS = 25
 team_color_list = [(0,0,255), (255,255,255), (0,255,0), (255,0,0)]
@@ -36,6 +37,53 @@ def load_tracab(filename, length):
         track[:,[4,5]] = track[:,[5,4]]
         yield track
 
+def interpolate_tracab(tracks):
+    # sort by player id and frame id
+    df = pd.DataFrame(tracks)
+    df.columns = ['frame_id', 'jersey_number', 'x', 'y', 'team_id', 'player_id']
+    df = df.sort_values(by=['player_id', 'frame_id']).reset_index()
+
+    grouped = df.groupby('player_id')
+    interpolated_rows = []
+    interp_frames = np.linspace(1, 250, 550)
+    for player, group in grouped:
+        print("interpolate for player %d" %player)
+        record = group.values[:,1:]
+        in_frames = np.where((interp_frames >= record[0,0]) & (interp_frames <= record[-1,0]))[0]
+        group_interp = np.tile(record[0], (in_frames.shape[0],1))
+        frames = record[:,0]
+        fx = interpolate.interp1d(frames, record[:,2])
+        fy = interpolate.interp1d(frames, record[:,3])
+        group_interp[:,0] = in_frames + 1
+        group_interp[:,2] = fx(interp_frames[in_frames])
+        group_interp[:,3] = fy(interp_frames[in_frames])
+        print(record.shape[0], group_interp.shape[0])
+
+        interpolated_rows.append(group_interp)
+
+
+    # # interpolate every two consecutive consecutive frames
+    # for index, row in df.iterrows():
+    #     new_row = row
+    #     if index == df.index[-1]:
+    #         break
+    #     next_row = df.iloc[index+1]
+    #     # print(df.loc[index], df.iloc[index])
+    #     if row['player_id'] != next_row['player_id']:
+    #         continue
+    #     print(row['frame_id'],next_row['frame_id'])
+    #     new_row['frame_id'] = (row['frame_id'] + next_row['frame_id'])/2
+    #     new_row['x'] = (row['x'] + next_row['x'])/2
+    #     new_row['y'] = (row['y'] + next_row['y'])/2
+    #     # print(new_row)
+    #     # input("...")
+    #     interpolated_rows.append(new_row.values[1:])
+    interpolated_tracks = np.vstack(interpolated_rows)
+    print(interpolated_tracks[:5])
+    # print(df.values[:5,1:].shape)
+    # interpolated_tracks = np.vstack([df.values[:,1:],interpolated_rows])
+    # print(interpolated_tracks[-5:])
+    return interpolated_tracks
 if __name__ == "__main__":
     import argparse
     parser = argparse.ArgumentParser()
@@ -91,3 +139,6 @@ def load_tracab(filename, length):
     tracks_on_pitch = tracks[:,[0,3,4,6,1,2]]
     print(tracks_on_pitch)
     np.savetxt(os.path.join(os.path.dirname(args.gt_path),'gt_pitch.txt'),tracks_on_pitch,delimiter=',')
+
+    gt_interp = interpolate_tracab(tracks_on_pitch)
+    np.savetxt(os.path.join(os.path.dirname(args.gt_path),'gt_pitch_550.txt'),gt_interp,delimiter=',')

src/visualize_on_pitch.py

@@ -105,7 +105,7 @@
 ]
 
 
-def visualize_tracks_on_pitch(tracks, gt=None):
+def visualize_tracks_on_pitch(tracks, gt=None, invert=False):
     W, H = int(WIDTH // RESOLUTION), int(HEIGHT // RESOLUTION)
     cx, cz = WIDTH / 2, HEIGHT / 2
 
@@ -144,20 +144,34 @@ def visualize_tracks_on_pitch(tracks, gt=None):
             cv2.polylines(img, pitch_lines, False, (255, 255, 255), 3)
 
         for track in tracks_cur:
-            _, track_id, x, z, teamid, objid = track
+            # _, track_id, x, z, teamid, objid = track
+            _, track_id, x, z = track
+            if invert:
+                x = -x
 
             # draw individual points with id
+            # cv2.putText(img,
+            #             str(int(track_id)) + '(' + str(int(objid)) + ')', (int(
+            #                 (x + cx) // RESOLUTION), int(
+            #                     (z + cz) // RESOLUTION) - 8),
+            #             cv2.FONT_HERSHEY_PLAIN, 2,
+            #             team_color_list[int(teamid)], 2)
             cv2.putText(img,
-                        str(int(track_id)) + '(' + str(int(objid)) + ')', (int(
+                        str(int(track_id)), (int(
                             (x + cx) // RESOLUTION), int(
                                 (z + cz) // RESOLUTION) - 8),
                         cv2.FONT_HERSHEY_PLAIN, 2,
-                        team_color_list[int(teamid)], 2)
+                        color_list[int(track_id)%len(color_list)], 2)
 
+            # cv2.circle(img, (int(
+            #     (x + cx) // RESOLUTION), int((z + cz) // RESOLUTION)),
+            #            radius=10,
+            #            color=team_color_list[int(teamid)],
+            #            thickness=-1)
             cv2.circle(img, (int(
                 (x + cx) // RESOLUTION), int((z + cz) // RESOLUTION)),
                        radius=10,
-                       color=team_color_list[int(teamid)],
+                       color=color_list[int(track_id)%len(color_list)],
                        thickness=-1)
 
         out = None
@@ -168,12 +182,13 @@ def visualize_tracks_on_pitch(tracks, gt=None):
             if gt_frame.shape[0] > 0:
                 overlay = np.zeros((H, W, 3), np.uint8)
                 for pl in gt_frame:
+
                     # frame_id, jersey_number, X, Z, team_id, player_id
                     cv2.circle(overlay, (int((pl[2] + cx) // RESOLUTION),
-                                         int((pl[3] + cz) // RESOLUTION)),
-                               radius=30,
-                               color=team_color_list[int(pl[4])],
-                               thickness=-1)
+                                        int((pl[3] + cz) // RESOLUTION)),
+                            radius=30,
+                            color=team_color_list[int(pl[4])],
+                            thickness=-1)
                 # blend ground truth overlay with the pitch
                 out = cv2.addWeighted(img, 1.0, overlay, 0.5, 1)
         # prev_frame = frame
@@ -194,7 +209,7 @@ def convert_to_footyviz(tracks):
     df['team'] = 'others'
     # print(df.columns)
     df.team[df.teamid == 0] = 'defense'
-    df.team[df.teamid == 1] = 'attach'
+    df.team[df.teamid == 1] = 'attack'
     df.team[df.teamid == 2] = 'referee'
     df['bgcolor'] = 'black'
     df['edgecolor'] = 'black'
@@ -244,6 +259,7 @@ def make_animation(df, fps=FPS, voronoi=False):
                    type=str,
                    help="path to the ground truth tracks",
                    default=None)
+    a.add_argument("--fixed_cam", action='store_true', help="Whehter is fixed camera or not")
 
     opt = a.parse_args()
 
@@ -272,16 +288,16 @@ def make_animation(df, fps=FPS, voronoi=False):
         (int(WIDTH // RESOLUTION), int(HEIGHT // RESOLUTION)))
 
     # visualize tracking results together with ground truth
-    visualize_tracks_on_pitch(track_res, gt)
-
-    # visualize basic result with footyviz
-    df = convert_to_footyviz(track_res)
-    # print(df.loc[0])
-    # input("...")
-    # fig, ax, dfFrame = footyviz.draw_frame(df, t=0.1, fps=25)
-    # fig, ax, dfFrame = footyviz.add_voronoi_to_fig(fig, ax, dfFrame)
-
-    # plt.show()
-    clip = make_animation(df, voronoi=True)
-    clip.ipython_display()
-    clip.write_videofile(output_file.replace('.', '_voronoi.'))
+    visualize_tracks_on_pitch(track_res, gt, opt.fixed_cam)
+
+    # # visualize basic result with footyviz
+    # df = convert_to_footyviz(track_res)
+    # # print(df.loc[0])
+    # # input("...")
+    # # fig, ax, dfFrame = footyviz.draw_frame(df, t=0.1, fps=25)
+    # # fig, ax, dfFrame = footyviz.add_voronoi_to_fig(fig, ax, dfFrame)
+
+    # # plt.show()
+    # clip = make_animation(df, voronoi=True)
+    # clip.ipython_display()
+    # clip.write_videofile(output_file.replace('.', '_voronoi.'))