train and test on HEVI is ready

Author: MoonBlvd

README.md

@@ -1,6 +1,30 @@
-# Spatio-Temporal Anomaly Detection in First-Person Videos
+# Unsupervised Traffic Accident Detection in First-Person Videos
 
-Created by Yu Yao and Mingze Xu for IROS 2019
+*Yu Yao, Mingze Xu, Yuchen Wang, David Crandall and Ella Atkins*
+
+This repo contains the code for our [paper](https://arxiv.org/pdf/1903.00618.pdf) on unsupervised traffic accident detection.
+
+:boom: The full code will be released upon the acceptance of our paper.
+
+:boom: So far we have released the pytorch implementation of our ICRA paper [*Egocentric Vision-based Future Vehicle Localization for Intelligent Driving Assistance Systems*](https://arxiv.org/pdf/1809.07408.pdf), which is an important building block for the traffic accident detection. The original project repo is https://github.com/MoonBlvd/fvl-ICRA2019
+
+## Future Object Localization
+To train the model, run:
+
+	python train_fol.py --load_config YOUR_CONFIG_FILE
+
+To test the model, run:
+
+	python test_fol.py --load_config YOUR_CONFIG_FILE
+ 
+ An example of the config file can be found in ```config/fol_ego_train.yaml```
+
+#### evaluation result
+Note that we have only evaluated the model performance with prediction horizon 0.5 seconds. We are working on proving the 1 second and 2 seconds results.
+
+|     Model      | pred horizon | FDE  | ADE | FIOU |
+|:--------------:|--------------|------|-----|------|
+| FOL + Ego pred | 0.5 sec      | 10.9 | 6.6 | 0.95 |
 
 ## Run detection
 Go to Mask-RCNN root directory run:

config/config.py

@@ -1,6 +1,9 @@
 import yaml
 import argparse
 
+# class Config()
+
+
 def visualize_config(args):
     """
     Visualize the configuration on the terminal to check the state

config/fol_ego_test.yaml

@@ -0,0 +1,60 @@
+# Directories arguments
+data_root: "/media/DATA/HEVI_dataset/fol_data"
+ego_data_root: "/media/DATA/HEVI_dataset/ego_motion"
+checkpoint_dir: "checkpoints/fol_ego_checkpoints"
+
+best_fol_model: 'checkpoints/fol_ego_checkpoints/fol_epoch_078_loss_0.0013.pt'
+best_ego_pred_model: 'checkpoints/fol_ego_checkpoints/ego_pred_epoch_078_loss_0.0016.pt'
+
+test_dataset: "taiwan_sa" #"A3D" #"taiwan_sa"
+test_root: #"../data/taiwan_sa/testing" #"/media/DATA/A3D" #"/media/DATA/VAD_datasets/taiwan_sa/testing" AnAnAccident_Detection_Dataset
+label_file: '../data/A3D/A3D_labels.pkl'
+
+# dataset arguments
+seed_max: 5
+segment_len: 16
+
+# training parameters
+nb_fol_epoch: 100
+nb_ego_pred_epoch: 200
+lr: 0.0001
+
+lambda_fol: 1
+lambda_ego: 1
+device: 'cuda'
+
+# fol model parameters
+pred_timesteps: 5
+input_embed_size: 512
+flow_enc_size: 512
+box_enc_size: 512
+with_ego: True
+
+enc_hidden_size: 512 # no use
+enc_concat_type: "average"
+predictor_input_size: 512
+dec_hidden_size: 512
+pred_dim: 4
+
+
+# ego_pred model parameters
+ego_embed_size: 128
+ego_enc_size: 128
+ego_dec_size: 128
+ego_pred_input_size: 128
+ego_dim: 3
+
+# dataloader parameters
+batch_size: 1
+shuffle: False
+num_workers: 0
+
+# image parameters
+H: 720
+W: 1280
+channels: 3
+
+flow_roi_size: [5,5,2]
+
+# Anomaly detection parameters
+max_age: 10

config/fol_config.yaml config/fol_ego_train.yaml

@@ -1,9 +1,9 @@
 # Directories arguments
 data_root: "/media/DATA/HEVI_dataset/fol_data"
 ego_data_root: "/media/DATA/HEVI_dataset/ego_motion"
-checkpoint_dir: "/home/brianyao/Documents/stad2019iros-pytorch/checkpoints/fol_ego_checkpoints"
+checkpoint_dir: "checkpoints/fol_ego_checkpoints"
 
-best_ego_pred_model: "/home/brianyao/Documents/stad2019iros-pytorch/checkpoints/ego_pred_checkpoints/epoch_080_loss_0.001.pt"
+best_ego_pred_model: "checkpoints/ego_pred_checkpoints/epoch_080_loss_0.001.pt"
 test_dataset: "taiwan_sa" #"A3D" #"taiwan_sa"
 test_root: #"../data/taiwan_sa/testing" #"/media/DATA/A3D" #"/media/DATA/VAD_datasets/taiwan_sa/testing" AnAnAccident_Detection_Dataset
 label_file: '../data/A3D/A3D_labels.pkl'

lib/utils/data_prep_utils.py

@@ -6,6 +6,19 @@
 import copy
 import pickle as pkl
 
+def cxcywh_to_x1y1x2y2(boxes):
+    '''
+    Params:
+        boxes:(Cx, Cy, w, h)
+    Returns:
+        (x1, y1, x2, y2 or tlbr
+    '''
+    new_boxes = np.zeros_like(boxes)
+    new_boxes[...,0] = boxes[...,0] - boxes[...,2]/2
+    new_boxes[...,1] = boxes[...,1] - boxes[...,3]/2
+    new_boxes[...,2] = boxes[...,0] + boxes[...,2]/2
+    new_boxes[...,3] = boxes[...,1] + boxes[...,3]/2
+    return new_boxes
 
 def bbox_normalize(bbox,W=1280,H=640):
     '''
@@ -32,10 +45,10 @@ def bbox_denormalize(bbox,W=1280,H=640):
         bbox: [cx, cy, w, h] with size (times, 4), value from 0 to W or H
     '''
     new_bbox = copy.deepcopy(bbox)
-    new_bbox[:,0] *= W
-    new_bbox[:,1] *= H
-    new_bbox[:,2] *= W
-    new_bbox[:,3] *= H
+    new_bbox[..., 0] *= W
+    new_bbox[..., 1] *= H
+    new_bbox[..., 2] *= W
+    new_bbox[..., 3] *= H
 
     return new_bbox
 

lib/utils/fol_dataloader.py

@@ -11,6 +11,8 @@
 class HEVIDataset(data.Dataset):
     def __init__(self, args, phase):
         '''
+        HEV-I dataset object. Contains bbox, flow and ego motion.
+        
         Params:
             args: arguments passed from main file
             phase: 'train' or 'val'

lib/utils/train_val_utils.py

@@ -32,8 +32,6 @@ def train_fol(epoch, model, optimizer, train_gen, verbose=True):
             optimizer.step()
 
             #write summery for tensorboardX
-            # writer.add_scalar('data/train_loss', object_pred_loss, n_iters)
-            # n_iters += 1
             if verbose and batch_idx % 100 == 0:
                 print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                     epoch, batch_idx * len(data), len(train_gen.dataset),
@@ -228,8 +226,6 @@ def val_fol_ego(epoch, args, fol_model, ego_pred_model, val_gen, verbose=True):
             fol_loss += rmse_loss_fol(fol_predictions, target_bbox).item()
             ego_pred_loss += rmse_loss_fol(ego_predictions, target_ego_motion).item()
 
-    # total_val_loss = fol_loss + ego_pred_loss
-    # avg_val_loss = total_val_loss/len(val_gen.dataset)
     fol_loss /= len(val_gen.dataset)
     ego_pred_loss /= len(val_gen.dataset)
     avg_val_loss = fol_loss + ego_pred_loss

test_fol.py

@@ -2,16 +2,119 @@
 import os 
 import numpy as np
 import time
+from tqdm import tqdm
 
 import torch
 from torch import nn, optim
-from torch.nn import functional as F
 from torch.utils import data
 from torchsummaryX import summary
 
-from lib.utils.train_val_utils import train_fol_ego, val_fol_ego
+from lib.utils.train_val_utils import val_fol_ego
 from lib.models.rnn_ed import FolRNNED, EgoRNNED
 from lib.utils.fol_dataloader import HEVIDataset
 from config.config import * 
 from lib.ego_motion_tracker import EgoTracker
-from lib.object_tracker import ObjTracker, AllTrackers
+from lib.object_tracker import ObjTracker, AllTrackers
+from lib.utils.data_prep_utils import bbox_denormalize, cxcywh_to_x1y1x2y2
+from lib.utils.eval_utils import compute_IOU
+
+print("Cuda available: ", torch.cuda.is_available())
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def test_fol_ego(fol_model, ego_pred_model, test_gen):
+    '''
+    Validate future vehicle localization module 
+    Params:
+        fol_model: The fol model as nn.Module
+        ego_pred_model: the ego motion prediction model as nn.Module
+        test_gen: test data generator
+    Returns:
+        
+    '''
+    fol_model.eval() # Sets the module in training mode.
+    ego_pred_model.eval()
+
+    fol_loss = 0
+    ego_pred_loss = 0
+    loader = tqdm(test_gen, total=len(test_gen))
+
+    FDE = 0
+    ADE = 0
+    FIOU = 0
+    with torch.set_grad_enabled(False):
+        for batch_idx, data in enumerate(loader):
+            input_bbox, input_flow, input_ego_motion, target_bbox, target_ego_motion = data
+
+            # run forward
+            ego_predictions = ego_pred_model(input_ego_motion)
+            fol_predictions = fol_model(input_bbox, input_flow, ego_predictions)
+            
+            # convert to numpy array, use [0] since batchsize if 1 for test
+            # the prediction is the box changes
+            ego_predictions = ego_predictions.to('cpu').numpy()[0]
+            fol_predictions = fol_predictions.to('cpu').numpy()[0] 
+            input_bbox = input_bbox.to('cpu').numpy()[0]
+            target_bbox = target_bbox.to('cpu').numpy()[0]
+
+            # compute FDE, ADE and FIOU metrics used in FVL2019ICRA paper
+            input_bbox = np.expand_dims(input_bbox, axis=1)
+            target_bbox = input_bbox + target_bbox
+            fol_predictions = input_bbox + fol_predictions
+
+            input_bbox = bbox_denormalize(input_bbox, W=1280, H=640)
+            fol_predictions = bbox_denormalize(fol_predictions, W=1280, H=640)
+            target_bbox = bbox_denormalize(target_bbox, W=1280, H=640)
+            
+            fol_predictions_xyxy = cxcywh_to_x1y1x2y2(fol_predictions)
+            target_bbox_xyxy = cxcywh_to_x1y1x2y2(target_bbox)
+
+            # print(fol_predictions_xyxy[15,...])
+            # print(target_bbox_xyxy[15,...])
+
+            ADE += np.mean(np.sqrt(np.sum((target_bbox_xyxy[:,:,:2] - fol_predictions_xyxy[:,:,:2]) ** 2, axis=-1)))
+            FDE += np.mean(np.sqrt(np.sum((target_bbox_xyxy[:,-1,:2] - fol_predictions_xyxy[:,-1,:2]) ** 2, axis=-1)))
+            tmp_FIOU = []
+            for i in range(target_bbox_xyxy.shape[0]):
+                tmp_FIOU.append(compute_IOU(target_bbox_xyxy[i,-1,:], fol_predictions_xyxy[i,-1,:]))
+            FIOU += np.mean(tmp_FIOU)
+    print("FDE: %4f;    ADE: %4f;   FIOU: %4f" % (FDE, ADE, FIOU))
+    ADE /= len(test_gen.dataset)
+    FDE /= len(test_gen.dataset)
+    FIOU /= len(test_gen.dataset)
+    print("FDE: %4f;    ADE: %4f;   FIOU: %4f" % (FDE, ADE, FIOU))
+
+def main(args):
+    # initialize model
+    fol_model = FolRNNED(args).to(device)
+    fol_model.load_state_dict(torch.load(args.best_fol_model))
+
+    if args.with_ego:
+        print("Initializing pre-trained ego motion predictor...")
+        ego_pred_model = EgoRNNED(args).to(device)
+        ego_pred_model.load_state_dict(torch.load(args.best_ego_pred_model))
+        print("Pre-trained ego_motion predictor done!")
+
+    # initialize datasets
+    print("Initializing test dataset...")
+    dataloader_params ={
+            "batch_size": args.batch_size,
+            "shuffle": args.shuffle,
+            "num_workers": args.num_workers
+        }
+
+    test_set = HEVIDataset(args, 'val')
+    print("Number of test samples:", test_set.__len__())
+    test_gen = data.DataLoader(test_set, **dataloader_params)
+
+    input_bbox, input_flow, input_ego_motion, target_bbox, target_ego_motion = test_set.__getitem__(1)
+    print("input shape: ", input_bbox.shape)
+    print("target shape: ", target_bbox.shape)
+
+    test_fol_ego(fol_model, ego_pred_model, test_gen)
+
+
+if __name__=='__main__':
+    # load args
+    args = parse_args()
+    main(args)

train_fol.py

@@ -47,8 +47,8 @@
 # print("Number of training samples:", train_set.__len__())
 
 val_set = HEVIDataset(args, 'val')
-val_gen = data.DataLoader(val_set, **dataloader_params)
 print("Number of validation samples:", val_set.__len__())
+val_gen = data.DataLoader(val_set, **dataloader_params)
 
 # print model summary
 if args.with_ego:
@@ -120,7 +120,7 @@
         print("Saving checkpoints: " + saved_fol_model_name + ' and ' + saved_ego_pred_model_name)
         if not os.path.isdir(args.checkpoint_dir):
             os.mkdir(args.checkpoint_dir)
-            
+
         torch.save(fol_model.state_dict(), os.path.join(args.checkpoint_dir, saved_fol_model_name))
         torch.save(ego_pred_model.state_dict(), os.path.join(args.checkpoint_dir, saved_ego_pred_model_name))