Merge pull request wayveai#6 from wayveai/camera-ready

Update configs

README.md

@@ -11,7 +11,7 @@ described in:
 [‪Vijay Badrinarayanan](https://sites.google.com/site/vijaybacademichomepage/home), 
 [Roberto Cipolla](https://mi.eng.cam.ac.uk/~cipolla/index.htm) and [Alex Kendall](https://alexgkendall.com/)  
 >
-> [preprint (2021)](https://arxiv.org/abs/2104.10490)<br/>
+> [ICCV 2021 (Oral)](https://arxiv.org/abs/2104.10490)<br/>
 > [Blog post](https://wayve.ai/blog/fiery-future-instance-prediction-birds-eye-view)
 
 <p align="center">
@@ -29,7 +29,7 @@ If you find our work useful, please consider citing:
   title     = {{FIERY}: Future Instance Segmentation in Bird's-Eye view from Surround Monocular Cameras},
   author    = {Anthony Hu and Zak Murez and Nikhil Mohan and Sofía Dudas and 
                Jeffrey Hawke and Vijay Badrinarayanan and Roberto Cipolla and Alex Kendall},
-  booktitle = {arXiv preprint},
+  booktitle = {Proceedings of the International Conference on Computer Vision ({ICCV})},
   year = {2021}
 }
 ```
@@ -57,13 +57,15 @@ them to an `output_vis` folder.
 
 All the configs are in the folder `fiery/configs`
 
-| Config       | Dataset | Past context | Future horizon | BEV size | IoU  | VPQ|
+| Config and weights      | Dataset | Past context | Future horizon | BEV size | IoU  | VPQ|
 |--------------|---------|-----------------------|----------------|----------|------|----|
 | [`baseline.yml`](https://github.com/wayveai/fiery/releases/download/v1.0/fiery.ckpt) | NuScenes | 1.0s | 2.0s | 100mx100m (50cm res.) | 36.7 | 29.9 |
-| [`lyft/baseline.yml`](https://github.com/wayveai/fiery/releases/download/v1.0/lyft_fiery.ckpt) | Lyft | 0.8s | 2.0s | 100mx100m (50cm res.) | 36.3 | 29.2 |
-| [`literature/pon_setting.yml`](https://github.com/wayveai/fiery/releases/download/v1.0/pon_setting.ckpt) | NuScenes| 1.0s | 0.0s | 100mx50m (25cm res.) | 40.0 | - |
-| [`literature/lift_splat_setting.yml`](https://github.com/wayveai/fiery/releases/download/v1.0/lift_splat_setting.ckpt) | NuScenes | 1.0s | 0.0s | 100mx100m (50cm res.) | 37.8 | - |
-| [`literature/fishing_setting.yml`](https://github.com/wayveai/fiery/releases/download/v1.0/fishing_setting.ckpt) | NuScenes | 1.0s | 2.0s | 32.0mx19.2m (10cm res.) | 59.4 | - |
+| [`lyft/baseline.yml`](https://github.com/wayveai/fiery/releases/download/v1.0/lyft_fiery.ckpt) | Lyft | 0.8s | 2.0s| 100mx100m (50cm res.) | 36.3 | 29.2 |
+| [`literature/static_pon_setting.yml`](https://github.com/wayveai/fiery/releases/download/v1.0/static_pon_setting.ckpt) | NuScenes| 0.0s | 0.0s | 100mx50m (25cm res.) | 37.7| - |
+| [`literature/pon_setting.yml`](https://github.com/wayveai/fiery/releases/download/v1.0/pon_setting.ckpt) | NuScenes| 1.0s | 0.0s | 100mx50m (25cm res.) |39.9 | - |
+| [`literature/static_lss_setting.yml`](https://github.com/wayveai/fiery/releases/download/v1.0/static_lift_splat_setting.ckpt) | NuScenes | 0.0s | 0.0s | 100mx100m (50cm res.) | 35.8 | - |
+| [`literature/lift_splat_setting.yml`](https://github.com/wayveai/fiery/releases/download/v1.0/lift_splat_setting.ckpt) | NuScenes | 1.0s | 0.0s | 100mx100m (50cm res.) | 38.2 | - |
+| [`literature/fishing_setting.yml`](https://github.com/wayveai/fiery/releases/download/v1.0/fishing_setting.ckpt) | NuScenes | 1.0s | 2.0s | 32.0mx19.2m (10cm res.) | 57.6 | - |
 
 
 ## 🏊 Training
@@ -73,6 +75,6 @@ To train the model from scratch on NuScenes:
 To train on single GPU add the flag `GPUS [0]`, and to change the batch size use the flag `BATCHSIZE ${DESIRED_BATCHSIZE}`.
 
 ## 🙌 Credits
-Big thanks to Piotr Sokólski ([@pyetras](https://github.com/pyetras)) for the panoptic metric implementation, and to 
-Hannes Liik ([@hannesliik](https://github.com/hannesliik)) for the awesome future trajectory 
-visualisation on the ground plane.
+Big thanks to Giulio D'Ippolito ([@gdippolito](https://github.com/gdippolito)) for the technical help on the gpu 
+servers, Piotr Sokólski ([@pyetras](https://github.com/pyetras)) for the panoptic metric implementation, and to Hannes Liik ([@hannesliik](https://github.com/hannesliik)) 
+for the awesome future trajectory visualisation on the ground plane.

environment.yml

@@ -20,3 +20,4 @@ dependencies:
     - nuscenes-devkit==1.1.0
     - lyft-dataset-sdk==0.0.8
     - opencv-python==4.5.1.48
+    - moviepy==1.0.3

fiery/config.py

@@ -47,14 +47,14 @@ def convert_to_dict(self):
 _C.PRETRAINED.PATH = ''
 
 _C.DATASET = CN()
-_C.DATASET.DATAROOT = './datasets/nuscenes/'
+_C.DATASET.DATAROOT = './nuscenes/'
 _C.DATASET.VERSION = 'trainval'
 _C.DATASET.NAME = 'nuscenes'
 _C.DATASET.IGNORE_INDEX = 255  # Ignore index when creating flow/offset labels
-_C.DATASET.FILTER_INVISIBLE_VEHICLES = True
+_C.DATASET.FILTER_INVISIBLE_VEHICLES = True  # Filter vehicles that are not visible from the cameras
 
 _C.TIME_RECEPTIVE_FIELD = 3  # how many frames of temporal context (1 for single timeframe)
-_C.N_FUTURE_FRAMES = 3  # how many time steps into the future to predict
+_C.N_FUTURE_FRAMES = 4  # how many time steps into the future to predict
 
 _C.IMAGE = CN()
 _C.IMAGE.FINAL_DIM = (224, 480)
@@ -83,7 +83,7 @@ def convert_to_dict(self):
 _C.MODEL.TEMPORAL_MODEL.START_OUT_CHANNELS = 64
 _C.MODEL.TEMPORAL_MODEL.EXTRA_IN_CHANNELS = 0
 _C.MODEL.TEMPORAL_MODEL.INBETWEEN_LAYERS = 0
-_C.MODEL.TEMPORAL_MODEL.PYRAMID_POOLING = False
+_C.MODEL.TEMPORAL_MODEL.PYRAMID_POOLING = True
 _C.MODEL.TEMPORAL_MODEL.INPUT_EGOPOSE = True
 
 _C.MODEL.DISTRIBUTION = CN()

fiery/configs/baseline.yml

@@ -1,6 +1,4 @@
-_BASE_: 'single_timeframe.yml'
-
-TAG: 'fiery'
+TAG: 'baseline'
 
 GPUS: [0, 1, 2, 3]
 

fiery/configs/debug_baseline.yml

@@ -2,8 +2,8 @@ _BASE_: 'baseline.yml'
 
 TAG: 'debug'
 
-BATCHSIZE: 1
 EPOCHS: 2
+BATCHSIZE: 1
 GPUS: [0]
 LOGGING_INTERVAL: 10
 

fiery/configs/literature/fishing_setting.yml

@@ -2,7 +2,7 @@ _BASE_: '../baseline.yml'
 
 TAG: 'fishing_setting'
 
-BATCHSIZE: 2
+BATCHSIZE: 3
 
 DATASET:
   FILTER_INVISIBLE_VEHICLES: False

fiery/configs/literature/lift_splat_setting.yml

@@ -1,6 +1,6 @@
-_BASE_: '../single_timeframe.yml'
+_BASE_: '../temporal_single_timeframe.yml'
 
-TAG: 'lift_splat_setting'
+TAG: 'temporal_lift_splat_setting'
 
 DATASET:
   FILTER_INVISIBLE_VEHICLES: False

fiery/configs/literature/pon_setting.yml

@@ -1,6 +1,9 @@
-_BASE_: 'lift_splat_setting.yml'
+_BASE_: '../temporal_single_timeframe.yml'
 
-TAG: 'pyramid_occupancy_network_setting'
+TAG: 'temporal_pon_setting'
+
+DATASET:
+  FILTER_INVISIBLE_VEHICLES: False
 
 LIFT:
   X_BOUND: [-50.0, 50.0, 0.25]

fiery/configs/literature/static_lss_setting.yml

@@ -0,0 +1,6 @@
+_BASE_: '../single_timeframe.yml'
+
+TAG: 'lift_splat_setting'
+
+DATASET:
+  FILTER_INVISIBLE_VEHICLES: False

fiery/configs/literature/static_pon_setting.yml

@@ -0,0 +1,7 @@
+_BASE_: 'lift_splat_setting.yml'
+
+TAG: 'pyramid_occupancy_network_setting'
+
+LIFT:
+  X_BOUND: [-50.0, 50.0, 0.25]
+  Y_BOUND: [-25.0, 25.0, 0.25]

fiery/configs/lyft/baseline.yml

@@ -1,6 +1,6 @@
 _BASE_: '../baseline.yml'
 
-TAG: 'lyft_baseline_latest'
+TAG: 'lyft_baseline'
 
 GPUS: [0, 1, 2, 3]
 
@@ -16,8 +16,5 @@ IMAGE:
   W: 1920
   RESIZE_SCALE: 0.25
 
-PRETRAINED:
-  PATH: ''
-
 MODEL:
   SUBSAMPLE: True

fiery/configs/single_timeframe.yml

@@ -1,6 +1,6 @@
 TAG: 'single_timeframe_model'
 
-GPUS: [0, 1, 2, 3]
+GPUS: [0, 1]
 
 BATCHSIZE: 8
 
@@ -21,4 +21,4 @@ INSTANCE_FLOW:
 OPTIMIZER:
   LR: 1e-3
 
-N_WORKERS: 20
+N_WORKERS: 10

fiery/configs/temporal_single_timeframe.yml

@@ -0,0 +1,14 @@
+_BASE_: 'single_timeframe.yml'
+
+TAG: 'temporal_single_timeframe'
+
+BATCHSIZE: 4
+PRECISION: 16
+
+TIME_RECEPTIVE_FIELD: 3
+
+MODEL:
+  BN_MOMENTUM: 0.05
+  TEMPORAL_MODEL:
+    NAME: 'temporal_block'
+    INPUT_EGOPOSE: True

fiery/data.py

@@ -426,7 +426,7 @@ def __getitem__(self, index):
         return data
 
 
-def prepare_dataloaders(cfg):
+def prepare_dataloaders(cfg, return_dataset=False):
     version = cfg.DATASET.VERSION
     train_on_training_data = True
 
@@ -445,9 +445,9 @@ def prepare_dataloaders(cfg):
     traindata = FuturePredictionDataset(nusc, train_on_training_data, cfg)
     valdata = FuturePredictionDataset(nusc, False, cfg)
 
-    if version == 'mini':
+    if cfg.DATASET.VERSION == 'mini':
         traindata.indices = traindata.indices[:10]
-        valdata.indices = valdata.indices[30:36]
+        valdata.indices = valdata.indices[:10]
 
     nworkers = cfg.N_WORKERS
     trainloader = torch.utils.data.DataLoader(
@@ -456,4 +456,7 @@ def prepare_dataloaders(cfg):
     valloader = torch.utils.data.DataLoader(
         valdata, batch_size=cfg.BATCHSIZE, shuffle=False, num_workers=nworkers, pin_memory=True, drop_last=False)
 
-    return trainloader, valloader
+    if return_dataset:
+        return trainloader, valloader, traindata, valdata
+    else:
+        return trainloader, valloader