1. add a sample config for velocity

2. fix traveller59#154 small problem of multi-gpu

NUSCENES-GUIDE.md

@@ -20,7 +20,7 @@
 
 2. change ```size``` and z-center in ```anchor_ranges``` in ```anchor_generator_range```.
 
-3. choose thresholds, add some print function in target assigner code and train some steps to see if the threshold is too large or too small. Then tune them.
+3. choose thresholds: use ```helper_tune_target_assigner``` to get instance count and assigned anchors count. Then tune them.
 
 4. add ```region_similarity_calculator```. If your anchors are too sparse, you need to use ```distance_similarity``` instead of ```nearest_iou_similarity``` for small classes such as pedestrian.
 

second/configs/nuscenes/all.pp.velo.config

@@ -0,0 +1,310 @@
+model: {
+  second: {
+    voxel_generator {
+      point_cloud_range : [-49.6, -49.6, -5, 49.6, 49.6, 3]
+      voxel_size : [0.2, 0.2, 8]
+      max_number_of_points_per_voxel : 40
+    }
+    voxel_feature_extractor: {
+      module_class_name: "PillarFeatureNet"
+      num_filters: [64]
+      with_distance: false
+      num_input_features: 4
+    }
+    middle_feature_extractor: {
+      module_class_name: "PointPillarsScatter"
+      downsample_factor: 1
+      num_input_features: 64
+    }
+    rpn: {
+      module_class_name: "RPNV2"
+      layer_nums: [3, 5, 5]
+      layer_strides: [2, 2, 2]
+      num_filters: [64, 128, 256]
+      upsample_strides: [1, 2, 4]
+      num_upsample_filters: [128, 128, 128]
+      use_groupnorm: false
+      num_groups: 32
+      num_input_features: 64
+    }
+    loss: {
+      classification_loss: {
+        weighted_sigmoid_focal: {
+          alpha: 0.25
+          gamma: 2.0
+          anchorwise_output: true
+        }
+      }
+      localization_loss: {
+        weighted_smooth_l1: {
+          sigma: 3.0
+          code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.5, 0.5]
+        }
+      }
+      classification_weight: 1.0
+      localization_weight: 2.0
+    }
+    num_point_features: 4 # model's num point feature should be independent of dataset
+    # Outputs
+    use_sigmoid_score: true
+    encode_background_as_zeros: true
+    encode_rad_error_by_sin: true
+
+    use_direction_classifier: true
+    direction_loss_weight: 0.2
+
+    # Loss
+    pos_class_weight: 1.0
+    neg_class_weight: 1.0
+
+    loss_norm_type: NormByNumPositives
+    # Postprocess
+    post_center_limit_range: [-59.6, -59.6, -6, 59.6, 59.6, 4]
+    use_rotate_nms: false
+    use_multi_class_nms: false
+    nms_pre_max_size: 1000
+    nms_post_max_size: 300
+    nms_score_threshold: 0.05
+    nms_iou_threshold: 0.5
+
+    box_coder: {
+      ground_box3d_coder: {
+        linear_dim: false
+        encode_angle_vector: false
+      }
+    }
+    target_assigner: {
+      anchor_generators: {
+        anchor_generator_range: {
+          sizes: [1.95017717, 4.60718145, 1.72270761] # wlh
+          anchor_ranges: [-49.6, -49.6, -0.93897414, 49.6, 49.6, -0.93897414]
+          rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
+          matched_threshold : 0.6
+          unmatched_threshold : 0.45
+          class_name: "car"
+          custom_values: [0, 0]
+        }
+        region_similarity_calculator: {
+          nearest_iou_similarity: {
+          }
+        }
+      }
+      anchor_generators: {
+        anchor_generator_range: {
+          sizes: [0.60058911, 1.68452161, 1.27192197] # wlh
+          anchor_ranges: [-49.6, -49.6, -1.03743013, 49.6, 49.6, -1.03743013]
+          rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
+          matched_threshold : 0.4
+          unmatched_threshold : 0.2
+          class_name: "bicycle"
+          custom_values: [0, 0]
+        }
+        region_similarity_calculator: {
+          nearest_iou_similarity: {
+          }
+        }
+
+      }
+      anchor_generators: {
+        anchor_generator_range: {
+          sizes: [2.94046906, 11.1885991, 3.47030982] # wlh
+          anchor_ranges: [-49.6, -49.6, -0.0715754, 49.6, 49.6, -0.0715754]
+          rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
+          matched_threshold : 0.7
+          unmatched_threshold : 0.4
+          class_name: "bus"
+          custom_values: [0, 0]
+        }
+        region_similarity_calculator: {
+          nearest_iou_similarity: {
+          }
+        }
+
+      }
+      anchor_generators: {
+        anchor_generator_range: {
+          sizes: [2.73050468, 6.38352896, 3.13312415] # wlh
+          anchor_ranges: [-49.6, -49.6, -0.08168083, 49.6, 49.6, -0.08168083]
+          rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
+          matched_threshold : 0.6
+          unmatched_threshold : 0.45
+          class_name: "construction_vehicle"
+          custom_values: [0, 0]
+        }
+        region_similarity_calculator: {
+          nearest_iou_similarity: {
+          }
+        }
+
+      }
+      anchor_generators: {
+        anchor_generator_range: {
+          sizes: [0.76279481, 2.09973778, 1.44403034] # wlh
+          anchor_ranges: [-49.6, -49.6, -0.99194854, 49.6, 49.6, -0.99194854]
+          rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
+          matched_threshold : 0.4
+          unmatched_threshold : 0.2
+          class_name: "motorcycle"
+          custom_values: [0, 0]
+        }
+        region_similarity_calculator: {
+          nearest_iou_similarity: {
+          }
+        }
+
+      }
+      anchor_generators: {
+        anchor_generator_range: {
+          sizes: [0.66344886, 0.7256437, 1.75748069] # wlh
+          anchor_ranges: [-49.6, -49.6, -0.73911038, 49.6, 49.6, -0.73911038]
+          rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
+          matched_threshold : 0.4
+          unmatched_threshold : 0.2
+          class_name: "pedestrian"
+          custom_values: [0, 0]
+        }
+        region_similarity_calculator: {
+          nearest_iou_similarity: {
+          }
+        }
+
+      }
+      anchor_generators: {
+        anchor_generator_range: {
+          sizes: [0.39694519, 0.40359262, 1.06232151] # wlh
+          anchor_ranges: [-49.6, -49.6, -1.27868911, 49.6, 49.6, -1.27868911]
+          rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
+          matched_threshold : 0.3
+          unmatched_threshold : 0.15
+          class_name: "traffic_cone"
+          custom_values: [0, 0]
+        }
+        region_similarity_calculator: {
+          nearest_iou_similarity: {
+          }
+        }
+      }
+      anchor_generators: {
+        anchor_generator_range: {
+          sizes: [2.87427237, 12.01320693, 3.81509561] # wlh
+          anchor_ranges: [-49.6, -49.6, 0.22228277, 49.6, 49.6, 0.22228277]
+          rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
+          matched_threshold : 0.6
+          unmatched_threshold : 0.45
+          class_name: "trailer"
+          custom_values: [0, 0]
+        }
+        region_similarity_calculator: {
+          nearest_iou_similarity: {
+          }
+        }
+      }
+      anchor_generators: {
+        anchor_generator_range: {
+          sizes: [2.4560939, 6.73778078, 2.73004906] # wlh
+          anchor_ranges: [-49.6, -49.6, -0.37937912, 49.6, 49.6, -0.37937912]
+          rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
+          matched_threshold : 0.6
+          unmatched_threshold : 0.45
+          class_name: "truck"
+          custom_values: [0, 0]
+        }
+        region_similarity_calculator: {
+          nearest_iou_similarity: {
+          }
+        }
+      }
+      anchor_generators: {
+        anchor_generator_range: {
+          sizes: [2.49008838, 0.48578221, 0.98297065] # wlh
+          anchor_ranges: [-49.6, -49.6, -1.27247968, 49.6, 49.6, -1.27247968]
+          rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
+          matched_threshold : 0.5
+          unmatched_threshold : 0.35
+          class_name: "barrier"
+          custom_values: [0, 0]
+        }
+        region_similarity_calculator: {
+          nearest_iou_similarity: {
+          }
+        }
+      }
+      sample_positive_fraction : -1
+      sample_size : 512
+    }
+  }
+}
+
+train_input_reader: {
+  dataset: {
+    dataset_class_name: "NuScenesDatasetD8Velo"
+    kitti_info_path: "/media/yy/960evo/datasets/nuscene/v1.0-trainval/infos_train.pkl"
+    kitti_root_path: "/media/yy/960evo/datasets/nuscene/v1.0-trainval"
+  }
+
+  batch_size: 3
+  preprocess: {
+    max_number_of_voxels: 30000
+    shuffle_points: false
+    num_workers: 4
+    groundtruth_localization_noise_std: [0, 0, 0]
+    groundtruth_rotation_uniform_noise: [0, 0]
+    # groundtruth_localization_noise_std: [0.25, 0.25, 0.25]
+    # groundtruth_rotation_uniform_noise: [-0.15707963267, 0.15707963267]
+    # global_rotation_uniform_noise: [-0.78539816, 0.78539816]
+    global_rotation_uniform_noise: [0, 0]
+    global_scaling_uniform_noise: [0.95, 1.05]
+    global_random_rotation_range_per_object: [0, 0]
+    global_translate_noise_std: [0.2, 0.2, 0.2]
+    anchor_area_threshold: -1
+    remove_points_after_sample: false
+    groundtruth_points_drop_percentage: 0.0
+    groundtruth_drop_max_keep_points: 15
+    remove_unknown_examples: false
+    remove_environment: false
+    database_sampler {
+    }
+  }
+}
+
+train_config: {
+  optimizer: {
+    adam_optimizer: {
+      learning_rate: {
+        one_cycle: {
+          lr_max: 3e-3
+          moms: [0.95, 0.85]
+          div_factor: 10.0
+          pct_start: 0.4
+        }
+      }
+      weight_decay: 0.01
+    }
+    fixed_weight_decay: true
+    use_moving_average: false
+  }
+  steps: 58650 # 1173 * 50 (3517 // 3 + 1)
+  steps_per_eval: 5865 # 1173 * 5
+  save_checkpoints_secs : 1800 # half hour
+  save_summary_steps : 10
+  enable_mixed_precision: false
+  loss_scale_factor: -1
+  clear_metrics_every_epoch: true
+}
+
+eval_input_reader: {
+  dataset: {
+    dataset_class_name: "NuScenesDataset"
+    kitti_info_path: "/media/yy/960evo/datasets/nuscene/v1.0-trainval/infos_val.pkl"
+    kitti_root_path: "/media/yy/960evo/datasets/nuscene/v1.0-trainval"
+  }
+  batch_size: 1
+
+  preprocess: {
+    max_number_of_voxels: 40000
+    shuffle_points: false
+    num_workers: 3
+    anchor_area_threshold: -1
+    remove_environment: false
+  }
+}

second/data/preprocess.py

@@ -23,7 +23,7 @@ def merge_second_batch(batch_list):
     ret = {}
     for key, elems in example_merged.items():
         if key in [
-                'voxels', 'num_points', 'num_gt', 'gt_boxes', 'voxel_labels'
+                'voxels', 'num_points', 'num_gt', 'voxel_labels', 'gt_names', 'gt_classes', 'gt_boxes'
         ]:
             ret[key] = np.concatenate(elems, axis=0)
         elif key == 'metadata':
@@ -77,8 +77,11 @@ def merge_second_batch_multigpu(batch_list):
                     coor, ((0, 0), (1, 0)), mode='constant', constant_values=i)
                 coors.append(coor_pad)
             ret[key] = np.stack(coors, axis=0)
+        elif key in ['gt_names', 'gt_classes', 'gt_boxes']:
+            continue
         else:
             ret[key] = np.stack(elems, axis=0)
+
     return ret
 
 
@@ -345,6 +348,7 @@ def prep_pointcloud(input_dict,
     metrics["prep_time"] = time.time() - t
     if not training:
         return example
+    example["gt_names"] = gt_dict["gt_names"]
     # voxel_labels = box_np_ops.assign_label_to_voxel(gt_boxes, coordinates,
     #                                                 voxel_size, coors_range)
     if create_targets:
@@ -367,15 +371,17 @@ def prep_pointcloud(input_dict,
             matched_thresholds=matched_thresholds,
             unmatched_thresholds=unmatched_thresholds,)
         """
-        # boxes_lidar = gt_dict["gt_boxes"]
-        # bev_map = simplevis.nuscene_vis(points, boxes_lidar, gt_dict["gt_names"])
-        # assigned_anchors = anchors[targets_dict['labels'] > 0]
-        # ignored_anchors = anchors[targets_dict['labels'] == -1]
-        # bev_map = simplevis.draw_box_in_bev(bev_map, [-50, -50, 3, 50, 50, 1], ignored_anchors, [128, 128, 128], 2)
-        # bev_map = simplevis.draw_box_in_bev(bev_map, [-50, -50, 3, 50, 50, 1], assigned_anchors, [255, 0, 0])
-        # cv2.imshow('anchors', bev_map)
-        # cv2.waitKey(0)
-
+        """
+        if "trailer" in gt_dict["gt_names"]:
+            boxes_lidar = gt_dict["gt_boxes"]
+            bev_map = simplevis.nuscene_vis(points, boxes_lidar, gt_dict["gt_names"])
+            assigned_anchors = anchors[targets_dict['labels'] > 0]
+            ignored_anchors = anchors[targets_dict['labels'] == -1]
+            bev_map = simplevis.draw_box_in_bev(bev_map, [-50, -50, 3, 50, 50, 1], ignored_anchors, [128, 128, 128], 2)
+            bev_map = simplevis.draw_box_in_bev(bev_map, [-50, -50, 3, 50, 50, 1], assigned_anchors, [255, 0, 0])
+            cv2.imshow('anchors', bev_map)
+            cv2.waitKey(0)
+        """
         example.update({
             'labels': targets_dict['labels'],
             'reg_targets': targets_dict['bbox_targets'],