fix test bug

experiments/scripts/FP_Net_end2end.sh

@@ -50,7 +50,7 @@ echo Logging output to "$LOG"
 
 time ./tools/train_net.py --gpu ${GPU_ID} \
   --solver models/${PT_DIR}/${NET}/FP_Net_end2end/solver.prototxt \
-  --weights data/pretrained_model/ResNet50.v2.caffemodel  \
+  --weights data/pretrained_model/ResNet50.v2.caffemodel \
   --imdb ${TRAIN_IMDB} \
   --iters ${ITERS} \
   --cfg experiments/cfgs/FP_Net_end2end.yml \

lib/fast_rcnn/bbox_transform.py

@@ -43,7 +43,7 @@ def bbox_transform_inv(boxes, deltas):
     dw = deltas[:, 2::4]
     dh = deltas[:, 3::4]
 
-
+    
     pred_ctr_x = dx * widths[:, np.newaxis] + ctr_x[:, np.newaxis]
     pred_ctr_y = dy * heights[:, np.newaxis] + ctr_y[:, np.newaxis]
     pred_w = np.exp(dw) * widths[:, np.newaxis]

lib/fast_rcnn/config.py

@@ -45,7 +45,7 @@
 __C.TRAIN.IMS_PER_BATCH = 2
 
 # Minibatch size (number of regions of interest [ROIs])
-__C.TRAIN.BATCH_SIZE = 128
+__C.TRAIN.BATCH_SIZE = 256
 
 # Fraction of minibatch that is labeled foreground (i.e. class > 0)
 __C.TRAIN.FG_FRACTION = 0.25
@@ -56,7 +56,7 @@
 # Overlap threshold for a ROI to be considered background (class = 0 if
 # overlap in [LO, HI))
 __C.TRAIN.BG_THRESH_HI = 0.5
-__C.TRAIN.BG_THRESH_LO = 0.0
+__C.TRAIN.BG_THRESH_LO = 0.001
 
 # Use horizontally-flipped images during training?
 __C.TRAIN.USE_FLIPPED = True
@@ -69,7 +69,7 @@
 __C.TRAIN.BBOX_THRESH = 0.7
 
 # Iterations between snapshots
-__C.TRAIN.SNAPSHOT_ITERS = 5000
+__C.TRAIN.SNAPSHOT_ITERS = 10000
 
 # solver.prototxt specifies the snapshot path prefix, this adds an optional
 # infix to yield the path: <prefix>[_<infix>]_iters_XYZ.caffemodel
@@ -85,7 +85,7 @@
 __C.TRAIN.BBOX_INSIDE_WEIGHTS = (1.0, 1.0, 1.0, 1.0)
 # Normalize the targets using "precomputed" (or made up) means and stdevs
 # (BBOX_NORMALIZE_TARGETS must also be True)
-__C.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED = False
+__C.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED = True
 __C.TRAIN.BBOX_NORMALIZE_MEANS = (0.0, 0.0, 0.0, 0.0)
 __C.TRAIN.BBOX_NORMALIZE_STDS = (0.1, 0.1, 0.2, 0.2)
 
@@ -160,7 +160,7 @@
 ## Number of top scoring boxes to keep before apply NMS to RPN proposals
 __C.TEST.RPN_PRE_NMS_TOP_N = 6000
 ## Number of top scoring boxes to keep after applying NMS to RPN proposals
-__C.TEST.RPN_POST_NMS_TOP_N = 2000
+__C.TEST.RPN_POST_NMS_TOP_N = 500
 # Proposal height and width both need to be greater than RPN_MIN_SIZE (at orig image scale)
 __C.TEST.RPN_MIN_SIZE = 16
 

lib/fast_rcnn/test.py

@@ -115,7 +115,7 @@ def _get_blobs(im, rois):
         blobs['rois'] = _get_rois_blob(rois, im_scale_factors)
     return blobs, im_scale_factors
 
-def im_detect(net, im, boxes=None):
+def im_detect(net, im, boxes=None,num_classes=21):
     """Detect object classes in an image given object proposals.
 
     Arguments:
@@ -167,7 +167,10 @@ def im_detect(net, im, boxes=None):
         assert len(im_scales) == 1, "Only single-image batch implemented"
         rois = net.blobs['rois'].data.copy()
         # unscale back to raw image space
-        boxes = rois[:, 1:5] / im_scales[0]
+        boxes = rois[:, 1:5] 
+        index= np.where(np.sum(boxes,axis=1)!=0)[0]
+        boxes = boxes[index,:]/ im_scales[0]
+
 
     if cfg.TEST.SVM:
         # use the raw scores before softmax under the assumption they
@@ -176,10 +179,24 @@ def im_detect(net, im, boxes=None):
     else:
         # use softmax estimated probabilities
         scores = blobs_out['cls_prob']
+        scores = scores[index]
+
+      #  print scores[0:10]
 
     if cfg.TEST.BBOX_REG:
         # Apply bounding-box regression deltas
         box_deltas = blobs_out['bbox_pred']
+
+        box_deltas = box_deltas[index,:]
+
+        if cfg.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED:
+            means = np.tile(
+                    np.array(cfg.TRAIN.BBOX_NORMALIZE_MEANS), (num_classes, 1)).ravel()
+            stds = np.tile(
+                    np.array(cfg.TRAIN.BBOX_NORMALIZE_STDS), (num_classes, 1)).ravel()
+      #  Optionally normalize targets by a precomputed mean and stdev
+            box_deltas = box_deltas * stds + means
+      #  print boxes.shape,box_deltas.shape
         pred_boxes = bbox_transform_inv(boxes, box_deltas)
         pred_boxes = clip_boxes(pred_boxes, im.shape)
     else:
@@ -190,6 +207,7 @@ def im_detect(net, im, boxes=None):
         # Map scores and predictions back to the original set of boxes
         scores = scores[inv_index, :]
         pred_boxes = pred_boxes[inv_index, :]
+    pred_boxes = pred_boxes
 
     return scores, pred_boxes
 
@@ -236,6 +254,7 @@ def apply_nms(all_boxes, thresh):
 
 def test_net(net, imdb, max_per_image=100, thresh=0.05, vis=False):
     """Test a Fast R-CNN network on an image database."""
+
     num_images = len(imdb.image_index)
     # all detections are collected into:
     #    all_boxes[cls][image] = N x 5 array of detections in
@@ -265,43 +284,45 @@ def test_net(net, imdb, max_per_image=100, thresh=0.05, vis=False):
 
         im = cv2.imread(imdb.image_path_at(i))
         _t['im_detect'].tic()
-        scores, boxes = im_detect(net, im, box_proposals)
+        scores, boxes = im_detect(net, im, box_proposals,imdb.num_classes)
         _t['im_detect'].toc()
 
         _t['misc'].tic()
-        imj =im
-        name = 'output/bads/'+ str(i) + '.jpg'
-        for jj in xrange(1, imdb.num_classes):  
-            indsj = np.where(scores[:, jj] > thresh)[0]
-            cls_scoresj = scores[indsj, jj]
-            cls_boxesj = boxes[indsj, jj*4:(jj+1)*4]
-            cls_detsj = np.hstack((cls_boxesj, cls_scoresj[:, np.newaxis])) \
-                .astype(np.float32, copy=False)
-            keep = nms(cls_detsj, cfg.TEST.NMS)
-            cls_detsj = cls_detsj[keep, :]
-            detsj = cls_detsj
-            for ii in xrange(np.minimum(10, detsj.shape[0])):
-                 bboxj = detsj[ii, :4]
-                 scorej = detsj[ii, -1]
-                 if bboxj != []:
-                     x1 = bboxj[0]
-                     y1 = bboxj[3]
-                     x2 = bboxj[2]
-                     y2 = bboxj[1]
-                     if x1 < 0:
-                         x1=0
-                     if y1> imj.shape[1]:
-                         y1=imj.shape[1]-1
-                     if x2 > imj.shape[0]:
-                         x2 = imj.shape[0]-1
-                     if y2 < 0:
-                         y2 = 0
-                     if scorej > thresh:
-                         cv2.rectangle(imj, (x1, y1), (x2,y2),(0,255,0), 4)
-                         text = str(jj) + ": " + str(scorej)
-                         font = cv2.FONT_HERSHEY_SIMPLEX
-                         cv2.putText(imj, text, (x1, y1),font , 1, (0,0,255), 4)
-        cv2.imwrite(name, imj)
+        vis = True
+        if  vis:
+            imj =im
+            name = 'output/bads/'+ str(i) + '.jpg'
+            for jj in xrange(1, imdb.num_classes):  
+                indsj = np.where(scores[:, jj] > thresh)[0]
+                cls_scoresj = scores[indsj, jj]
+                cls_boxesj = boxes[indsj, jj*4:(jj+1)*4]
+                cls_detsj = np.hstack((cls_boxesj, cls_scoresj[:, np.newaxis])) \
+                    .astype(np.float32, copy=False)
+                keep = nms(cls_detsj, cfg.TEST.NMS)
+                cls_detsj = cls_detsj[keep, :]
+                detsj = cls_detsj
+                for ii in xrange(np.minimum(10, detsj.shape[0])):
+                    bboxj = detsj[ii, :4]
+                    scorej = detsj[ii, -1]
+                    if bboxj != []:
+                        x1 = bboxj[0]
+                        y1 = bboxj[3]
+                        x2 = bboxj[2]
+                        y2 = bboxj[1]
+                        if x1 < 0:
+                            x1=0
+                        if y1> imj.shape[1]:
+                            y1=imj.shape[1]-1
+                        if x2 > imj.shape[0]:
+                            x2 = imj.shape[0]-1
+                        if y2 < 0:
+                            y2 = 0
+                        if scorej > thresh:
+                            cv2.rectangle(imj, (x1, y1), (x2,y2),(0,255,0), 4)
+                            text = str(jj) + ": " + str(scorej)
+                            font = cv2.FONT_HERSHEY_SIMPLEX
+                            cv2.putText(imj, text, (x1, y1),font , 1, (0,0,255), 4)
+            cv2.imwrite(name, imj)
             #aaa
         # skip j = 0, because it's the background class
         for j in xrange(1, imdb.num_classes):

lib/fast_rcnn/train.py

@@ -16,6 +16,8 @@
 import google.protobuf.text_format
 from caffe.proto import caffe_pb2
 import google.protobuf as pb2
+import matplotlib
+matplotlib.use("Agg")
 
 class SolverWrapper(object):
     """A simple wrapper around Caffe's solver.
@@ -36,6 +38,7 @@ def __init__(self, solver_prototxt, roidb, output_dir,
 
         if cfg.TRAIN.BBOX_REG:
             print 'Computing bounding-box regression targets...'
+
             self.bbox_means, self.bbox_stds = \
                     rdl_roidb.add_bbox_regression_targets(roidb)
             print 'done'
@@ -61,18 +64,44 @@ def snapshot(self):
         scale_bbox_params = (cfg.TRAIN.BBOX_REG and
                              cfg.TRAIN.BBOX_NORMALIZE_TARGETS and
                              net.params.has_key('bbox_pred'))
-
+     
         if scale_bbox_params:
+            print "------------"
             # save original values
-            orig_0 = net.params['bbox_pred'][0].data.copy()
-            orig_1 = net.params['bbox_pred'][1].data.copy()
+            orig_0_h2 = net.params['bbox_pred/h2'][0].data.copy()
+            orig_1_h2 = net.params['bbox_pred/h2'][1].data.copy()     
+
+            orig_0_h3 = net.params['bbox_pred/h3'][0].data.copy()
+            orig_1_h3 = net.params['bbox_pred/h3'][1].data.copy()
+            orig_0_h4 = net.params['bbox_pred/h4'][0].data.copy()
+            orig_1_h4 = net.params['bbox_pred/h4'][1].data.copy()
+            orig_0_h5 = net.params['bbox_pred/h5'][0].data.copy()
+            orig_1_h5 = net.params['bbox_pred/h5'][1].data.copy()
 
             # scale and shift with bbox reg unnormalization; then save snapshot
-            net.params['bbox_pred'][0].data[...] = \
-                    (net.params['bbox_pred'][0].data *
+            net.params['bbox_pred/h2'][0].data[...] = \
+                    (net.params['bbox_pred/h2'][0].data *
+                     self.bbox_stds[:, np.newaxis])
+            net.params['bbox_pred/h2'][1].data[...] = \
+                    (net.params['bbox_pred/h2'][1].data *
+                     self.bbox_stds + self.bbox_means)
+            net.params['bbox_pred/h3'][0].data[...] = \
+                    (net.params['bbox_pred/h3'][0].data *
+                     self.bbox_stds[:, np.newaxis])
+            net.params['bbox_pred/h3'][1].data[...] = \
+                    (net.params['bbox_pred/h3'][1].data *
+                     self.bbox_stds + self.bbox_means)
+            net.params['bbox_pred/h4'][0].data[...] = \
+                    (net.params['bbox_pred/h4'][0].data *
+                     self.bbox_stds[:, np.newaxis])
+            net.params['bbox_pred/h4'][1].data[...] = \
+                    (net.params['bbox_pred/h4'][1].data *
+                     self.bbox_stds + self.bbox_means)
+            net.params['bbox_pred/h5'][0].data[...] = \
+                    (net.params['bbox_pred/h5'][0].data *
                      self.bbox_stds[:, np.newaxis])
-            net.params['bbox_pred'][1].data[...] = \
-                    (net.params['bbox_pred'][1].data *
+            net.params['bbox_pred/h5'][1].data[...] = \
+                    (net.params['bbox_pred/h5'][1].data *
                      self.bbox_stds + self.bbox_means)
 
         infix = ('_' + cfg.TRAIN.SNAPSHOT_INFIX
@@ -86,8 +115,14 @@ def snapshot(self):
 
         if scale_bbox_params:
             # restore net to original state
-            net.params['bbox_pred'][0].data[...] = orig_0
-            net.params['bbox_pred'][1].data[...] = orig_1
+            net.params['bbox_pred/h2'][0].data[...] = orig_0_h2
+            net.params['bbox_pred/h2'][1].data[...] = orig_1_h2
+            net.params['bbox_pred/h3'][0].data[...] = orig_0_h3
+            net.params['bbox_pred/h3'][1].data[...] = orig_1_h3
+            net.params['bbox_pred/h4'][0].data[...] = orig_0_h4
+            net.params['bbox_pred/h4'][1].data[...] = orig_1_h4
+            net.params['bbox_pred/h5'][0].data[...] = orig_0_h5
+            net.params['bbox_pred/h5'][1].data[...] = orig_1_h5
         return filename
 
     def train_model(self, max_iters):

lib/pycocotools/coco.py

@@ -47,7 +47,10 @@
 import json
 import datetime
 import time
+import matplotlib
+matplotlib.use('Agg')
 import matplotlib.pyplot as plt
+
 from matplotlib.collections import PatchCollection
 from matplotlib.patches import Polygon
 import numpy as np

lib/rpn/anchor_target_layer.py

@@ -50,7 +50,7 @@ def setup(self, bottom, top):
             self._count = 0
 
         # allow boxes to sit over the edge by a small amount
-        self._allowed_border = layer_params.get('allowed_border', 1000)
+        self._allowed_border = layer_params.get('allowed_border', 100)
 
         height, width = bottom[0].data.shape[-2:]
         if DEBUG:
@@ -168,7 +168,7 @@ def forward(self, bottom, top):
             labels[disable_inds] = -1
             #print "was %s inds, disabling %s, now %s inds" % (
                 #len(bg_inds), len(disable_inds), np.sum(labels == 0))
-
+    
         bbox_targets = np.zeros((len(inds_inside), 4), dtype=np.float32)
         bbox_targets = _compute_targets(anchors, gt_boxes[argmax_overlaps, :])
 
@@ -207,6 +207,7 @@ def forward(self, bottom, top):
         bbox_targets = _unmap(bbox_targets, total_anchors, inds_inside, fill=0)
         bbox_inside_weights = _unmap(bbox_inside_weights, total_anchors, inds_inside, fill=0)
         bbox_outside_weights = _unmap(bbox_outside_weights, total_anchors, inds_inside, fill=0)
+
 
         if DEBUG:
             print 'rpn: max max_overlap', np.max(max_overlaps)

lib/rpn/as_rois.py

@@ -23,7 +23,7 @@ class As_roisLayer(caffe.Layer):
 
     def setup(self, bottom, top):
         layer_params = yaml.load(self.param_str_)
-        self._batch_rois = cfg.TEST.RPN_POST_NMS_TOP_N/4
+        self._batch_rois = cfg.TEST.RPN_POST_NMS_TOP_N
 
         # sampled rois (0, x1, y1, x2, y2)
         top[0].reshape(1, 5)

lib/rpn/proposal_layer.py

@@ -12,6 +12,7 @@
 from generate_anchors import generate_anchors
 from fast_rcnn.bbox_transform import bbox_transform_inv, clip_boxes
 from fast_rcnn.nms_wrapper import nms
+import numpy.random as npr
 
 DEBUG = False
 
@@ -71,9 +72,11 @@ def forward(self, bottom, top):
         # the first set of _num_anchors channels are bg probs
         # the second set are the fg probs, which we want
         scores_list = bottom[0].data
+
         bbox_deltas_list = bottom[1].data
         im_info = bottom[2].data[0, :]
 
+
         p2_shape =  bottom[3].data.shape
         p3_shape =  bottom[4].data.shape
         p4_shape =  bottom[5].data.shape
@@ -147,6 +150,7 @@ def forward(self, bottom, top):
 
         if post_nms_topN > 0:
             keep = keep[:post_nms_topN]
+
         # pad to ensure output size remains unchanged
         if len(keep) < post_nms_topN:
             try:
@@ -161,6 +165,7 @@ def forward(self, bottom, top):
                 top[0].data[...] = blob
                 return
             keep = np.hstack((keep, pad))
+
         proposals = proposals[keep, :]
         scores = scores[keep]
         #print 'roi concate t_2 spends :{:.4f}s'.format(time.time()-t_2)
@@ -169,6 +174,8 @@ def forward(self, bottom, top):
         # batch inds are 0
         batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32)
         blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False)))
+
+
         top[0].reshape(*(blob.shape))
         top[0].data[...] = blob