From feb05cc1afd8298b79b4478d8c44b809d3aff883 Mon Sep 17 00:00:00 2001 From: Mulong Date: Mon, 2 Mar 2020 16:18:40 +1100 Subject: [PATCH] add f1 score --- result_processing/box_plot.ipynb | 310 ------------------------------- result_processing/evaluation.py | 15 +- 2 files changed, 9 insertions(+), 316 deletions(-) delete mode 100644 result_processing/box_plot.ipynb diff --git a/result_processing/box_plot.ipynb b/result_processing/box_plot.ipynb deleted file mode 100644 index d3ec952..0000000 --- a/result_processing/box_plot.ipynb +++ /dev/null @@ -1,310 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import numpy as np\n", - "import cv2\n", - "from glob import glob\n", - "from os.path import join as pjoin\n", - "from tqdm import tqdm" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "def resize_label(bboxes, d_height, gt_height, bias=0):\n", - " bboxes_new = []\n", - " scale = gt_height/d_height\n", - " for bbox in bboxes:\n", - " bbox = [int(b * scale + bias) for b in bbox]\n", - " bboxes_new.append(bbox)\n", - " return bboxes_new\n", - "\n", - "\n", - "def draw_bounding_box(org, corners, color=(0, 255, 0), line=2, show=False):\n", - " board = org.copy()\n", - " for i in range(len(corners)):\n", - " board = cv2.rectangle(board, (corners[i][0], corners[i][1]), (corners[i][2], corners[i][3]), color, line)\n", - " if show:\n", - " cv2.imshow('a', cv2.resize(board, (300, 600)))\n", - " cv2.waitKey(0)\n", - " return board\n", - "\n", - "\n", - "def load_detect_result_json(reslut_file_root):\n", - " result_files = glob(pjoin(reslut_file_root, '*ip.json'))\n", - " compos_reform = {}\n", - " print('Loading %d detection results' % len(result_files))\n", - " for reslut_file in tqdm(result_files):\n", - " img_name = reslut_file.split('\\\\')[-1].split('_')[0]\n", - " compos = json.load(open(reslut_file, 'r'))['compos']\n", - " for compo in compos:\n", - " if img_name not in compos_reform:\n", - " compos_reform[img_name] = {'bboxes': [[compo['column_min'], compo['row_min'], compo['column_max'], compo['row_max']]],\n", - " 'categories': [compo['category']]}\n", - " else:\n", - " compos_reform[img_name]['bboxes'].append([compo['column_min'], compo['row_min'], compo['column_max'], compo['row_max']])\n", - " compos_reform[img_name]['categories'].append(compo['category'])\n", - " return compos_reform\n", - "\n", - "\n", - "def load_ground_truth_json(gt_file):\n", - " def get_img_by_id(img_id):\n", - " for image in images:\n", - " if image['id'] == img_id:\n", - " return image['file_name'].split('/')[-1][:-4], (image['height'], image['width'])\n", - "\n", - " def cvt_bbox(bbox):\n", - " '''\n", - " :param bbox: [x,y,width,height]\n", - " :return: [col_min, row_min, col_max, row_max]\n", - " '''\n", - " bbox = [int(b) for b in bbox]\n", - " return [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]]\n", - "\n", - " data = json.load(open(gt_file, 'r'))\n", - " images = data['images']\n", - " annots = data['annotations']\n", - " compos = {}\n", - " print('Loading %d ground truth' % len(annots))\n", - " for annot in tqdm(annots):\n", - " img_name, size = get_img_by_id(annot['image_id'])\n", - " if img_name not in compos:\n", - " compos[img_name] = {'bboxes': [cvt_bbox(annot['bbox'])], 'categories': [annot['category_id']], 'size':size}\n", - " else:\n", - " compos[img_name]['bboxes'].append(cvt_bbox(annot['bbox']))\n", - " compos[img_name]['categories'].append(annot['category_id'])\n", - " return compos\n", - "\n", - "\n", - "def eval(detection, ground_truth, img_root, show=True):\n", - " def match(org, d_bbox, gt_bboxes, matched):\n", - " '''\n", - " :param matched: mark if the ground truth component is matched\n", - " :param d_bbox: [col_min, row_min, col_max, row_max]\n", - " :param gt_bboxes: list of ground truth [[col_min, row_min, col_max, row_max]]\n", - " :return: Boolean: if IOU large enough or detected box is contained by ground truth\n", - " '''\n", - " area_d = (d_bbox[2] - d_bbox[0]) * (d_bbox[3] - d_bbox[1])\n", - " for i, gt_bbox in enumerate(gt_bboxes):\n", - " if matched[i] == 0:\n", - " continue\n", - " area_gt = (gt_bbox[2] - gt_bbox[0]) * (gt_bbox[3] - gt_bbox[1])\n", - " col_min = max(d_bbox[0], gt_bbox[0])\n", - " row_min = max(d_bbox[1], gt_bbox[1])\n", - " col_max = min(d_bbox[2], gt_bbox[2])\n", - " row_max = min(d_bbox[3], gt_bbox[3])\n", - " # if not intersected, area intersection should be 0\n", - " w = max(0, col_max - col_min)\n", - " h = max(0, row_max - row_min)\n", - " area_inter = w * h\n", - " if area_inter == 0:\n", - " continue\n", - " iod = area_inter / area_d\n", - " iou = area_inter / (area_d + area_gt - area_inter)\n", - "\n", - " # if show:\n", - " # print(\"IoDetection: %.3f, IoU: %.3f\" % (iod, iou))\n", - " # broad = draw_bounding_box(org, [d_bbox], color=(0, 0, 255))\n", - " # draw_bounding_box(broad, [gt_bbox], color=(0, 255, 0), show=True)\n", - "\n", - " if iou >= 0.5 or iod == 1:\n", - " matched[i] = 0\n", - " return True\n", - " return False\n", - "\n", - " pres = []\n", - " recalls = []\n", - " amount = len(detection)\n", - " for i, image_id in enumerate(detection):\n", - " TP, FP, FN = 0, 0, 0\n", - " img = cv2.imread(pjoin(img_root, image_id + '.jpg'))\n", - " d_compos = detection[image_id]\n", - " gt_compos = ground_truth[image_id]\n", - " d_compos['bboxes'] = resize_label(d_compos['bboxes'], 800, gt_compos['size'][0])\n", - " matched = np.ones(len(gt_compos['bboxes']), dtype=int)\n", - " for d_bbox in d_compos['bboxes']:\n", - " if match(img, d_bbox, gt_compos['bboxes'], matched):\n", - " TP += 1\n", - " else:\n", - " FP += 1\n", - " FN += sum(matched)\n", - "\n", - " precesion = TP / (TP+FP)\n", - " recall = TP / (TP+FN)\n", - "\n", - " pres.append(precesion)\n", - " recalls.append(recall)\n", - " if show:\n", - " print(\"Number of gt boxes: %d, Number of detected boxes: %d\" % (\n", - " len(gt_compos['bboxes']), len(d_compos['bboxes'])))\n", - " print('[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f' % (i, amount, TP, FP, FN, precesion, recall))\n", - "\n", - " broad = draw_bounding_box(img, d_compos['bboxes'], color=(0, 0, 255), line=3)\n", - " draw_bounding_box(broad, gt_compos['bboxes'], color=(0, 255, 0), show=True, line=2)\n", - "\n", - " if i % 200 == 0:\n", - " print('[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f' % (i, amount, TP, FP, FN, precesion, recall))\n", - "\n", - " return pres, recalls" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 8%|██████ | 372/4709 [00:00<00:01, 3692.35it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading 4709 detection results\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|████████████████████████████████████████████████████████████████████████████| 4709/4709 [00:01<00:00, 3363.71it/s]\n", - " 6%|████▌ | 5332/86646 [00:00<00:01, 52934.62it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading 86646 ground truth\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████████████████████████████████████████████████████████████████████| 86646/86646 [00:14<00:00, 5794.24it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[0/4708] TP:3, FP:3, FN:0, Precesion:0.500, Recall:1.000\n", - "[200/4708] TP:7, FP:32, FN:9, Precesion:0.179, Recall:0.438\n", - "[400/4708] TP:12, FP:5, FN:0, Precesion:0.706, Recall:1.000\n", - "[600/4708] TP:8, FP:2, FN:0, Precesion:0.800, Recall:1.000\n", - "[800/4708] TP:13, FP:3, FN:1, Precesion:0.812, Recall:0.929\n", - "[1000/4708] TP:5, FP:3, FN:1, Precesion:0.625, Recall:0.833\n", - "[1200/4708] TP:1, FP:20, FN:3, Precesion:0.048, Recall:0.250\n", - "[1400/4708] TP:5, FP:19, FN:0, Precesion:0.208, Recall:1.000\n", - "[1600/4708] TP:10, FP:5, FN:0, Precesion:0.667, Recall:1.000\n", - "[1800/4708] TP:13, FP:6, FN:3, Precesion:0.684, Recall:0.812\n", - "[2000/4708] TP:10, FP:3, FN:12, Precesion:0.769, Recall:0.455\n", - "[2200/4708] TP:13, FP:13, FN:8, Precesion:0.500, Recall:0.619\n", - "[2400/4708] TP:2, FP:8, FN:7, Precesion:0.200, Recall:0.222\n", - "[2600/4708] TP:4, FP:33, FN:0, Precesion:0.108, Recall:1.000\n", - "[2800/4708] TP:7, FP:24, FN:10, Precesion:0.226, Recall:0.412\n", - "[3000/4708] TP:6, FP:20, FN:4, Precesion:0.231, Recall:0.600\n", - "[3200/4708] TP:10, FP:18, FN:0, Precesion:0.357, Recall:1.000\n", - "[3400/4708] TP:13, FP:24, FN:16, Precesion:0.351, Recall:0.448\n" - ] - }, - { - "ename": "KeyError", - "evalue": "'557'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mKeyError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[0mdetect\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mload_detect_result_json\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'E:\\\\Mulong\\\\Result\\\\rico\\\\rico_uied\\\\rico_new_uied_cls\\\\ip'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[0mgt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mload_ground_truth_json\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'E:/Mulong/Datasets/rico/instances_test.json'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mpres\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrecalls\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdetect\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'E:\\\\Mulong\\\\Datasets\\\\rico\\\\combined'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshow\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[1;32m\u001b[0m in \u001b[0;36meval\u001b[1;34m(detection, ground_truth, img_root, show)\u001b[0m\n\u001b[0;32m 107\u001b[0m \u001b[0mimg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mimg_root\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mimage_id\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m'.jpg'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 108\u001b[0m \u001b[0md_compos\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdetection\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mimage_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 109\u001b[1;33m \u001b[0mgt_compos\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mground_truth\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mimage_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 110\u001b[0m \u001b[0md_compos\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'bboxes'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mresize_label\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0md_compos\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'bboxes'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m800\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgt_compos\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'size'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 111\u001b[0m \u001b[0mmatched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mones\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgt_compos\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'bboxes'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;31mKeyError\u001b[0m: '557'" - ] - } - ], - "source": [ - "detect = load_detect_result_json('E:\\\\Mulong\\\\Result\\\\rico\\\\rico_uied\\\\rico_new_uied_cls\\\\ip')\n", - "gt = load_ground_truth_json('E:/Mulong/Datasets/rico/instances_test.json')\n", - "pres, recalls = eval(detect, gt, 'E:\\\\Mulong\\\\Datasets\\\\rico\\\\combined', show=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "D:\\Anaconda\\lib\\site-packages\\matplotlib\\figure.py:448: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.\n", - " % get_backend())\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAD9JJREFUeJzt3X+QXWddx/H3pwnhR5u2xOCKbSEdCEJkhOLSAhVJBIcUnUZHio3yyylkxrEggmIZnBLrH/xwlEENaMAOCAylMFBjJ1AYzE5LIZBNf9G0dCaGYmIZy49SBhBK6Nc/7glctru5d3fv7qZP3q+ZO3vOeZ4953tvnnz27HPv2ZOqQpLUlhOWugBJ0ugZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGLV+qA69evbrWrFmzVIeXpAelvXv3fqOqHjWo35KF+5o1a5icnFyqw0vSg1KSrw7Tz2kZSWqQ4S5JDTLcJalBhrskNchwl6QGDQz3JJcnuTvJrTO0J8k/JNmf5JYkTxt9mZKk2RjmzP29wMajtJ8HrO0eW4B3zb8sSdJ8DAz3qroW+NZRumwC/q16dgOnJnn0qAqUJM3eKC5iOg042Ld+qNv2takdk2yhd3bP2NgYExMTIzj88WPDhg1z+r5du3aNuBLpgeYyPh2bC2cU4Z5ptk171+2q2g5sBxgfH6/169eP4PDHj6PdzDzJUdulhTbT+HNsLo1RfFrmEHBG3/rpwF0j2K8kaY5GEe47gJd2n5p5BnBvVT1gSkaStHgGTssk+RCwHlid5BDwJuAhAFX1z8BO4AXAfuD7wB8tVLGSpOEMDPeq2jygvYA/GVlFkqR58wpVSWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrukWVm1ahVJhn4As+qfhFWrVi3xs3zwW77UBUh6cLnnnnuoqgU9xpEfCpo7z9wlqUFDhXuSjUnuSLI/ySXTtD8mya4kNya5JckLRl+qJGlYA8M9yTJgG3AesA7YnGTdlG5/BVxZVWcBFwLvHHWhkqThDXPmfjawv6oOVNV9wBXApil9Cji5Wz4FuGt0JUqSZmuYN1RPAw72rR8CzpnSZyvwqSSvAk4EnjeS6iRJczJMuE/3tvXUt8o3A++tqr9L8kzg/UmeXFX3/8yOki3AFoCxsTEmJibmULJm4uupxbIYY83xPD8Z9JGmLqy3VtXzu/U3AFTVm/v67AM2VtXBbv0A8Iyqunum/Y6Pj9fk5OT8n4GA3kfHFvrjaRIszlhzPM8syd6qGh/Ub5g59z3A2iRnJllB7w3THVP6/Dfw3O7ATwIeBnx9diVLkkZlYLhX1WHgYuAa4HZ6n4rZl+SyJOd33V4HvDLJzcCHgJeXP3YlackMdYVqVe0Edk7Zdmnf8m3AuaMtTZI0V16hKkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMP9GDTbmyHM5YYI3gxBaps36zgGeTMESfPlmbskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBy4fplGQj8A5gGfCeqnrLNH1eBGwFCri5qv5ghHVKOkbUm06Gracs/DE0LwPDPckyYBvwm8AhYE+SHVV1W1+ftcAbgHOr6p4kP79QBUtaWvnr71BVC3uMhNq6oIdo3jDTMmcD+6vqQFXdB1wBbJrS55XAtqq6B6Cq7h5tmZKk2RhmWuY04GDf+iHgnCl9ngCQ5Hp6Uzdbq+qTU3eUZAuwBWBsbIyJiYk5lHx8WIzXxtdfc+X4PPZl0K9XSS4Anl9Vr+jWXwKcXVWv6utzNfAj4EXA6cB1wJOr6tsz7Xd8fLwmJyfn/wwalGRxfu1d4GOoTY7PpZVkb1WND+o3zLTMIeCMvvXTgbum6fPvVfWjqvoKcAewdthiJUmjNUy47wHWJjkzyQrgQmDHlD5XARsAkqymN01zYJSFSpKGNzDcq+owcDFwDXA7cGVV7UtyWZLzu27XAN9MchuwC/iLqvrmQhUtSTq6gXPuC8U595k5p6ljmeNzaQ075z7URUxaXF4kImm+DPdjkBeJSJov/7aMJDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGjRUuCfZmOSOJPuTXHKUfi9MUknGR1eiJGm2BoZ7kmXANuA8YB2wOcm6afqtBF4NfGHURUqSZmeYM/ezgf1VdaCq7gOuADZN0+9vgLcBPxhhfZKkORgm3E8DDvatH+q2/USSs4AzqurqEdYmSZqj5UP0yTTb6ieNyQnA24GXD9xRsgXYAjA2NsbExMRQRR6PFuO18fXXXCXTxcLorFy50vE5T6mqo3dInglsrarnd+tvAKiqN3frpwD/BXy3+5ZfAL4FnF9VkzPtd3x8vCYnZ2w+riVh0L/Lg+EYEjjWRi3J3qoa+KGVYaZl9gBrk5yZZAVwIbDjSGNV3VtVq6tqTVWtAXYzINglSQtrYLhX1WHgYuAa4Hbgyqral+SyJOcvdIGSpNkbZs6dqtoJ7Jyy7dIZ+q6ff1mSpPnwClVJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSg4a6iEmLb6H/MNMjH/nIBd2/pKVluB+D5vJHlvzjTJL6OS0jSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KChwj3JxiR3JNmf5JJp2l+b5LYktyT5TJLHjr5USdKwBoZ7kmXANuA8YB2wOcm6Kd1uBMar6leAjwJvG3WhkqThDXPmfjawv6oOVNV9wBXApv4OVbWrqr7fre4GTh9tmZKk2Vg+RJ/TgIN964eAc47S/yLgE9M1JNkCbAEYGxtjYmJiuCo1FF9PHascm4tvmHDPNNtq2o7Ji4Fx4DnTtVfVdmA7wPj4eK1fv364KjUUX08dqxybi2+YcD8EnNG3fjpw19ROSZ4HvBF4TlX9cDTlSZLmYpg59z3A2iRnJlkBXAjs6O+Q5CzgX4Dzq+ru0ZcpSZqNgeFeVYeBi4FrgNuBK6tqX5LLkpzfdftb4CTgI0luSrJjht1JkhbBMNMyVNVOYOeUbZf2LT9vxHVJkubBK1QlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSg4YK9yQbk9yRZH+SS6Zpf2iSD3ftX0iyZtSFSpKGNzDckywDtgHnAeuAzUnWTel2EXBPVT0eeDvw1lEXKkka3jBn7mcD+6vqQFXdB1wBbJrSZxPwvm75o8Bzk2R0ZUqSZmOYcD8NONi3fqjbNm2fqjoM3Av83CgKlCTN3vIh+kx3Bl5z6EOSLcAWgLGxMSYmJoY4vI7YsGHDUdtn+mVp165dC1GO9DOONj4dm4tvmHA/BJzRt346cNcMfQ4lWQ6cAnxr6o6qajuwHWB8fLzWr18/h5KPX1UP+HkpHTMcn8eWYaZl9gBrk5yZZAVwIbBjSp8dwMu65RcC/1n+S0vSkhl45l5Vh5NcDFwDLAMur6p9SS4DJqtqB/CvwPuT7Kd3xn7hQhYtSTq6YaZlqKqdwM4p2y7tW/4BcMFoS5MkzZVXqEpSgwx3SWqQ4S5JDTLcJalBhrskNShL9XH0JF8HvrokB2/TauAbS12ENA3H5mg9tqoeNajTkoW7RivJZFWNL3Ud0lSOzaXhtIwkNchwl6QGGe7t2L7UBUgzcGwuAefcJalBnrlLUoMM9wexJJ8b0L4zyamLVY80G0nWJLm1W16f5OqlrqklQ/1VSC28JMuq6sez+Z6qetaA9hfMryrpgbr7I6eq7l/qWjQzz9wXQXeG8uUk70tyS5KPJnlEkjuTXJrks8AFSR6X5JNJ9ia5LskTu+8fS/LxJDd3j2d127/bfX10kmuT3JTk1iTP7rbfmWR1t/zaru3WJK/pq+v2JO9Osi/Jp5I8vGt7dZLbunqvWIKXTceQvrHyTuAG4CVJPp/khiQfSXJS1+/pST7XjdMvJlnZfe91Xd8bjozfoxzrOd1YvinJjUlWLsZzbE5V+VjgB7CG3j1lz+3WLwf+HLgTeH1fv88Aa7vlc+jd0Qrgw8BruuVlwCnd8ne7r68D3tjXvrJbvpPe1YG/CnwJOBE4CdgHnNXVdRh4atf/SuDF3fJdwEO75VOX+jX0cUyM4fuBZ3Rj6lrgxK7tL4FLgRXAAeDp3faT6c0OPAJ4WLdtLb2b/BzZ563d8nrg6m75P/r+r5wELF/q5/9gfDgts3gOVtX13fIHgFd3yx8G6M58ngV8pO9mwg/tvv4G8FKA6k3d3Dtl33uAy5M8BLiqqm6a0v5rwMer6nvdsT4GPJve7RG/0td/L73/cAC3AB9MchVw1VyesJrz1araneS3gXXA9d1YXQF8Hvgl4GtVtQegqr4DkORE4J+SPBX4MfCEAce5Hvj7JB8EPlZVhxbk2TTOaZnFM/Uzp0fWv9d9PQH4dlU9te/xpKF2XHUt8OvA/9C73eFLp3SZ/tbzPT/sW/4xP30f5reAbfTO+vd2Nz7X8e3IWA3w6b5xuq6qLuq2T/fZ6j8D/hd4CjBO74fBjKrqLcArgIcDu49MT2p2DPfF85gkz+yWNwOf7W/sznK+kuQC6L1pleQpXfNngD/uti9LcnL/9yZ5LHB3Vb2b3v1snzbl2NcCv9PN858I/C5w3UyFJjkBOKOqdgGvB06l9+uxBLAbODfJ4wG6cfUE4MvALyZ5erd9ZXdScAq9M/r7gZfQmzqcUZLHVdWXquqtwCRguM+B4b54bgdeluQWYBXwrmn6/CFwUZKb6c2Lb+q2/ymwIcmX6E2d/PKU71sP3JTkRuD3gHf0N1bVDcB7gS8CXwDeU1U3HqXWZcAHuuPdCLy9qr495PNU46rq68DLgQ9143k38MSqug/4feAfuzH8aeBhwDvpjf3d9KZkvjftjn/qNd0b/zcD/wd8YmGeSdu8QnURJFlD782iJy9xKZKOE565S1KDPHOXpAZ55i5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIa9P965KgdOKExlwAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "fig = plt.figure() # 创建画布\n", - "ax = plt.subplot() # 创建作图区域\n", - "ax.boxplot([pres, recalls])\n", - "ax.set_xticklabels(['precisions', 'recalls'])\n", - "plt.grid(axis='y')\n", - "fig.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.6" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/result_processing/evaluation.py b/result_processing/evaluation.py index 5b5778a..7bdb19c 100644 --- a/result_processing/evaluation.py +++ b/result_processing/evaluation.py @@ -135,7 +135,7 @@ def match(org, d_bbox, gt_bboxes, matched): amount = len(detection) TP, FP, FN = 0, 0, 0 - pres, recalls = [], [] + pres, recalls, f1s = [], [], [] for i, image_id in enumerate(detection): TP_this, FP_this, FN_this = 0, 0, 0 img = cv2.imread(pjoin(img_root, image_id + '.jpg')) @@ -162,12 +162,14 @@ def match(org, d_bbox, gt_bboxes, matched): try: pre_this = TP_this / (TP_this + FP_this) recall_this = TP_this / (TP_this + FN_this) + f1_this = 2 * (pre_this * recall_this) / (pre_this + recall_this) except: print('empty') continue pres.append(pre_this) recalls.append(recall_this) + f1s.append(f1_this) if show: print(image_id + '.jpg') print('[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f' % ( @@ -177,14 +179,15 @@ def match(org, d_bbox, gt_bboxes, matched): draw_bounding_box(broad, gt_compos['bboxes'], color=(0, 0, 255), show=True, line=2) if i % 200 == 0: - precesion = TP / (TP + FP) + precision = TP / (TP + FP) recall = TP / (TP + FN) + f1 = 2 * (precision * recall) / (precision + recall) print( - '[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f' % (i, amount, TP, FP, FN, precesion, recall)) + '[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f, F1:%.3f' % (i, amount, TP, FP, FN, precision, recall, f1)) - precesion = TP / (TP + FP) + precision = TP / (TP + FP) recall = TP / (TP + FN) - print('[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f' % (i, amount, TP, FP, FN, precesion, recall)) + print('[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f, F1:%.3f' % (i, amount, TP, FP, FN, precision, recall, f1)) # print("Average precision:%.4f; Average recall:%.3f" % (sum(pres)/len(pres), sum(recalls)/len(recalls))) return pres, recalls @@ -197,4 +200,4 @@ def match(org, d_bbox, gt_bboxes, matched): detect = load_detect_result_json('E:\\Mulong\\Result\\rico\\rico_uied\\rico_new_uied_cls\\merge') # detect = load_detect_result_json('E:\\Mulong\\Result\\rico\\rico_uied\\rico_new_uied_v3\\merge', no_text=no_text, only_text=only_text) gt = load_ground_truth_json('E:\\Mulong\\Datasets\\rico\\instances_test.json') -eval(detect, gt, 'E:\\Mulong\\Datasets\\rico\\combined', show=False, no_text=no_text, only_text=only_text) +eval(detect, gt, 'E:\\Mulong\\Datasets\\rico\\combined', show=True, no_text=no_text, only_text=only_text)