diff --git a/ILSVRC_evaluate_bbox.m b/ILSVRC_evaluate_bbox.m new file mode 100644 index 0000000..5d8c10c --- /dev/null +++ b/ILSVRC_evaluate_bbox.m @@ -0,0 +1,152 @@ + + +datasetName = 'ILSVRCvalSet'; +load('imagenet_toolkit/ILSVRC2014_devkit/evaluation/cache_groundtruth.mat'); +load('imagenet_toolkit/ILSVRC2014_devkit/data/meta_clsloc.mat'); +datasetPath = 'dataset/ILSVRC2012'; +load([datasetPath '/imageListVal.mat']); +load('sizeImg_ILSVRC2014.mat'); + +% datasetName = 'ILSVRCtestSet'; +% datasetPath = '/data/vision/torralba/deeplearning/imagenet_toolkit'; +% load([datasetPath '/imageListTest.mat']); + + +nImgs = size(imageList,1); + +ground_truth_file='imagenet_toolkit/ILSVRC2014_devkit/data/ILSVRC2014_clsloc_validation_ground_truth.txt'; +gt_labels = dlmread(ground_truth_file); + +categories_gt = []; +categoryIDMap = containers.Map(); +for i=1:numel(synsets) + categories_gt{synsets(i).ILSVRC2014_ID,1} = synsets(i).words; + categories_gt{synsets(i).ILSVRC2014_ID,2} = synsets(i).WNID; + categoryIDMap(synsets(i).WNID) = i; +end + + + +%% network to evaluate +% backpropa-heatmap +%netName = 'caffeNet_imagenet'; +%netName = 'googlenetBVLC_imagenet'; +%netName = 'VGG16_imagenet'; + +% CAM-based network +%netName = 'NIN'; +%netName = 'CAM_imagenetCNNaveSumDeep'; +%netName = 'CAM_googlenetBVLC_imagenet';% the direct output +netName = 'CAM_googlenetBVLCshrink_imagenet'; +%netName = 'CAM_googlenetBVLCshrink_imagenet_maxpool'; +%netName = 'CAM_VGG16_imagenet'; +%netName = 'CAM_alexnet'; + +load('categoriesImageNet.mat'); + +visualizationPointer = 0; + +topCategoryNum = 5; +predictionResult_bbox1 = zeros(nImgs, topCategoryNum*5); +predictionResult_bbox2 = zeros(nImgs, topCategoryNum*5); +predictionResult_bboxCombine = zeros(nImgs, topCategoryNum*5); + +if matlabpool('size')==0 + try + matlabpool + catch e + end +end + +heatMapFolder = ['heatMap-' datasetName '-' netName]; +bbox_threshold = [20, 100, 110]; +curParaThreshold = [num2str(bbox_threshold(1)) ' ' num2str(bbox_threshold(2)) ' ' num2str(bbox_threshold(3))]; +parfor i=1:size(imageList,1) + curImgIDX = i; + + height_original = sizeFull_imageList(curImgIDX,1);%tmp.Height; + weight_original = sizeFull_imageList(curImgIDX,2);%tmp.Width; + + [a b c] = fileparts(imageList{curImgIDX,1}); + curPath_fullSizeImg = ['/data/vision/torralba/deeplearning/imagenet_toolkit/ILSVRC2012_img_val/' b c]; + curMatFile = [heatMapFolder '/' b '.mat']; + [heatMapSet, value_category, IDX_category] = loadHeatMap( curMatFile); + + curResult_bbox1 = []; + curResult_bbox2 = []; + curResult_bboxCombine = []; + for j=1:5 + curHeatMapFile = [heatMapFolder '/top' num2str(j) '/' b '.jpg']; + + curBBoxFile = [heatMapFolder '/top' num2str(j) '/' b '_default.txt']; + %curBBoxFileGraphcut = [heatMapFolder '/top' num2str(j) '/' b '_graphcut.txt']; + curCategory = categories{IDX_category(j),1}; + %imwrite(curHeatMap, ['result_bbox/heatmap_tmp' b randString '.jpg']); + if ~exist(curBBoxFile) + %system(['/data/vision/torralba/deeplearning/package/bbox_hui/final ' curHeatMapFile ' ' curBBoxFile]); + + system(['/data/vision/torralba/deeplearning/package/bbox_hui_new/./dt_box ' curHeatMapFile ' ' curParaThreshold ' ' curBBoxFile]); + end + curPredictCategory = categories{IDX_category(j),1}; + curPredictCategoryID = categories{IDX_category(j),1}(1:9); + curPredictCategoryGTID = categoryIDMap(curPredictCategoryID); + + + boxData = dlmread(curBBoxFile); + boxData_formulate = [boxData(1:4:end)' boxData(2:4:end)' boxData(1:4:end)'+boxData(3:4:end)' boxData(2:4:end)'+boxData(4:4:end)']; + boxData_formulate = [min(boxData_formulate(:,1),boxData_formulate(:,3)),min(boxData_formulate(:,2),boxData_formulate(:,4)),max(boxData_formulate(:,1),boxData_formulate(:,3)),max(boxData_formulate(:,2),boxData_formulate(:,4))]; + +% try +% boxDataGraphcut = dlmread(curBBoxFileGraphcut); +% boxData_formulateGraphcut = [boxDataGraphcut(1:4:end)' boxDataGraphcut(2:4:end)' boxDataGraphcut(1:4:end)'+boxDataGraphcut(3:4:end)' boxDataGraphcut(2:4:end)'+boxDataGraphcut(4:4:end)']; +% catch exception +% boxDataGraphcut = dlmread(curBBoxFile); +% boxData_formulateGraphcut = [boxDataGraphcut(1:4:end)' boxDataGraphcut(2:4:end)' boxDataGraphcut(1:4:end)'+boxDataGraphcut(3:4:end)' boxDataGraphcut(2:4:end)'+boxDataGraphcut(4:4:end)']; +% boxData_formulateGraphcut = boxData_formulateGraphcut(1,:); +% end + + bbox = boxData_formulate(1,:); + curPredictTuple = [curPredictCategoryGTID bbox(1) bbox(2) bbox(3) bbox(4)]; + curResult_bbox1 = [curResult_bbox1 curPredictTuple]; + curResult_bboxCombine = [curResult_bboxCombine curPredictTuple]; + + bbox = boxData_formulate(2,:); + %bbox = boxData_formulateGraphcut(1,:); + curPredictTuple = [curPredictCategoryGTID bbox(1) bbox(2) bbox(3) bbox(4)]; + curResult_bbox2 = [curResult_bbox2 curPredictTuple]; + + curResult_bboxCombine = [curResult_bboxCombine curPredictTuple]; + if visualizationPointer == 1 + + curHeatMap = imread(curHeatMapFile); + curHeatMap = imresize(curHeatMap,[height_original weight_original]); + + subplot(1,2,1),hold off, imshow(curPath_fullSizeImg); + hold on + curBox = boxData_formulate(1,:); + rectangle('Position',[curBox(1) curBox(2) curBox(3)-curBox(1) curBox(4)-curBox(2)],'EdgeColor',[1 0 0]); + subplot(1,2,2),imagesc(curHeatMap); + title(curCategory); + waitforbuttonpress + end + end + + predictionResult_bbox1(i, :) = curResult_bbox1; + predictionResult_bbox2(i, :) = curResult_bbox2; + predictionResult_bboxCombine(i,:) = curResult_bboxCombine(1:topCategoryNum*5); + disp([netName ' processing ' b]) +end + + +addpath('imagenet_toolkit/ILSVRC2014_devkit/evaluation'); +disp([netName '--------bbox1' ]); +[cls_error, clsloc_error] = simpleEvaluation(predictionResult_bbox1); +disp([(1:5)',clsloc_error,cls_error]); + +disp([netName '--------bbox2' ]); +[cls_error, clsloc_error] = simpleEvaluation(predictionResult_bbox2); +disp([(1:5)',clsloc_error,cls_error]); + +disp([netName '--------bboxCombine' ]); +[cls_error, clsloc_error] = simpleEvaluation(predictionResult_bboxCombine); +disp([(1:5)',clsloc_error,cls_error]); diff --git a/ILSVRC_generate_heatmap.m b/ILSVRC_generate_heatmap.m new file mode 100644 index 0000000..52a360e --- /dev/null +++ b/ILSVRC_generate_heatmap.m @@ -0,0 +1,138 @@ +% raw script used to generate heatmaps for ILSVRC localization experiment +% please load the necessary packages like matcaffe and ILSVRC toolbox correctly, some functions in matcaffe might be already deprecated. +% you could take it as an example to see how to reproduce the ILSVRC localization experiment. +% +% Bolei Zhou. + +addpath('caffeCPU2/matlab/caffe'); + +modelSetFolder = 'CAMnet'; + +%% CAMnet + + +% netName = 'CAM_googlenetBVLC_imagenet'; +% model_file = [modelSetFolder '/googlenet_imagenet/bvlc_googlenet.caffemodel']; +% model_def_file = [modelSetFolder '/googlenet_imagenet/deploy.protxt']; + +% netName = 'CAM_alexnet'; +% model_file = [modelSetFolder '/alexnet/CAMmodels/caffeNetCAM_imagenet_train_iter_100000.caffemodel']; +% model_def_file = [modelSetFolder '/alexnet/deploy_caffeNetCAM.prototxt']; + +netName = 'CAM_googlenetBVLCshrink_imagenet'; +model_file = [modelSetFolder '/googlenet_imagenet/CAMmodels/imagenet_googleletCAM_train_iter_80000.caffemodel']; +model_def_file = [modelSetFolder '/googlenet_imagenet/deploy_googlenetCAM.prototxt']; + + +% netName = 'CAM_VGG16_imagenet'; +% model_file = [modelSetFolder '/VGGnet/models/vgg16CAM_train_iter_50000.caffemodel']; +% model_def_file = [modelSetFolder '/VGGnet/deploy_vgg16CAM.prototxt']; + + +%% loading the network +caffe('init', model_def_file, model_file,'test'); +caffe('set_mode_gpu'); +caffe('set_device',0); + +%% testing to predict some image + +weights = caffe('get_weights'); +weights_LR = squeeze(weights(end,1).weights{1,1}); +bias_LR = weights(end,1).weights{2,1}; +layernames = caffe('get_names'); +response = caffe('get_all_layers'); +netInfo = cell(size(layernames,1),3); +for i=1:size(layernames,1) + netInfo{i,1} = layernames{i}; + netInfo{i,2} = i; + netInfo{i,3} = size(response{i,1}); +end + +load('categoriesImageNet.mat'); +d = load('/data/vision/torralba/small-projects/bolei_deep/caffe/ilsvrc_2012_mean.mat'); +IMAGE_MEAN = d.image_mean; +IMAGE_DIM = 256; +CROPPED_DIM = netInfo{1,3}(1); + +weightInfo = cell(size(weights,1),1); +for i=1:size(weights,1) + weightInfo{i,1} = weights(i,1).layer_names; + weightInfo{i,2} = weights(i,1).weights{1,1}; + weightInfo{i,3} = size(weights(i,1).weights{1,1}); +end + +%% testing to predict some image + +datasetName = 'ILSVRCvalSet'; +datasetPath = '/data/vision/torralba/gigaSUN/deeplearning/dataset/ILSVRC2012'; +load([datasetPath '/imageListVal.mat']); +load('sizeImg_ILSVRC2014.mat'); +% datasetName = 'ILSVRCtestSet'; +% datasetPath = '/data/vision/torralba/deeplearning/imagenet_toolkit'; +% load([datasetPath '/imageListTest.mat']); + + + +saveFolder = ['heatMap-' datasetName '-' netName]; +if ~exist(saveFolder) + mkdir(saveFolder); +end +for i=1:5 + if ~exist([saveFolder '/top' num2str(i)]) + mkdir([saveFolder '/top' num2str(i)]); + end +end + +for i = 1:size(imageList,1) + curImgIDX = i; + [a b c] = fileparts(imageList{curImgIDX,1}); + saveMatFile = [saveFolder '/' b '.mat']; + if ~exist(saveMatFile) + height_original = sizeFull_imageList(curImgIDX,1);%tmp.Height; + weight_original = sizeFull_imageList(curImgIDX,2);%tmp.Width; + + + curImg = imread(imageList{curImgIDX,1}); + + if size(curImg,3)==1 + curImg = repmat(curImg,[1 1 3]); + end + + + scores = caffe('forward', {prepare_img(curImg, IMAGE_MEAN, CROPPED_DIM)}); + response = caffe('get_all_layers'); + scoresMean = mean(squeeze(scores{1}),2); + [value_category, IDX_category] = sort(scoresMean,'descend'); + + + featureObjectSwitchSpatial = squeeze(response{end-3,1}); + [curColumnMap] = returnColumnMap(featureObjectSwitchSpatial, weights_LR(:,IDX_category(1:5))); + + + + for j=1:5 + curFeatureMap = squeeze(curColumnMap(:,:,j,:)); + curFeatureMap_crop = imresize(curFeatureMap,[netInfo{1,3}(1) netInfo{1,3}(2)]); + gradients = zeros([netInfo{1,3}(1) netInfo{1,3}(2) 3 10]); + gradients(:,:,1,:) = curFeatureMap_crop; + gradients(:,:,2,:) = curFeatureMap_crop; + gradients(:,:,3,:) = curFeatureMap_crop; + + [alignImgMean alignImgSet] = crop2img(gradients); + alignImgMean = single(alignImgMean); + alignImgMean = imresize(alignImgMean, [height_original weight_original]); + alignImgMean = alignImgMean./max(alignImgMean(:)); + + + imwrite(alignImgMean, [saveFolder '/top' num2str(j) '/' b '.jpg']); + + end + value_category = single(value_category); + IDX_category = single(IDX_category); + save(saveMatFile,'value_category','IDX_category'); + disp([netName ' processing ' b]); + end +end + + +