Skip to content

Commit

Permalink
ILSVRC localization raw scripts are uploaded
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhoubolei
zhoubolei committed Aug 10, 2016
1 parent fa451e6 commit df80e4b
Show file tree
Hide file tree
Showing 2 changed files with 290 additions and 0 deletions.
152 changes: 152 additions & 0 deletions ILSVRC_evaluate_bbox.m
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@


datasetName = 'ILSVRCvalSet';
load('imagenet_toolkit/ILSVRC2014_devkit/evaluation/cache_groundtruth.mat');
load('imagenet_toolkit/ILSVRC2014_devkit/data/meta_clsloc.mat');
datasetPath = 'dataset/ILSVRC2012';
load([datasetPath '/imageListVal.mat']);
load('sizeImg_ILSVRC2014.mat');

% datasetName = 'ILSVRCtestSet';
% datasetPath = '/data/vision/torralba/deeplearning/imagenet_toolkit';
% load([datasetPath '/imageListTest.mat']);


nImgs = size(imageList,1);

ground_truth_file='imagenet_toolkit/ILSVRC2014_devkit/data/ILSVRC2014_clsloc_validation_ground_truth.txt';
gt_labels = dlmread(ground_truth_file);

categories_gt = [];
categoryIDMap = containers.Map();
for i=1:numel(synsets)
categories_gt{synsets(i).ILSVRC2014_ID,1} = synsets(i).words;
categories_gt{synsets(i).ILSVRC2014_ID,2} = synsets(i).WNID;
categoryIDMap(synsets(i).WNID) = i;
end



%% network to evaluate
% backpropa-heatmap
%netName = 'caffeNet_imagenet';
%netName = 'googlenetBVLC_imagenet';
%netName = 'VGG16_imagenet';

% CAM-based network
%netName = 'NIN';
%netName = 'CAM_imagenetCNNaveSumDeep';
%netName = 'CAM_googlenetBVLC_imagenet';% the direct output
netName = 'CAM_googlenetBVLCshrink_imagenet';
%netName = 'CAM_googlenetBVLCshrink_imagenet_maxpool';
%netName = 'CAM_VGG16_imagenet';
%netName = 'CAM_alexnet';

load('categoriesImageNet.mat');

visualizationPointer = 0;

topCategoryNum = 5;
predictionResult_bbox1 = zeros(nImgs, topCategoryNum*5);
predictionResult_bbox2 = zeros(nImgs, topCategoryNum*5);
predictionResult_bboxCombine = zeros(nImgs, topCategoryNum*5);

if matlabpool('size')==0
try
matlabpool
catch e
end
end

heatMapFolder = ['heatMap-' datasetName '-' netName];
bbox_threshold = [20, 100, 110];
curParaThreshold = [num2str(bbox_threshold(1)) ' ' num2str(bbox_threshold(2)) ' ' num2str(bbox_threshold(3))];
parfor i=1:size(imageList,1)
curImgIDX = i;

height_original = sizeFull_imageList(curImgIDX,1);%tmp.Height;
weight_original = sizeFull_imageList(curImgIDX,2);%tmp.Width;

[a b c] = fileparts(imageList{curImgIDX,1});
curPath_fullSizeImg = ['/data/vision/torralba/deeplearning/imagenet_toolkit/ILSVRC2012_img_val/' b c];
curMatFile = [heatMapFolder '/' b '.mat'];
[heatMapSet, value_category, IDX_category] = loadHeatMap( curMatFile);

curResult_bbox1 = [];
curResult_bbox2 = [];
curResult_bboxCombine = [];
for j=1:5
curHeatMapFile = [heatMapFolder '/top' num2str(j) '/' b '.jpg'];

curBBoxFile = [heatMapFolder '/top' num2str(j) '/' b '_default.txt'];
%curBBoxFileGraphcut = [heatMapFolder '/top' num2str(j) '/' b '_graphcut.txt'];
curCategory = categories{IDX_category(j),1};
%imwrite(curHeatMap, ['result_bbox/heatmap_tmp' b randString '.jpg']);
if ~exist(curBBoxFile)
%system(['/data/vision/torralba/deeplearning/package/bbox_hui/final ' curHeatMapFile ' ' curBBoxFile]);

system(['/data/vision/torralba/deeplearning/package/bbox_hui_new/./dt_box ' curHeatMapFile ' ' curParaThreshold ' ' curBBoxFile]);
end
curPredictCategory = categories{IDX_category(j),1};
curPredictCategoryID = categories{IDX_category(j),1}(1:9);
curPredictCategoryGTID = categoryIDMap(curPredictCategoryID);


boxData = dlmread(curBBoxFile);
boxData_formulate = [boxData(1:4:end)' boxData(2:4:end)' boxData(1:4:end)'+boxData(3:4:end)' boxData(2:4:end)'+boxData(4:4:end)'];
boxData_formulate = [min(boxData_formulate(:,1),boxData_formulate(:,3)),min(boxData_formulate(:,2),boxData_formulate(:,4)),max(boxData_formulate(:,1),boxData_formulate(:,3)),max(boxData_formulate(:,2),boxData_formulate(:,4))];

% try
% boxDataGraphcut = dlmread(curBBoxFileGraphcut);
% boxData_formulateGraphcut = [boxDataGraphcut(1:4:end)' boxDataGraphcut(2:4:end)' boxDataGraphcut(1:4:end)'+boxDataGraphcut(3:4:end)' boxDataGraphcut(2:4:end)'+boxDataGraphcut(4:4:end)'];
% catch exception
% boxDataGraphcut = dlmread(curBBoxFile);
% boxData_formulateGraphcut = [boxDataGraphcut(1:4:end)' boxDataGraphcut(2:4:end)' boxDataGraphcut(1:4:end)'+boxDataGraphcut(3:4:end)' boxDataGraphcut(2:4:end)'+boxDataGraphcut(4:4:end)'];
% boxData_formulateGraphcut = boxData_formulateGraphcut(1,:);
% end

bbox = boxData_formulate(1,:);
curPredictTuple = [curPredictCategoryGTID bbox(1) bbox(2) bbox(3) bbox(4)];
curResult_bbox1 = [curResult_bbox1 curPredictTuple];
curResult_bboxCombine = [curResult_bboxCombine curPredictTuple];

bbox = boxData_formulate(2,:);
%bbox = boxData_formulateGraphcut(1,:);
curPredictTuple = [curPredictCategoryGTID bbox(1) bbox(2) bbox(3) bbox(4)];
curResult_bbox2 = [curResult_bbox2 curPredictTuple];

curResult_bboxCombine = [curResult_bboxCombine curPredictTuple];
if visualizationPointer == 1

curHeatMap = imread(curHeatMapFile);
curHeatMap = imresize(curHeatMap,[height_original weight_original]);

subplot(1,2,1),hold off, imshow(curPath_fullSizeImg);
hold on
curBox = boxData_formulate(1,:);
rectangle('Position',[curBox(1) curBox(2) curBox(3)-curBox(1) curBox(4)-curBox(2)],'EdgeColor',[1 0 0]);
subplot(1,2,2),imagesc(curHeatMap);
title(curCategory);
waitforbuttonpress
end
end

predictionResult_bbox1(i, :) = curResult_bbox1;
predictionResult_bbox2(i, :) = curResult_bbox2;
predictionResult_bboxCombine(i,:) = curResult_bboxCombine(1:topCategoryNum*5);
disp([netName ' processing ' b])
end


addpath('imagenet_toolkit/ILSVRC2014_devkit/evaluation');
disp([netName '--------bbox1' ]);
[cls_error, clsloc_error] = simpleEvaluation(predictionResult_bbox1);
disp([(1:5)',clsloc_error,cls_error]);

disp([netName '--------bbox2' ]);
[cls_error, clsloc_error] = simpleEvaluation(predictionResult_bbox2);
disp([(1:5)',clsloc_error,cls_error]);

disp([netName '--------bboxCombine' ]);
[cls_error, clsloc_error] = simpleEvaluation(predictionResult_bboxCombine);
disp([(1:5)',clsloc_error,cls_error]);
138 changes: 138 additions & 0 deletions ILSVRC_generate_heatmap.m
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
% raw script used to generate heatmaps for ILSVRC localization experiment
% please load the necessary packages like matcaffe and ILSVRC toolbox correctly, some functions in matcaffe might be already deprecated.
% you could take it as an example to see how to reproduce the ILSVRC localization experiment.
%
% Bolei Zhou.

addpath('caffeCPU2/matlab/caffe');

modelSetFolder = 'CAMnet';

%% CAMnet


% netName = 'CAM_googlenetBVLC_imagenet';
% model_file = [modelSetFolder '/googlenet_imagenet/bvlc_googlenet.caffemodel'];
% model_def_file = [modelSetFolder '/googlenet_imagenet/deploy.protxt'];

% netName = 'CAM_alexnet';
% model_file = [modelSetFolder '/alexnet/CAMmodels/caffeNetCAM_imagenet_train_iter_100000.caffemodel'];
% model_def_file = [modelSetFolder '/alexnet/deploy_caffeNetCAM.prototxt'];

netName = 'CAM_googlenetBVLCshrink_imagenet';
model_file = [modelSetFolder '/googlenet_imagenet/CAMmodels/imagenet_googleletCAM_train_iter_80000.caffemodel'];
model_def_file = [modelSetFolder '/googlenet_imagenet/deploy_googlenetCAM.prototxt'];


% netName = 'CAM_VGG16_imagenet';
% model_file = [modelSetFolder '/VGGnet/models/vgg16CAM_train_iter_50000.caffemodel'];
% model_def_file = [modelSetFolder '/VGGnet/deploy_vgg16CAM.prototxt'];


%% loading the network
caffe('init', model_def_file, model_file,'test');
caffe('set_mode_gpu');
caffe('set_device',0);

%% testing to predict some image

weights = caffe('get_weights');
weights_LR = squeeze(weights(end,1).weights{1,1});
bias_LR = weights(end,1).weights{2,1};
layernames = caffe('get_names');
response = caffe('get_all_layers');
netInfo = cell(size(layernames,1),3);
for i=1:size(layernames,1)
netInfo{i,1} = layernames{i};
netInfo{i,2} = i;
netInfo{i,3} = size(response{i,1});
end

load('categoriesImageNet.mat');
d = load('/data/vision/torralba/small-projects/bolei_deep/caffe/ilsvrc_2012_mean.mat');
IMAGE_MEAN = d.image_mean;
IMAGE_DIM = 256;
CROPPED_DIM = netInfo{1,3}(1);

weightInfo = cell(size(weights,1),1);
for i=1:size(weights,1)
weightInfo{i,1} = weights(i,1).layer_names;
weightInfo{i,2} = weights(i,1).weights{1,1};
weightInfo{i,3} = size(weights(i,1).weights{1,1});
end

%% testing to predict some image

datasetName = 'ILSVRCvalSet';
datasetPath = '/data/vision/torralba/gigaSUN/deeplearning/dataset/ILSVRC2012';
load([datasetPath '/imageListVal.mat']);
load('sizeImg_ILSVRC2014.mat');
% datasetName = 'ILSVRCtestSet';
% datasetPath = '/data/vision/torralba/deeplearning/imagenet_toolkit';
% load([datasetPath '/imageListTest.mat']);



saveFolder = ['heatMap-' datasetName '-' netName];
if ~exist(saveFolder)
mkdir(saveFolder);
end
for i=1:5
if ~exist([saveFolder '/top' num2str(i)])
mkdir([saveFolder '/top' num2str(i)]);
end
end

for i = 1:size(imageList,1)
curImgIDX = i;
[a b c] = fileparts(imageList{curImgIDX,1});
saveMatFile = [saveFolder '/' b '.mat'];
if ~exist(saveMatFile)
height_original = sizeFull_imageList(curImgIDX,1);%tmp.Height;
weight_original = sizeFull_imageList(curImgIDX,2);%tmp.Width;


curImg = imread(imageList{curImgIDX,1});

if size(curImg,3)==1
curImg = repmat(curImg,[1 1 3]);
end


scores = caffe('forward', {prepare_img(curImg, IMAGE_MEAN, CROPPED_DIM)});
response = caffe('get_all_layers');
scoresMean = mean(squeeze(scores{1}),2);
[value_category, IDX_category] = sort(scoresMean,'descend');


featureObjectSwitchSpatial = squeeze(response{end-3,1});
[curColumnMap] = returnColumnMap(featureObjectSwitchSpatial, weights_LR(:,IDX_category(1:5)));



for j=1:5
curFeatureMap = squeeze(curColumnMap(:,:,j,:));
curFeatureMap_crop = imresize(curFeatureMap,[netInfo{1,3}(1) netInfo{1,3}(2)]);
gradients = zeros([netInfo{1,3}(1) netInfo{1,3}(2) 3 10]);
gradients(:,:,1,:) = curFeatureMap_crop;
gradients(:,:,2,:) = curFeatureMap_crop;
gradients(:,:,3,:) = curFeatureMap_crop;

[alignImgMean alignImgSet] = crop2img(gradients);
alignImgMean = single(alignImgMean);
alignImgMean = imresize(alignImgMean, [height_original weight_original]);
alignImgMean = alignImgMean./max(alignImgMean(:));


imwrite(alignImgMean, [saveFolder '/top' num2str(j) '/' b '.jpg']);

end
value_category = single(value_category);
IDX_category = single(IDX_category);
save(saveMatFile,'value_category','IDX_category');
disp([netName ' processing ' b]);
end
end



0 comments on commit df80e4b

Please sign in to comment.