Merge pull request Smorodov#64 from Nuzhny007/master

New SSD detector and refactoring

CMakeLists.txt

@@ -26,25 +26,36 @@ set(LIBRARY_OUTPUT_PATH ${CMAKE_BINARY_DIR})
 # ----------------------------------------------------------------------------
 # ищем все .cpp и .h файлы и добавляем их в наш проект
 # ----------------------------------------------------------------------------
-  set(folder_source  main.cpp
-             Detector/Detector.cpp
+  set(folder_source
+             main.cpp
+
+             Detector/BaseDetector.cpp
+             Detector/MotionDetector.cpp
              Detector/BackgroundSubtract.cpp
              Detector/vibe_src/vibe.cpp
              Detector/Subsense/BackgroundSubtractorLBSP.cpp
              Detector/Subsense/BackgroundSubtractorLOBSTER.cpp
              Detector/Subsense/BackgroundSubtractorSuBSENSE.cpp
              Detector/Subsense/LBSP.cpp
+             Detector/FaceDetector.cpp
+             Detector/PedestrianDetector.cpp
+             Detector/pedestrians/c4-pedestrian-detector.cpp
+             Detector/DNNDetector.cpp
 
              Tracker/Ctracker.cpp
              Tracker/track.cpp
              Tracker/HungarianAlg/HungarianAlg.cpp
              Tracker/LocalTracker.cpp
              Tracker/Kalman.cpp)
 
-  set(folder_header MouseExample.h
+  set(folder_header
+             MouseExample.h
              VideoExample.h
              nms.h
-             Detector/Detector.h
+             defines.h
+
+             Detector/BaseDetector.h
+             Detector/MotionDetector.h
              Detector/BackgroundSubtract.h
              Detector/vibe_src/vibe.hpp
              Detector/Subsense/BackgroundSubtractorLBSP.h
@@ -53,14 +64,16 @@ set(LIBRARY_OUTPUT_PATH ${CMAKE_BINARY_DIR})
              Detector/Subsense/DistanceUtils.h
              Detector/Subsense/LBSP.h
              Detector/Subsense/RandUtils.h
+             Detector/FaceDetector.h
+             Detector/PedestrianDetector.h
+             Detector/pedestrians/c4-pedestrian-detector.h
+             Detector/DNNDetector.h
 
-             pedestrians/c4-pedestrian-detector.h
              Tracker/Ctracker.h
              Tracker/track.h
              Tracker/HungarianAlg/HungarianAlg.h
              Tracker/LocalTracker.h
-             Tracker/Kalman.h
-             Tracker/defines.h)
+             Tracker/Kalman.h)
 
   set(graph_source
              Tracker/graph/tokenise.cpp

Detector/BaseDetector.cpp

@@ -0,0 +1,83 @@
+#include "BaseDetector.h"
+#include "MotionDetector.h"
+#include "FaceDetector.h"
+#include "PedestrianDetector.h"
+#include "DNNDetector.h"
+
+///
+/// \brief CreateDetector
+/// \param detectorType
+/// \param collectPoints
+/// \param gray
+/// \return
+///
+BaseDetector* CreateDetector(
+        tracking::Detectors detectorType,
+        bool collectPoints,
+        cv::UMat& gray
+        )
+{
+    switch (detectorType)
+    {
+    case tracking::Motion_VIBE:
+        return new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_VIBE, collectPoints, gray);
+
+    case tracking::Motion_MOG:
+        return new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_MOG, collectPoints, gray);
+
+    case tracking::Motion_GMG:
+        return new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_GMG, collectPoints, gray);
+
+    case tracking::Motion_CNT:
+        return new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_CNT, collectPoints, gray);
+
+    case tracking::Motion_SuBSENSE:
+        return new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_SuBSENSE, collectPoints, gray);
+
+    case tracking::Motion_LOBSTER:
+        return new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_LOBSTER, collectPoints, gray);
+
+    case tracking::Motion_MOG2:
+        return new MotionDetector(BackgroundSubtract::BGFG_ALGS::ALG_MOG2, collectPoints, gray);
+
+    case tracking::Face_HAAR:
+    {
+        FaceDetector* detector = new FaceDetector(collectPoints, gray);
+        if (!detector->Init("../data/haarcascade_frontalface_alt2.xml"))
+        {
+            delete detector;
+            detector = nullptr;
+        }
+        return detector;
+    }
+
+    case tracking::Pedestrian_HOG:
+    case tracking::Pedestrian_C4:
+    {
+        PedestrianDetector* detector = new PedestrianDetector(collectPoints, gray);
+        if (!detector->Init((detectorType == tracking::Pedestrian_HOG) ? PedestrianDetector::HOG : PedestrianDetector::C4,
+                            "../data/combined.txt.model", "../data/combined.txt.model_"))
+        {
+            delete detector;
+            detector = nullptr;
+        }
+        return detector;
+    }
+
+    case tracking::DNN:
+    {
+        DNNDetector* detector = new DNNDetector(collectPoints, gray);
+        if (!detector->Init("../data/MobileNetSSD_deploy.prototxt", "../data/MobileNetSSD_deploy.caffemodel"))
+        {
+            delete detector;
+            detector = nullptr;
+        }
+        return detector;
+    }
+
+
+    default:
+        return nullptr;
+    }
+    return nullptr;
+}

Detector/BaseDetector.h

@@ -0,0 +1,91 @@
+#pragma once
+
+#include <memory>
+#include "defines.h"
+
+///
+/// \brief The BaseDetector class
+///
+class BaseDetector
+{
+public:
+    BaseDetector(bool collectPoints, cv::UMat& frame)
+        : m_collectPoints(collectPoints)
+    {
+        m_minObjectSize.width = std::max(5, frame.cols / 100);
+        m_minObjectSize.height = m_minObjectSize.width;
+    }
+    virtual ~BaseDetector(void)
+    {
+    }
+
+    virtual void Detect(cv::UMat& frame) = 0;
+
+    void SetMinObjectSize(cv::Size minObjectSize)
+    {
+        m_minObjectSize = minObjectSize;
+    }
+
+    const regions_t& GetDetects() const
+    {
+        return m_regions;
+    }
+
+    virtual void CalcMotionMap(cv::Mat frame)
+    {
+        if (m_motionMap.size() != frame.size())
+        {
+            m_motionMap = cv::Mat(frame.size(), CV_32FC1, cv::Scalar(0, 0, 0));
+        }
+
+        cv::Mat foreground(m_motionMap.size(), CV_8UC1, cv::Scalar(0, 0, 0));
+        for (const auto& region : m_regions)
+        {
+            cv::ellipse(foreground,
+                        cv::RotatedRect((region.m_rect.tl() + region.m_rect.br()) / 2, region.m_rect.size(), 0),
+                                        cv::Scalar(255, 255, 255), CV_FILLED);
+        }
+
+        cv::Mat normFor;
+        cv::normalize(foreground, normFor, 255, 0, cv::NORM_MINMAX, m_motionMap.type());
+
+        double alpha = 0.95;
+        cv::addWeighted(m_motionMap, alpha, normFor, 1 - alpha, 0, m_motionMap);
+
+        const int chans = frame.channels();
+
+        for (int y = 0; y < frame.rows; ++y)
+        {
+            uchar* imgPtr = frame.ptr(y);
+            float* moPtr = reinterpret_cast<float*>(m_motionMap.ptr(y));
+            for (int x = 0; x < frame.cols; ++x)
+            {
+                for (int ci = chans - 1; ci < chans; ++ci)
+                {
+                    imgPtr[ci] = cv::saturate_cast<uchar>(imgPtr[ci] + moPtr[0]);
+                }
+                imgPtr += chans;
+                ++moPtr;
+            }
+        }
+    }
+
+protected:
+    regions_t m_regions;
+
+    cv::Size m_minObjectSize;
+
+    bool m_collectPoints;
+
+    cv::Mat m_motionMap;
+};
+
+
+///
+/// \brief CreateDetector
+/// \param detectorType
+/// \param collectPoints
+/// \param gray
+/// \return
+///
+BaseDetector* CreateDetector(tracking::Detectors detectorType, bool collectPoints, cv::UMat& gray);

Detector/DNNDetector.cpp

@@ -0,0 +1,111 @@
+#include "DNNDetector.h"
+
+///
+/// \brief DNNDetector::DNNDetector
+/// \param collectPoints
+/// \param gray
+///
+DNNDetector::DNNDetector(
+	bool collectPoints,
+    cv::UMat& colorFrame
+	)
+    :
+      BaseDetector(collectPoints, colorFrame)
+{
+}
+
+///
+/// \brief DNNDetector::~DNNDetector
+///
+DNNDetector::~DNNDetector(void)
+{
+}
+
+///
+/// \brief DNNDetector::Init
+/// \return
+///
+bool DNNDetector::Init(std::string modelConfiguration, std::string modelBinary)
+{
+    m_net = cv::dnn::readNetFromCaffe(modelConfiguration, modelBinary);
+
+    return !m_net.empty();
+}
+
+///
+/// \brief DNNDetector::Detect
+/// \param gray
+///
+void DNNDetector::Detect(cv::UMat& colorFrame)
+{
+    const int inWidth = 300;
+    const int inHeight = 300;
+    const float WHRatio = inWidth / (float)inHeight;
+    const float inScaleFactor = 0.007843f;
+    const float meanVal = 127.5;
+    const float confidenceThreshold = 0.2f;
+    std::string classNames[] = {"background",
+                                "aeroplane", "bicycle", "bird", "boat",
+                                "bottle", "bus", "car", "cat", "chair",
+                                "cow", "diningtable", "dog", "horse",
+                                "motorbike", "person", "pottedplant",
+                                "sheep", "sofa", "train", "tvmonitor"};
+
+    cv::Size cropSize;
+    if (colorFrame.cols / (float)colorFrame.rows > WHRatio)
+    {
+        cropSize = cv::Size(cvRound(colorFrame.rows * WHRatio), colorFrame.rows);
+    }
+    else
+    {
+        cropSize = cv::Size(colorFrame.cols, cvRound(colorFrame.cols / WHRatio));
+    }
+
+    cv::Rect crop(cv::Point((colorFrame.cols - cropSize.width) / 2, (colorFrame.rows - cropSize.height) / 2), cropSize);
+
+    cv::Mat inputBlob = cv::dnn::blobFromImage(colorFrame.getMat(cv::ACCESS_READ), inScaleFactor, cv::Size(inWidth, inHeight), meanVal, false); //Convert Mat to batch of images
+
+    m_net.setInput(inputBlob, "data"); //set the network input
+
+    cv::Mat detection = m_net.forward("detection_out"); //compute output
+
+    std::vector<double> layersTimings;
+    //double freq = cv::getTickFrequency() / 1000;
+    //double time = m_net.getPerfProfile(layersTimings) / freq;
+
+    cv::Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
+
+    //cv::Mat frame = colorFrame(crop);
+
+    //ss << "FPS: " << 1000/time << " ; time: " << time << " ms";
+    //putText(frame, ss.str(), Point(20,20), 0, 0.5, Scalar(0,0,255));
+    //std::cout << "Inference time, ms: " << time << endl;
+
+    m_regions.clear();
+
+    for (int i = 0; i < detectionMat.rows; ++i)
+    {
+        float confidence = detectionMat.at<float>(i, 2);
+
+        if (confidence > confidenceThreshold)
+        {
+            size_t objectClass = (size_t)(detectionMat.at<float>(i, 1));
+
+            int xLeftBottom = cvRound(detectionMat.at<float>(i, 3) * colorFrame.cols);
+            int yLeftBottom = cvRound(detectionMat.at<float>(i, 4) * colorFrame.rows);
+            int xRightTop = cvRound(detectionMat.at<float>(i, 5) * colorFrame.cols);
+            int yRightTop = cvRound(detectionMat.at<float>(i, 6) * colorFrame.rows);
+
+            cv::Rect object((int)xLeftBottom, (int)yLeftBottom, (int)(xRightTop - xLeftBottom), (int)(yRightTop - yLeftBottom));
+
+            m_regions.push_back(CRegion(object, classNames[objectClass], confidence));
+
+            //cv::rectangle(frame, object, Scalar(0, 255, 0));
+            //std::string label = classNames[objectClass] + ": " + std::to_string(confidence);
+            //int baseLine = 0;
+            //cv::Size labelSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
+            //cv::rectangle(frame, cv::Rect(cv::Point(xLeftBottom, yLeftBottom - labelSize.height), cv::Size(labelSize.width, labelSize.height + baseLine)), cv::Scalar(255, 255, 255), CV_FILLED);
+            //cv::putText(frame, label, Point(xLeftBottom, yLeftBottom), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0,0,0));
+        }
+    }
+}

Detector/DNNDetector.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "BaseDetector.h"
+
+#include <opencv2/dnn.hpp>
+#include <opencv2/dnn/shape_utils.hpp>
+
+///
+/// \brief The DNNDetector class
+///
+class DNNDetector : public BaseDetector
+{
+public:
+    DNNDetector(bool collectPoints, cv::UMat& colorFrame);
+    ~DNNDetector(void);
+
+    bool Init(std::string modelConfiguration = "../data/MobileNetSSD_deploy.prototxt", std::string modelBinary = "../data/MobileNetSSD_deploy.caffemodel");
+
+    void Detect(cv::UMat& colorFrame);
+
+private:
+    cv::dnn::Net m_net;
+};