Merge pull request leggedrobotics#308 from leggedrobotics/noetic

Noetic

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "darknet"]
 	path = darknet
-	url = https://github.com/pjreddie/darknet
+	url = https://github.com/leggedrobotics/darknet

README.md

@@ -2,9 +2,11 @@
 
 ## Overview
 
-This is a ROS package developed for object detection in camera images. You only look once (YOLO) is a state-of-the-art, real-time object detection system. In the following ROS package you are able to use YOLO (V3) on GPU and CPU. The pre-trained model of the convolutional neural network is able to detect pre-trained classes including the data set from VOC and COCO, or you can also create a network with your own detection objects. For more information about YOLO, Darknet, available training data and training YOLO see the following link: [YOLO: Real-Time Object Detection](http://pjreddie.com/darknet/yolo/).
+This is a ROS package developed for **object detection in camera images**. You only look once (YOLO) is a state-of-the-art, real-time object detection system. In the following ROS package you are able to use **YOLO (V3) on GPU and CPU**. The pre-trained model of the convolutional neural network is able to detect pre-trained classes including the data set from VOC and COCO, or you can also create a network with your own detection objects. For more information about YOLO, Darknet, available training data and training YOLO see the following link: [YOLO: Real-Time Object Detection](http://pjreddie.com/darknet/yolo/).
 
-The YOLO packages have been tested under ROS Melodic and Ubuntu 18.04. This is research code, expect that it changes often and any fitness for a particular purpose is disclaimed.
+The YOLO packages have been tested under **ROS Noetic** and **Ubuntu 20.04**. Note: We also provide branches that work under **ROS Melodic**, **ROS Foxy** and **ROS2**.
+
+This is research code, expect that it changes often and any fitness for a particular purpose is disclaimed.
 
 **Author: [Marko Bjelonic](https://www.markobjelonic.com), [email protected]**
 

darknet_ros/CMakeLists.txt

@@ -1,8 +1,8 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 3.5.1)
 project(darknet_ros)
 
 # Set c++11 cmake flags
-set(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS}")
+set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_C_FLAGS "-Wall -Wno-unused-result -Wno-unknown-pragmas -Wno-unused-variable -Wfatal-errors -fPIC ${CMAKE_C_FLAGS}")
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
@@ -92,7 +92,7 @@ include_directories(
 )
 
 set(PROJECT_LIB_FILES
-    src/YoloObjectDetector.cpp                    src/image_interface.c
+    src/YoloObjectDetector.cpp                    src/image_interface.cpp
 )
 
 set(DARKNET_CORE_FILES
@@ -118,9 +118,10 @@ set(DARKNET_CORE_FILES
     ${DARKNET_PATH}/src/gru_layer.c               ${DARKNET_PATH}/src/utils.c
     ${DARKNET_PATH}/src/upsample_layer.c          ${DARKNET_PATH}/src/logistic_layer.c
     ${DARKNET_PATH}/src/l2norm_layer.c            ${DARKNET_PATH}/src/yolo_layer.c
+    ${DARKNET_PATH}/src/iseg_layer.c              ${DARKNET_PATH}/src/image_opencv.cpp
 
     ${DARKNET_PATH}/examples/art.c                ${DARKNET_PATH}/examples/lsd.c
-    ${DARKNET_PATH}/examples/attention.c          ${DARKNET_PATH}/examples/nightmare.c
+    ${DARKNET_PATH}/examples/nightmare.c          ${DARKNET_PATH}/examples/instance-segmenter.c
     ${DARKNET_PATH}/examples/captcha.c            ${DARKNET_PATH}/examples/regressor.c
     ${DARKNET_PATH}/examples/cifar.c              ${DARKNET_PATH}/examples/rnn.c
     ${DARKNET_PATH}/examples/classifier.c         ${DARKNET_PATH}/examples/segmenter.c

darknet_ros/include/darknet_ros/YoloObjectDetector.hpp

@@ -49,17 +49,19 @@ extern "C" {
 #include <sys/time.h>
 #include "box.h"
 #include "cost_layer.h"
-#include "darknet_ros/image_interface.h"
 #include "detection_layer.h"
 #include "network.h"
 #include "parser.h"
 #include "region_layer.h"
 #include "utils.h"
 }
 
-extern "C" void ipl_into_image(IplImage* src, image im);
-extern "C" image ipl_to_image(IplImage* src);
-extern "C" void show_image_cv(image p, const char* name, IplImage* disp);
+// Image interface.
+#include "darknet_ros/image_interface.hpp"
+
+extern "C" cv::Mat image_to_mat(image im);
+extern "C" image mat_to_image(cv::Mat m);
+extern "C" int show_image(image p, const char* name, int ms);
 
 namespace darknet_ros {
 
@@ -70,9 +72,9 @@ typedef struct {
 } RosBox_;
 
 typedef struct {
-  IplImage* image;
+  cv::Mat image;
   std_msgs::Header header;
-} IplImageWithHeader_;
+} CvMatWithHeader_;
 
 class YoloObjectDetector {
  public:
@@ -174,12 +176,11 @@ class YoloObjectDetector {
   image buffLetter_[3];
   int buffId_[3];
   int buffIndex_ = 0;
-  IplImage* ipl_;
   float fps_ = 0;
   float demoThresh_ = 0;
   float demoHier_ = .5;
   int running_ = 0;
-
+  cv::Mat disp_;
   int demoDelay_ = 0;
   int demoFrame_ = 3;
   float** predictions_;
@@ -234,7 +235,7 @@ class YoloObjectDetector {
 
   void yolo();
 
-  IplImageWithHeader_ getIplImageWithHeader();
+  CvMatWithHeader_ getCvMatWithHeader();
 
   bool getImageStatus(void);
 

darknet_ros/include/darknet_ros/image_interface.h → darknet_ros/include/darknet_ros/image_interface.hpp

@@ -10,9 +10,10 @@
 #define IMAGE_INTERFACE_H
 
 #include "image.h"
+#include "opencv2/opencv.hpp"
 
 static float get_pixel(image m, int x, int y, int c);
 image** load_alphabet_with_file(char* datafile);
-void generate_image(image p, IplImage* disp);
+void generate_image(image p, cv::Mat& disp);
 
 #endif

darknet_ros/src/YoloObjectDetector.cpp

@@ -368,9 +368,8 @@ void* YoloObjectDetector::detectInThread() {
 void* YoloObjectDetector::fetchInThread() {
   {
     boost::shared_lock<boost::shared_mutex> lock(mutexImageCallback_);
-    IplImageWithHeader_ imageAndHeader = getIplImageWithHeader();
-    IplImage* ROS_img = imageAndHeader.image;
-    ipl_into_image(ROS_img, buff_[buffIndex_]);
+    CvMatWithHeader_ imageAndHeader = getCvMatWithHeader();
+    buff_[buffIndex_] = mat_to_image(imageAndHeader.image);
     headerBuff_[buffIndex_] = imageAndHeader.header;
     buffId_[buffIndex_] = actionId_;
   }
@@ -380,8 +379,7 @@ void* YoloObjectDetector::fetchInThread() {
 }
 
 void* YoloObjectDetector::displayInThread(void* ptr) {
-  show_image_cv(buff_[(buffIndex_ + 1) % 3], "YOLO V3", ipl_);
-  int c = cv::waitKey(waitKeyDelay_);
+  int c = show_image(buff_[(buffIndex_ + 1) % 3], "YOLO", 1);
   if (c != -1) c = c % 256;
   if (c == 27) {
     demoDone_ = 1;
@@ -424,7 +422,7 @@ void YoloObjectDetector::setupNetwork(char* cfgfile, char* weightfile, char* dat
   demoThresh_ = thresh;
   demoHier_ = hier;
   fullScreen_ = fullscreen;
-  printf("YOLO V3\n");
+  printf("YOLO\n");
   net_ = load_network(cfgfile, weightfile, 0);
   set_batch_network(net_, 1);
 }
@@ -457,9 +455,8 @@ void YoloObjectDetector::yolo() {
 
   {
     boost::shared_lock<boost::shared_mutex> lock(mutexImageCallback_);
-    IplImageWithHeader_ imageAndHeader = getIplImageWithHeader();
-    IplImage* ROS_img = imageAndHeader.image;
-    buff_[0] = ipl_to_image(ROS_img);
+    CvMatWithHeader_ imageAndHeader = getCvMatWithHeader();
+    buff_[0] = mat_to_image(imageAndHeader.image);
     headerBuff_[0] = imageAndHeader.header;
   }
   buff_[1] = copy_image(buff_[0]);
@@ -469,17 +466,16 @@ void YoloObjectDetector::yolo() {
   buffLetter_[0] = letterbox_image(buff_[0], net_->w, net_->h);
   buffLetter_[1] = letterbox_image(buff_[0], net_->w, net_->h);
   buffLetter_[2] = letterbox_image(buff_[0], net_->w, net_->h);
-  ipl_ = cvCreateImage(cvSize(buff_[0].w, buff_[0].h), IPL_DEPTH_8U, buff_[0].c);
+  disp_ = image_to_mat(buff_[0]);
 
   int count = 0;
-
   if (!demoPrefix_ && viewImage_) {
-    cv::namedWindow("YOLO V3", cv::WINDOW_NORMAL);
+    cv::namedWindow("YOLO", cv::WINDOW_NORMAL);
     if (fullScreen_) {
-      cv::setWindowProperty("YOLO V3", cv::WND_PROP_FULLSCREEN, cv::WINDOW_FULLSCREEN);
+      cv::setWindowProperty("YOLO", cv::WND_PROP_FULLSCREEN, cv::WINDOW_FULLSCREEN);
     } else {
-      cv::moveWindow("YOLO V3", 0, 0);
-      cv::resizeWindow("YOLO V3", 640, 480);
+      cv::moveWindow("YOLO", 0, 0);
+      cv::resizeWindow("YOLO", 640, 480);
     }
   }
 
@@ -495,7 +491,7 @@ void YoloObjectDetector::yolo() {
       if (viewImage_) {
         displayInThread(0);
       } else {
-        generate_image(buff_[(buffIndex_ + 1) % 3], ipl_);
+        generate_image(buff_[(buffIndex_ + 1) % 3], disp_);
       }
       publishInThread();
     } else {
@@ -512,9 +508,8 @@ void YoloObjectDetector::yolo() {
   }
 }
 
-IplImageWithHeader_ YoloObjectDetector::getIplImageWithHeader() {
-  IplImage* ROS_img = new IplImage(camImageCopy_);
-  IplImageWithHeader_ header = {.image = ROS_img, .header = imageHeader_};
+CvMatWithHeader_ YoloObjectDetector::getCvMatWithHeader() {
+  CvMatWithHeader_ header = {.image = camImageCopy_, .header = imageHeader_};
   return header;
 }
 
@@ -530,7 +525,7 @@ bool YoloObjectDetector::isNodeRunning(void) {
 
 void* YoloObjectDetector::publishInThread() {
   // Publish image.
-  cv::Mat cvImage = cv::cvarrToMat(ipl_);
+  cv::Mat cvImage = disp_;
   if (!publishDetectionImage(cv::Mat(cvImage))) {
     ROS_DEBUG("Detection image has not been broadcasted.");
   }

darknet_ros/src/image_interface.c → darknet_ros/src/image_interface.cpp

@@ -6,7 +6,7 @@
  *   Institute: ETH Zurich, Robotic Systems Lab
  */
 
-#include "darknet_ros/image_interface.h"
+#include "darknet_ros/image_interface.hpp"
 
 static float get_pixel(image m, int x, int y, int c) {
   assert(x < m.w && y < m.h && c < m.c);
@@ -16,13 +16,13 @@ static float get_pixel(image m, int x, int y, int c) {
 image** load_alphabet_with_file(char* datafile) {
   int i, j;
   const int nsize = 8;
-  image** alphabets = calloc(nsize, sizeof(image));
+  image** alphabets = (image**)calloc(nsize, sizeof(image));
   char* labels = "/labels/%d_%d.png";
   char* files = (char*)malloc(1 + strlen(datafile) + strlen(labels));
   strcpy(files, datafile);
   strcat(files, labels);
   for (j = 0; j < nsize; ++j) {
-    alphabets[j] = calloc(128, sizeof(image));
+    alphabets[j] = (image*)calloc(128, sizeof(image));
     for (i = 32; i < 127; ++i) {
       char buff[256];
       sprintf(buff, files, i, j);
@@ -33,16 +33,16 @@ image** load_alphabet_with_file(char* datafile) {
 }
 
 #ifdef OPENCV
-void generate_image(image p, IplImage* disp) {
+void generate_image(image p, cv::Mat& disp) {
   int x, y, k;
   if (p.c == 3) rgbgr_image(p);
   // normalize_image(copy);
 
-  int step = disp->widthStep;
+  int step = disp.step;
   for (y = 0; y < p.h; ++y) {
     for (x = 0; x < p.w; ++x) {
       for (k = 0; k < p.c; ++k) {
-        disp->imageData[y * step + x * p.c + k] = (unsigned char)(get_pixel(p, x, y, k) * 255);
+        disp.data[y * step + x * p.c + k] = (unsigned char)(get_pixel(p, x, y, k) * 255);
       }
     }
   }

darknet_ros/test/ObjectDetection.cpp

@@ -70,7 +70,7 @@ bool sendImageToYolo(ros::NodeHandle nh, const std::string& pathToTestImage) {
 
   // Get test image
   cv_bridge::CvImagePtr cv_ptr(new cv_bridge::CvImage);
-  cv_ptr->image = cv::imread(pathToTestImage, CV_LOAD_IMAGE_COLOR);
+  cv_ptr->image = cv::imread(pathToTestImage, cv::IMREAD_COLOR);
   cv_ptr->encoding = sensor_msgs::image_encodings::RGB8;
   sensor_msgs::ImagePtr image = cv_ptr->toImageMsg();
 
@@ -154,7 +154,8 @@ TEST(ObjectDetection, DetectANYmal) {
   pathToTestImage += "quadruped_anymal_and_person";
   pathToTestImage += ".JPG";
 
-  // Send dog image to yolo.
+  // Send ANYmal and person image to yolo.
+  ASSERT_TRUE(sendImageToYolo(nodeHandle, pathToTestImage));
   ASSERT_TRUE(sendImageToYolo(nodeHandle, pathToTestImage));
   ASSERT_TRUE(sendImageToYolo(nodeHandle, pathToTestImage));