enable recording images from multiple cameras and types

AirLib/include/controllers/ImageCaptureBase.hpp

@@ -46,6 +46,7 @@ class ImageCaptureBase
         vector<uint8_t> image_data_uint8;
         vector<float> image_data_float;
 
+        int camera_id = -1; //should be filled later
         Vector3r camera_position = Vector3r::Zero();
         Quaternionr camera_orientation = Quaternionr::Identity();
         TTimePoint time_stamp = 0;

Unreal/Plugins/AirSim/Source/Recording/RecordingFile.cpp

@@ -11,27 +11,46 @@ RecordingFile::RecordingFile(std::vector <std::string> columns)
 {
     this->columns = columns;
 }
-void RecordingFile::appendRecord(TArray<uint8>& image_data, VehiclePawnWrapper* wrapper)
-{
-    if (image_data.Num() == 0)
-        return;
-
-    bool imageSavedOk = false;
-    FString filePath;
-
-    std::string filename = std::string("img_").append(std::to_string(images_saved_)).append(".png");
-
-    try {    
-        filePath = FString(common_utils::FileSystem::combine(image_path_, filename).c_str());
-        imageSavedOk = FFileHelper::SaveArrayToFile(image_data, *filePath);
-    }
-    catch(std::exception& ex) {
-        UAirBlueprintLib::LogMessage(TEXT("Image file save failed"), FString(ex.what()), LogDebugLevel::Failure);        
+void RecordingFile::appendRecord(const std::vector<msr::airlib::ImageCaptureBase::ImageResponse>& responses, VehiclePawnWrapper* wrapper)
+{   bool save_success = false;
+    std::stringstream image_file_names;
+
+    for (auto i = 0; i < responses.size(); ++i) {
+        const auto& response = responses.at(i);
+
+        std::stringstream image_file_name;
+        image_file_name << "img_" << response.camera_id << "_" <<
+            common_utils::Utils::toNumeric(response.image_type) << "_" <<
+            common_utils::Utils::getTimeSinceEpochNanos() << 
+            (response.pixels_as_float ? ".pfm" : ".png");
+
+        if (i > 0)
+            image_file_names << ";";
+        image_file_names << image_file_name.str();
+
+        std::string image_full_file_path = common_utils::FileSystem::combine(image_path_, image_file_name.str());
+
+        try {
+            if (response.pixels_as_float) {
+                common_utils::Utils::writePfmFile(response.image_data_float.data(), response.width, response.height,
+                    image_full_file_path);
+            }
+            else {
+                std::ofstream file(image_full_file_path, std::ios::binary);
+                file.write(reinterpret_cast<const char*>(response.image_data_uint8.data()), response.image_data_uint8.size());
+                file.close();
+            }
+
+            save_success = true;
+        }
+        catch(std::exception& ex) {
+            save_success = false;
+            UAirBlueprintLib::LogMessage(TEXT("Image file save failed"), FString(ex.what()), LogDebugLevel::Failure);        
+        }
     }
-    // If render command is complete, save image along with position and orientation
 
-    if (imageSavedOk) {
-        writeString(wrapper->getLogLine().append(filename).append("\n"));
+    if (save_success) {
+        writeString(wrapper->getLogLine().append(image_file_names.str()).append("\n"));
 
         //UAirBlueprintLib::LogMessage(TEXT("Screenshot saved to:"), filePath, LogDebugLevel::Success);
         images_saved_++;

Unreal/Plugins/AirSim/Source/Recording/RecordingFile.h

@@ -13,7 +13,7 @@ class RecordingFile {
     RecordingFile(std::vector <std::string> columns);
     ~RecordingFile();
 
-    void appendRecord(TArray<uint8>& compressedPng, VehiclePawnWrapper* wrapper);
+    void appendRecord(const std::vector<msr::airlib::ImageCaptureBase::ImageResponse>& responses, VehiclePawnWrapper* wrapper);
     void appendColumnHeader(std::vector <std::string> columns);
     void startRecording();
     void stopRecording(bool ignore_if_stopped);

Unreal/Plugins/AirSim/Source/Recording/RecordingSettings.h

@@ -5,4 +5,5 @@ struct RecordingSettings {
     float record_interval = 0.05f;
 
     std::vector<std::string> header_columns;
+    std::vector<msr::airlib::ImageCaptureBase::ImageRequest> requests;
 };

Unreal/Plugins/AirSim/Source/Recording/RecordingThread.cpp

@@ -84,14 +84,10 @@ uint32 FRecordingThread::Run()
 
                 // todo: should we go as fast as possible, or should we limit this to a particular number of
                 // frames per second?
-                std::vector<msr::airlib::ImageCaptureBase::ImageRequest> requests;
+
                 std::vector<msr::airlib::ImageCaptureBase::ImageResponse> responses;
-
-                requests.push_back(msr::airlib::ImageCaptureBase::ImageRequest(0, msr::airlib::ImageCaptureBase::ImageType::Scene, false, true));
-                image_capture_->getImages(requests, responses);
-                TArray<uint8_t> image_data;
-                image_data.Append(responses[0].image_data_uint8.data(), responses[0].image_data_uint8.size());
-                recording_file_->appendRecord(image_data, wrapper_);
+                image_capture_->getImages(settings_.requests, responses);
+                recording_file_->appendRecord(responses, wrapper_);
             }
         }
     }

Unreal/Plugins/AirSim/Source/Recording/RecordingThread.h

@@ -21,13 +21,13 @@ class FRecordingThread : public FRunnable
     static void stopRecording(); 
     static bool isRecording();
 
-private:
-    virtual bool Init();
-    virtual uint32 Run();
-    virtual void Stop();
-    virtual void Exit();
-
+protected:
+    virtual bool Init() override;
+    virtual uint32 Run() override;
+    virtual void Stop() override;
+    virtual void Exit() override;
 
+private:
     void EnsureCompletion();
 
 private:

Unreal/Plugins/AirSim/Source/RenderRequest.cpp

@@ -27,6 +27,7 @@ void RenderRequest::getScreenshot(std::shared_ptr<RenderParams> params[], std::v
             results[i]->bmp.Reset();
         else
             results[i]->bmp_float.Reset();
+        results[i]->time_stamp = 0;
     }
 
     //make sure we are not on the rendering thread
@@ -131,6 +132,8 @@ void RenderRequest::ExecuteTask()
                     );
                 }
             }
+
+            results_[i]->time_stamp = msr::airlib::ClockFactory::get()->nowNanos();
         }
 
         req_size_ = 0;

Unreal/Plugins/AirSim/Source/RenderRequest.h

@@ -3,6 +3,7 @@
 #include "CoreMinimal.h"
 #include <memory>
 #include "common/common_utils/WorkerThread.hpp"
+#include "common/Common.hpp"
 
 class RenderRequest : public FRenderCommand
 {
@@ -26,6 +27,8 @@ class RenderRequest : public FRenderCommand
 
         int width;
         int height;
+
+        msr::airlib::TTimePoint time_stamp;
     };
 
 private:

Unreal/Plugins/AirSim/Source/SimMode/SimModeBase.cpp

@@ -156,7 +156,28 @@ void ASimModeBase::readSettings()
     if (settings.getChild("Recording", record_settings)) {
         recording_settings.record_on_move = record_settings.getBool("RecordOnMove", recording_settings.record_on_move);
         recording_settings.record_interval = record_settings.getFloat("RecordInterval", recording_settings.record_interval);
+
+        Settings req_cameras_settings;
+        if (record_settings.getChild("Cameras", req_cameras_settings)) {
+            for (size_t child_index = 0; child_index < req_cameras_settings.size(); ++child_index) {
+                Settings req_camera_settings;
+                if (req_cameras_settings.getChild(child_index, req_camera_settings)) {
+                    int camera_id = req_camera_settings.getInt("CameraID", 0);
+                    msr::airlib::ImageCaptureBase::ImageType image_type =
+                        common_utils::Utils::toEnum<msr::airlib::ImageCaptureBase::ImageType>(
+                            req_camera_settings.getInt("ImageType", 0));
+                    bool compress = req_camera_settings.getBool("Compress", true);
+                    bool pixels_as_float = req_camera_settings.getBool("PixelsAsFloat", false);
+
+                    recording_settings.requests.push_back(msr::airlib::ImageCaptureBase::ImageRequest(
+                        camera_id, image_type, pixels_as_float, compress));
+                }
+            }
+        }
     }
+    if (recording_settings.requests.size() == 0)
+        recording_settings.requests.push_back(msr::airlib::ImageCaptureBase::ImageRequest(
+            0, msr::airlib::ImageCaptureBase::ImageType::Scene, false, true));
 
     if (simmode_name == "Multirotor") {
         recording_settings.header_columns = std::vector<std::string> {

Unreal/Plugins/AirSim/Source/UnrealImageCapture.cpp

@@ -61,11 +61,12 @@ void UnrealImageCapture::getSceneCaptureImage(const std::vector<msr::airlib::Ima
     render_request.getScreenshot(render_params.data(), render_results, render_params.size());
 
     for (unsigned int i = 0; i < requests.size(); ++i) {
-        const ImageRequest& request = requests[i];
-        ImageResponse& response = responses[i];
+        const ImageRequest& request = requests.at(i);
+        ImageResponse& response = responses.at(i);
         APIPCamera* camera = cameras_[i];
-
-        response.time_stamp = msr::airlib::ClockFactory::get()->nowNanos();
+
+        response.camera_id = request.camera_id;
+        response.time_stamp = render_results[i]->time_stamp;
         response.image_data_uint8 = std::vector<uint8_t>(render_results[i]->image_data_uint8.GetData(), render_results[i]->image_data_uint8.GetData() + render_results[i]->image_data_uint8.Num());
         response.image_data_float = std::vector<float>(render_results[i]->image_data_float.GetData(), render_results[i]->image_data_float.GetData() + render_results[i]->image_data_float.Num());
 

docs/settings.md

@@ -40,7 +40,10 @@ Below are complete list of settings available along with their default values. I
   "EnableCollisionPassthrogh": false,
   "Recording": {
     "RecordOnMove": false,
-    "RecordInterval": 0.05
+    "RecordInterval": 0.05,
+    "Cameras": [
+		  { "CameraID": 0, "ImageType": 0, "PixelsAsFloat": false, "Compress": true }
+	  ]
   },
   "CaptureSettings": [
     {
@@ -162,12 +165,12 @@ This setting determines what is shown in each of 3 subwindows which are visible
     {"WindowID": 2, "ImageType": 6, "CameraID": 4, "Visible": true}
   ]
 ```
-
 #### Recording
 The recording feature allows you to record data such as position, orientation, velocity along with image get recorded in real time at given interval. You can start recording by pressing red Record button on lower right or R key. The data is recorded in `Documents\AirSim` folder, in a timestamped subfolder for each recording session, as csv file.
 
 * RecordInterval: specifies minimal interval in seconds between capturing two images.
 * RecordOnMove: specifies that do not record frame if there was vehicle's position or orientation hasn't changed
+* Cameras: this element controls which images gets recorded. By default scene image from camera 0 is recorded as compressed png format. This setting is json array so you can add more elements that allows to capture images from multiple cameras in different [image types](settings.md#image-capture-settings). When PixelsAsFloat is true, image is saved as [pfm](pfm.md) file instead of png file.
 
 #### ClockSpeed
 Determines the speed of simulation clock with respect to wall clock. For example, value of 5.0 would mean simulation clock has 5 seconds elapsed when wall clock has 1 second elapsed (i.e. simulation is running faster). The value of 0.1 means that simulation clock is 10X slower than wall clock. The value of 1 means simulation is running in real time. It is important to realize that quality of simuation may decrease as the simulation clock runs faster. You might see artifacts like object moving past obstacles because collison is not detected. However slowing down simulation clock (i.e. values < 1.0) generally improves the quality of simulation.