Fix getImageForCamera so that it automatically turns on the required …

…view if it is not visible already.

.gitignore

@@ -234,6 +234,9 @@ FakesAssemblies/
 # Visual Studio 6 workspace options file
 *.opt
 
+# compiled python files
+*.pyc
+
 # LightSwitch generated files
 GeneratedArtifacts/
 _Pvt_Extensions/
@@ -313,3 +316,4 @@ deps/
 /cmake-3.7.2-win64-x64
 /cmake-3.7.2-win64-x64.zip
 /3.3.2.zip
+

DroneShell/src/main.cpp

@@ -1201,7 +1201,7 @@ class GetImageCommand : public DroneCommand {
 
             std::cout << "Image saved to: " << file_path_name << " (" << image.size() << " bytes)" << endl;
 
-            context->sleep_for(pause_time);
+            context->sleep_for(pause_time / 1000);
         }
 
     }

PythonClient/PythonClient.py

@@ -77,6 +77,8 @@ def moveOnPath(self, path, velocity, max_wait_seconds, drivetrain, yaw_mode, loo
             return self.client.call('moveOnPath', path, velocity, max_wait_seconds, drivetrain, (yaw_mode.is_rate, yaw_mode.yaw_or_rate), lookahead, adaptive_lookahead)
         def moveToZ(self, z, velocity, max_wait_seconds, yaw_mode, lookahead, adaptive_lookahead):
             return self.client.call('moveToZ', z, velocity, max_wait_seconds, (yaw_mode.is_rate, yaw_mode.yaw_or_rate), lookahead, adaptive_lookahead)
+        def moveToPosition(self, x, y, z, velocity, max_wait_seconds, drivetrain, yaw_mode, lookahead, adaptive_lookahead):
+            return self.client.call('moveToPosition', x, y, z, velocity, max_wait_seconds, drivetrain, (yaw_mode.is_rate, yaw_mode.yaw_or_rate), lookahead, adaptive_lookahead)
         def rotateToYaw(self, yaw, max_wait_seconds, margin):
             return self.client.call('rotateToYaw', yaw, max_wait_seconds, margin)
         def rotateByYawRate(self, yaw_rate, duration):

Unreal/Plugins/AirSim/Source/PIPCamera.cpp

@@ -177,55 +177,3 @@ void APIPCamera::setEnableCameraTypes(EPIPCameraType types)
 {
     enabled_camera_types_ = types;
 }
-
-//
-//bool APIPCamera::getScreenshot(EPIPCameraType camera_type, TArray<uint8>& compressedPng, float& width, float& height)
-//{
-//    USceneCaptureComponent2D* capture = getCaptureComponent(camera_type, true);;
-//
-//    if (capture == nullptr) {
-//        UAirBlueprintLib::LogMessage(TEXT("Can't take screenshot because eithercamera type is not active"), TEXT(""), LogDebugLevel::Failure);
-//        return false;
-//    }
-//
-//    if (capture->TextureTarget == nullptr) {
-//        UAirBlueprintLib::LogMessage(TEXT("Can't take screenshot because texture target is null"), TEXT(""), LogDebugLevel::Failure);
-//        return false;
-//    }
-//
-//    FTextureRenderTargetResource* resource = capture->TextureTarget->GameThread_GetRenderTargetResource();
-//
-//    if (resource == nullptr) {
-//        UAirBlueprintLib::LogMessage(TEXT("Can't take screenshot because texture target resource is not available"), TEXT(""), LogDebugLevel::Failure);
-//        return false;
-//    }
-//
-//    width = capture->TextureTarget->GetSurfaceWidth();
-//    height = capture->TextureTarget->GetSurfaceHeight();
-//
-//    TArray<FColor> bmp;
-//    bmp.AddUninitialized(width * height);
-//
-//    resource->ReadPixels(bmp);
-//
-//    FIntPoint dest(width, height);
-//    FImageUtils::CompressImageArray(dest.X, dest.Y, bmp, compressedPng);
-//
-//    return true;
-//}
-//
-//void APIPCamera::saveScreenshot(EPIPCameraType camera_type, FString fileSavePathPrefix, int fileSuffix)
-//{
-//    TArray<uint8> compressedPng;
-//    float width, height;
-//    if (getScreenshot(camera_type, compressedPng, width, height)) {
-//        FString filePath = fileSavePathPrefix + FString::FromInt(fileSuffix) + ".png";
-//        bool imageSavedOk = FFileHelper::SaveArrayToFile(compressedPng, *filePath);
-//
-//        if (!imageSavedOk)
-//            UAirBlueprintLib::LogMessage(TEXT("File save failed to:"), filePath, LogDebugLevel::Failure);
-//        else {
-//            UAirBlueprintLib::LogMessage(TEXT("Screenshot saved to:"), filePath, LogDebugLevel::Success);
-//        }
-//    }
-//}

Unreal/Plugins/AirSim/Source/RenderRequest.cpp

@@ -25,8 +25,8 @@ void RenderRequest::getScreenshot(UTextureRenderTarget2D* renderTarget, TArray<u
 
     //make sure we are not on the rendering thread
     CheckNotBlockedOnRenderThread();
-    // Queue up the task of rendering the scene in the render thread
 
+    // Queue up the task of rendering the scene in the render thread
     TGraphTask<RenderRequest>::CreateTask().ConstructAndDispatchWhenReady(*this);
 
     // wait for this task to complete

Unreal/Plugins/AirSim/Source/SimHUD/SimHUD.cpp

@@ -3,10 +3,13 @@
 #include "SimMode/SimModeWorldMultiRotor.h"
 #include "Kismet/KismetSystemLibrary.h"
 
+ASimHUD* ASimHUD::instance_ = nullptr;
+
 ASimHUD::ASimHUD()
 {
     static ConstructorHelpers::FClassFinder<UUserWidget> hud_widget_class(TEXT("WidgetBlueprint'/AirSim/Blueprints/BP_SimHUDWidget'"));
     widget_class_ = hud_widget_class.Succeeded() ? hud_widget_class.Class : nullptr;
+    instance_ = this;
 }
 
 void ASimHUD::BeginPlay()
@@ -81,6 +84,23 @@ void ASimHUD::inputEventToggleTrace()
     simmode_->CameraDirector->TargetPawn->toggleTrace();
 }
 
+
+bool ASimHUD::isPIPSceneVisible()
+{
+    return widget_->getPIPSceneVisibility();
+}
+
+bool ASimHUD::isPIPDepthVisible()
+{
+    return widget_->getPIPDepthVisibility();
+}
+
+bool ASimHUD::isPIPSegVisible()
+{
+    return widget_->getPIPSegVisibility();
+}
+
+
 void ASimHUD::inputEventTogglePIPScene()
 {
     bool is_visible = simmode_->CameraDirector->togglePIPScene();

Unreal/Plugins/AirSim/Source/SimHUD/SimHUD.h

@@ -26,11 +26,18 @@ class AIRSIM_API ASimHUD : public AHUD
     void inputEventTogglePIPSeg();
     void inputEventToggleAll();
 
+    bool isPIPSceneVisible();
+    bool isPIPDepthVisible();
+    bool isPIPSegVisible();
+
     ASimHUD();
     virtual void BeginPlay() override;
     virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
     virtual void Tick( float DeltaSeconds ) override;
 
+    static ASimHUD* GetInstance() {
+        return instance_;
+    }
 protected:
     virtual void setupInputBindings();
     std::string reportRefreshHandler();
@@ -44,4 +51,5 @@ class AIRSIM_API ASimHUD : public AHUD
     UPROPERTY()
     ASimModeBase* simmode_;
 
+    static ASimHUD* instance_;
 };

Unreal/Plugins/AirSim/Source/SimHUD/SimHUDWidget.h

@@ -36,10 +36,16 @@ class AIRSIM_API USimHUDWidget : public UUserWidget
     //is not there then below are treated as events instead of overridable functions
     UFUNCTION(BlueprintImplementableEvent, Category = "C++ Interface")
         bool setPIPSceneVisibility(bool is_viisble);
+    UFUNCTION(BlueprintImplementableEvent, Category = "C++ Interface")
+        bool getPIPSceneVisibility();
     UFUNCTION(BlueprintImplementableEvent, Category = "C++ Interface")
         bool setPIPDepthVisibility(bool is_viisble);
+    UFUNCTION(BlueprintImplementableEvent, Category = "C++ Interface")
+        bool getPIPDepthVisibility();
     UFUNCTION(BlueprintImplementableEvent, Category = "C++ Interface")
         bool setPIPSegVisibility(bool is_viisble);
+    UFUNCTION(BlueprintImplementableEvent, Category = "C++ Interface")
+        bool getPIPSegVisibility();
 
 protected:
     UFUNCTION(BlueprintImplementableEvent, Category = "C++ Interface")

Unreal/Plugins/AirSim/Source/SimMode/SimModeWorldMultiRotor.cpp

@@ -192,7 +192,6 @@ void ASimModeWorldMultiRotor::createVehicles(std::vector<VehiclePtr>& vehicles)
 ASimModeWorldBase::VehiclePtr ASimModeWorldMultiRotor::createVehicle(AFlyingPawn* pawn)
 {
     vehicle_params_ = MultiRotorParamsFactory::createConfig(fpv_vehicle_name);
-
     auto vehicle = std::make_shared<MultiRotorConnector>();
     vehicle->initialize(pawn, vehicle_params_.get(), enable_rpc, api_server_address);
     return std::static_pointer_cast<VehicleConnectorBase>(vehicle);

Unreal/Plugins/AirSim/Source/VehicleCameraConnector.h

@@ -2,6 +2,8 @@
 #include "PIPCamera.h"
 #include "controllers/VehicleCamera.hpp"
 #include "RenderRequest.h"
+#include "SimHUD/SimHUD.h"
+#include <chrono>
 
 class VehicleCameraConnector : public VehicleCamera
 {
@@ -26,19 +28,45 @@ class VehicleCameraConnector : public VehicleCamera
         if (imageType == DroneControllerBase::ImageType::None) {
             return false;
         }
+        ASimHUD* hud = ASimHUD::GetInstance();
         EPIPCameraType pip_type;
+        bool visibilityChanged = false;
         //TODO: merge these two different types?
         switch (imageType) {
         case DroneControllerBase::ImageType::Scene:
-            pip_type = EPIPCameraType::PIP_CAMERA_TYPE_SCENE; break;
+            pip_type = EPIPCameraType::PIP_CAMERA_TYPE_SCENE; 
+            if (!hud->isPIPSceneVisible()) {
+                hud->inputEventTogglePIPScene();
+                visibilityChanged = true;
+            }
+            break;
         case DroneControllerBase::ImageType::Depth:
-            pip_type = EPIPCameraType::PIP_CAMERA_TYPE_DEPTH; break;
+            pip_type = EPIPCameraType::PIP_CAMERA_TYPE_DEPTH;
+            if (!hud->isPIPDepthVisible()) {
+                hud->inputEventTogglePIPDepth();
+                visibilityChanged = true;
+            }
+            break;
         case DroneControllerBase::ImageType::Segmentation:
-            pip_type = EPIPCameraType::PIP_CAMERA_TYPE_SEG; break;
+            pip_type = EPIPCameraType::PIP_CAMERA_TYPE_SEG;
+            if (!hud->isPIPSegVisible()) {
+                hud->inputEventTogglePIPSeg();
+                visibilityChanged = true;
+            }
+            break;
         default:
             pip_type = EPIPCameraType::PIP_CAMERA_TYPE_NONE;
         }
 
+        if (visibilityChanged) {
+            // Wait for render so that view is ready for capture
+            std::this_thread::sleep_for(std::chrono::duration<double>(0.2));
+            // not sure why this doesn't work.
+            //DECLARE_CYCLE_STAT(TEXT("FNullGraphTask.CheckRenderStatus"), STAT_FNullGraphTask_CheckRenderStatus, STATGROUP_TaskGraphTasks);
+            //auto renderStatus = TGraphTask<FNullGraphTask>::CreateTask(NULL).ConstructAndDispatchWhenReady(GET_STATID(STAT_FNullGraphTask_CheckRenderStatus), ENamedThreads::RenderThread);
+            //FTaskGraphInterface::Get().WaitUntilTaskCompletes(renderStatus);
+        }
+
         using namespace msr::airlib;
         USceneCaptureComponent2D* capture = camera->getCaptureComponent(pip_type, true);
         if (capture == nullptr) {