Merge pull request microsoft#882 from aburgm/additional-drone-cameras

Allow to configure additional drone cameras

AirLib/include/common/AirSimSettings.hpp

@@ -63,6 +63,16 @@ struct AirSimSettings {
         }
     };
 
+    struct AdditionalCameraSetting {
+        // Additional camera positions
+        float x = 0.5f;
+        float y = 0.0f;
+        float z = 0.0f;
+        float yaw = 0.0f;
+        float pitch = 0.0f;
+        float roll = 0.0f;
+    };
+
     struct CaptureSetting {
         //below settinsg are obtained by using Unreal console command (press ~):
         // ShowFlag.VisualizeHDR 1.
@@ -146,6 +156,7 @@ struct AirSimSettings {
 
     std::vector<SubwindowSetting> subwindow_settings;
 
+    std::vector<AdditionalCameraSetting> additional_camera_settings;
     std::map<int, CaptureSetting> capture_settings;
     std::map<int, NoiseSetting>  noise_settings;
 
@@ -194,6 +205,7 @@ struct AirSimSettings {
         loadSubWindowsSettings(settings);
         loadViewModeSettings(settings);
         loadRecordingSettings(settings);
+        loadAdditionalCameraSettings(settings);
         loadCaptureSettings(settings);
         loadCameraNoiseSettings(settings);
         loadSegmentationSettings(settings);
@@ -463,6 +475,26 @@ struct AirSimSettings {
         noise_setting.HorzDistortionContrib = settings.getFloat("HorzDistortionContrib", noise_setting.HorzDistortionContrib);
     }
 
+    void loadAdditionalCameraSettings(const Settings& settings)
+    {
+        Settings json_parent;
+        if (settings.getChild("AdditionalCameras", json_parent)) {
+            for (size_t child_index = 0; child_index < json_parent.size(); ++child_index) {
+                Settings additional_camera_setting;
+                if (json_parent.getChild(child_index, additional_camera_setting)) {
+                    AdditionalCameraSetting setting;
+                    setting.x = additional_camera_setting.getFloat("X", setting.x);
+                    setting.y = additional_camera_setting.getFloat("Y", setting.y);
+                    setting.z = additional_camera_setting.getFloat("Z", setting.z);
+                    setting.yaw = additional_camera_setting.getFloat("Yaw", setting.yaw);
+                    setting.pitch = additional_camera_setting.getFloat("Pitch", setting.pitch);
+                    setting.roll = additional_camera_setting.getFloat("Roll", setting.roll);
+                    additional_camera_settings.push_back(setting);
+                }
+            }
+        }
+    }
+
     void createCaptureSettings(const msr::airlib::Settings& settings, CaptureSetting& capture_setting)
     {
         capture_setting.width = settings.getInt("Width", capture_setting.width);

Unreal/Plugins/AirSim/Source/Multirotor/FlyingPawn.cpp

@@ -8,16 +8,23 @@
 
 AFlyingPawn::AFlyingPawn()
 {
+    static ConstructorHelpers::FClassFinder<APIPCamera> pip_camera_class(TEXT("Blueprint'/AirSim/Blueprints/BP_PIPCamera'"));
+    pip_camera_class_ = pip_camera_class.Succeeded() ? pip_camera_class.Class : nullptr;
+
     wrapper_.reset(new VehiclePawnWrapper());
 }
 
-void AFlyingPawn::initializeForBeginPlay()
+void AFlyingPawn::initializeForBeginPlay(const std::vector<msr::airlib::AirSimSettings::AdditionalCameraSetting>& additionalCameras)
 {
     //get references of components so we can use later
-    setupComponentReferences();
+    setupComponentReferences(additionalCameras);
 
     std::vector<APIPCamera*> cameras = {fpv_camera_front_center_, fpv_camera_front_right_, fpv_camera_front_left_, 
         fpv_camera_bottom_center_, fpv_camera_back_center_};
+    for (APIPCamera* camera : AdditionalCameras) {
+        cameras.push_back(camera);
+    }
+
     wrapper_->initialize(this, cameras);
 }
 
@@ -43,7 +50,7 @@ void AFlyingPawn::setRotorSpeed(int rotor_index, float radsPerSec)
     }
 }
 
-void AFlyingPawn::setupComponentReferences()
+void AFlyingPawn::setupComponentReferences(const std::vector<msr::airlib::AirSimSettings::AdditionalCameraSetting>& additionalCameras)
 {
     fpv_camera_front_right_ = Cast<APIPCamera>(
         (UAirBlueprintLib::GetActorComponent<UChildActorComponent>(this, TEXT("FrontRightCamera")))->GetChildActor());
@@ -56,6 +63,23 @@ void AFlyingPawn::setupComponentReferences()
     fpv_camera_bottom_center_ = Cast<APIPCamera>(
         (UAirBlueprintLib::GetActorComponent<UChildActorComponent>(this, TEXT("BottomCenterCamera")))->GetChildActor());
 
+    //UStaticMeshComponent* bodyMesh = UAirBlueprintLib::GetActorComponent<UStaticMeshComponent>(this, TEXT("BodyMesh"));
+    USceneComponent* bodyMesh = GetRootComponent();
+    FActorSpawnParameters camera_spawn_params;
+    camera_spawn_params.SpawnCollisionHandlingOverride = ESpawnActorCollisionHandlingMethod::AdjustIfPossibleButAlwaysSpawn;
+
+    // Can't obtain NedTransform from wrapper because it's not initialized yet, so make our own.
+    NedTransform transform;
+    transform.initialize(this);
+
+    for (const msr::airlib::AirSimSettings::AdditionalCameraSetting& setting : additionalCameras) {
+        FVector position = transform.toNeuUU(NedTransform::Vector3r(setting.x, setting.y, setting.z)) - transform.toNeuUU(NedTransform::Vector3r(0.0, 0.0, 0.0));
+        FTransform camera_transform(FRotator(setting.pitch, setting.yaw, setting.roll), position, FVector(1., 1., 1.));
+        APIPCamera* camera = GetWorld()->SpawnActor<APIPCamera>(pip_camera_class_, camera_transform, camera_spawn_params);
+        camera->AttachToComponent(bodyMesh, FAttachmentTransformRules::KeepRelativeTransform);
+        AdditionalCameras.Add(camera);
+    }
+
     for (auto i = 0; i < rotor_count; ++i) {
         rotating_movements_[i] = UAirBlueprintLib::GetActorComponent<URotatingMovementComponent>(this, TEXT("Rotation") + FString::FromInt(i));
     }

Unreal/Plugins/AirSim/Source/Multirotor/FlyingPawn.h

@@ -19,15 +19,17 @@ class AIRSIM_API AFlyingPawn : public APawn
     float RotatorFactor = 1.0f;
 
     void setRotorSpeed(int rotor_index, float radsPerSec);
-    void initializeForBeginPlay();
+    void initializeForBeginPlay(const std::vector<msr::airlib::AirSimSettings::AdditionalCameraSetting>& additionalCameras);
     VehiclePawnWrapper* getVehiclePawnWrapper();
 
     virtual void NotifyHit(class UPrimitiveComponent* MyComp, class AActor* Other, class UPrimitiveComponent* OtherComp, bool bSelfMoved, FVector HitLocation,
         FVector HitNormal, FVector NormalImpulse, const FHitResult& Hit) override;
 private: //methods
-    void setupComponentReferences();
+    void setupComponentReferences(const std::vector<msr::airlib::AirSimSettings::AdditionalCameraSetting>& additionalCameras);
 
 private: //variables
+    UPROPERTY() UClass* pip_camera_class_;
+
          //Unreal components
     static constexpr size_t rotor_count = 4;
     UPROPERTY() APIPCamera* fpv_camera_front_left_;
@@ -38,5 +40,7 @@ class AIRSIM_API AFlyingPawn : public APawn
 
     UPROPERTY() URotatingMovementComponent* rotating_movements_[rotor_count];
 
+    UPROPERTY() TArray<APIPCamera*> AdditionalCameras;
+
     std::unique_ptr<VehiclePawnWrapper> wrapper_;
 };

Unreal/Plugins/AirSim/Source/Multirotor/SimModeWorldMultiRotor.cpp

@@ -117,7 +117,7 @@ void ASimModeWorldMultiRotor::setupVehiclesAndCamera(std::vector<VehiclePtr>& ve
         {
             //initialize each vehicle pawn we found
             TMultiRotorPawn* vehicle_pawn = static_cast<TMultiRotorPawn*>(pawn);
-            vehicle_pawn->initializeForBeginPlay();
+            vehicle_pawn->initializeForBeginPlay(getSettings().additional_camera_settings);
 
             //chose first pawn as FPV if none is designated as FPV
             VehiclePawnWrapper* wrapper = vehicle_pawn->getVehiclePawnWrapper();

docs/settings.md

@@ -116,6 +116,9 @@ Below are complete list of settings available along with their default values. I
       "HorzDistortionStrength": 0.002
     }
   ],
+  "AdditionalCameras": [
+    { "X": 0.00, "Y": 0.5, "Z": 0.0, "Roll": 0.0, "Pitch": 0.0, "Yaw": 90.0 }
+  ],
   "PX4": {
     "FirmwareName": "PX4",
     "LogViewerHostIp": "127.0.0.1",
@@ -253,4 +256,11 @@ This adds regions of noise on horizontal lines.
 #### Horizontal line distortion
 This adds fluctuations on horizontal line.
 * `HorzDistortionContrib`: This determines blend ratio of noise pixel with image pixel, 0 means no noise and 1 means only noise.
-* `HorzDistortionStrength`: This determines how large is the distortion.
+* `HorzDistortionStrength`: This determines how large is the distortion.
+
+### Additional Camera Settings
+This allows to configure cameras in addition to the [standard ones](image_apis.md#available-cameras). This is only implemented in the multirotor drone at the moment.
+The X, Y and Z fields specify the location of the new camera in the body frame, where X points forward, Y points to the right, and Z points downwards, and the values are given
+in SI units (meters). Yaw, Pitch, and Roll specify the orientation of the camera, where Yaw denotes rotation around the Z axis, Pitch rotation around the Y axis and Roll rotation around the X axis.
+
+This particular example adds a camera that is mounted on the right side of the drone, pointed to the right. The camera indices of the additional cameras are subsequent to the default ones, so camera index 5 is the first additional camera, camera index 6 the second additional camera, and so on.