add driver and rear views, subwindow settings improvement

Unreal/Plugins/AirSim/Source/Car/CarPawn.cpp

@@ -221,14 +221,21 @@ ACarPawn::ACarPawn()
 
     // Create In-Car camera component 
     InternalCameraBase1 = CreateDefaultSubobject<USceneComponent>(TEXT("InternalCameraBase1"));
-    InternalCameraBase1->SetRelativeLocation(FVector(-34.0f, 0, 50.0f));
+    InternalCameraBase1->SetRelativeLocation(FVector(36.0f, 0, 50.0f)); //center
     InternalCameraBase1->SetupAttachment(GetMesh());
     InternalCameraBase2 = CreateDefaultSubobject<USceneComponent>(TEXT("InternalCameraBase2"));
-    InternalCameraBase2->SetRelativeLocation(FVector(-34.0f, -10, 50.0f));
+    InternalCameraBase2->SetRelativeLocation(FVector(36.0f, -10, 50.0f)); //left
     InternalCameraBase2->SetupAttachment(GetMesh());
     InternalCameraBase3 = CreateDefaultSubobject<USceneComponent>(TEXT("InternalCameraBase3"));
-    InternalCameraBase3->SetRelativeLocation(FVector(-34.0f, 10, 50.0f));
+    InternalCameraBase3->SetRelativeLocation(FVector(36.0f, 10, 50.0f)); //right
     InternalCameraBase3->SetupAttachment(GetMesh());
+    InternalCameraBase4 = CreateDefaultSubobject<USceneComponent>(TEXT("InternalCameraBase4"));
+    InternalCameraBase4->SetRelativeLocation(FVector(25, -10, 75.0f)); //driver
+    InternalCameraBase4->SetupAttachment(GetMesh());
+    InternalCameraBase5 = CreateDefaultSubobject<USceneComponent>(TEXT("InternalCameraBase5"));
+    InternalCameraBase5->SetRelativeLocation(FVector(-36.0f, 0, 50.0f)); //rear
+    InternalCameraBase5->SetRelativeRotation(FRotator(0, 180, 0));
+    InternalCameraBase5->SetupAttachment(GetMesh());
 
     // In car HUD
     // Create text render component for in car speed display
@@ -278,7 +285,7 @@ void ACarPawn::initializeForBeginPlay(bool enable_rpc, const std::string& api_se
         EngineSoundComponent->Deactivate(); 
 
     //put camera little bit above vehicle
-    FTransform camera_transform(FVector(0, 0, 0));
+    FTransform camera_transform(FVector::ZeroVector);
     FActorSpawnParameters camera_spawn_params;
     camera_spawn_params.SpawnCollisionHandlingOverride = ESpawnActorCollisionHandlingMethod::AdjustIfPossibleButAlwaysSpawn;
     InternalCamera1 = this->GetWorld()->SpawnActor<APIPCamera>(pip_camera_class_, camera_transform, camera_spawn_params);
@@ -287,10 +294,15 @@ void ACarPawn::initializeForBeginPlay(bool enable_rpc, const std::string& api_se
     InternalCamera2->AttachToComponent(InternalCameraBase2, FAttachmentTransformRules::KeepRelativeTransform);
     InternalCamera3 = this->GetWorld()->SpawnActor<APIPCamera>(pip_camera_class_, camera_transform, camera_spawn_params);
     InternalCamera3->AttachToComponent(InternalCameraBase3, FAttachmentTransformRules::KeepRelativeTransform);
+    InternalCamera4 = this->GetWorld()->SpawnActor<APIPCamera>(pip_camera_class_, camera_transform, camera_spawn_params);
+    InternalCamera4->AttachToComponent(InternalCameraBase4, FAttachmentTransformRules::KeepRelativeTransform);
+    InternalCamera5 = this->GetWorld()->SpawnActor<APIPCamera>(pip_camera_class_, FTransform(FRotator(0, 180, 0), FVector::ZeroVector), camera_spawn_params);
+    InternalCamera5->AttachToComponent(InternalCameraBase4, FAttachmentTransformRules::KeepRelativeTransform);
+
 
     setupInputBindings();
 
-    std::vector<APIPCamera*> cameras = { InternalCamera1, InternalCamera2, InternalCamera3 };
+    std::vector<APIPCamera*> cameras = { InternalCamera1, InternalCamera2, InternalCamera3, InternalCamera4, InternalCamera5 };
     wrapper_->initialize(this, cameras);
 
     startApiServer(enable_rpc, api_server_address);

Unreal/Plugins/AirSim/Source/Car/CarPawn.h

@@ -31,6 +31,10 @@ class ACarPawn : public AWheeledVehicle
     class USceneComponent* InternalCameraBase2;
     UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true"))
     class USceneComponent* InternalCameraBase3;
+    UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true"))
+    class USceneComponent* InternalCameraBase4;
+    UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true"))
+    class USceneComponent* InternalCameraBase5;
 
     /** Camera component for the In-Car view */
     UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true"))
@@ -39,6 +43,10 @@ class ACarPawn : public AWheeledVehicle
     APIPCamera* InternalCamera2;
     UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true"))
     APIPCamera* InternalCamera3;
+    UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true"))
+    APIPCamera* InternalCamera4;
+    UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true"))
+    APIPCamera* InternalCamera5;
 
     /** Text component for the In-Car speed */
     UPROPERTY(Category = Display, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true"))

Unreal/Plugins/AirSim/Source/SimHUD/SimHUD.cpp

@@ -222,6 +222,7 @@ void ASimHUD::initializeSubWindows()
         subwindow_cameras_[0] = subwindow_cameras_[1] = subwindow_cameras_[2] = simmode_->getFpvVehiclePawnWrapper()->getCamera();
     else
         subwindow_cameras_[0] = subwindow_cameras_[1] = subwindow_cameras_[2] = nullptr;
+
     subwindow_camera_types_[0] = ImageType::DepthVis;
     subwindow_camera_types_[1] = ImageType::Segmentation;
     subwindow_camera_types_[2] = ImageType::Scene;
@@ -233,16 +234,24 @@ void ASimHUD::initializeSubWindows()
         for (size_t child_index = 0; child_index < json_settings_parent.size(); ++child_index) {
             Settings json_settings_child;     
             if (json_settings_parent.getChild(child_index, json_settings_child)) {
-                int index = json_settings_child.getInt("Index", -1);
+                int index = json_settings_child.getInt("WindowID", -1);
 
                 if (index == -1) {
-                    UAirBlueprintLib::LogMessageString("Index not set in <SubWindows> element(s) in settings.json", 
+                    UAirBlueprintLib::LogMessageString("WindowID not set in <SubWindows> element(s) in settings.json", 
                         std::to_string(child_index), LogDebugLevel::Failure);
                     continue;
                 }
 
                 subwindow_camera_types_[index] = Utils::toEnum<ImageType>(json_settings_child.getInt("ImageType", 0));
                 subwindow_visible_[index] = json_settings_child.getBool("Visible", false);
+
+                int camera_id = json_settings_child.getInt("CameraID", 0);
+                if (camera_id >= 0 && camera_id < simmode_->getFpvVehiclePawnWrapper()->getCameraCount())
+                    subwindow_cameras_[index] = simmode_->getFpvVehiclePawnWrapper()->getCamera(camera_id);
+                else 
+
+                    UAirBlueprintLib::LogMessageString("CameraID in <SubWindows> element in settings.json is invalid", 
+                        std::to_string(child_index), LogDebugLevel::Failure);
             }
         }
     }

docs/image_apis.md

@@ -165,16 +165,27 @@ To change resolution, FOV etc, you can use [settings.json](settings.md). For exa
 ```
 
 ## What Does Pixel Values Mean in Different Image Types?
-### Depth Image
+### Available ImageType
+```
+  Scene = 0, 
+  DepthPlanner = 1, 
+  DepthPerspective = 2,
+  DepthVis = 3, 
+  DisparityNormalized = 4,
+  Segmentation = 5,
+  SurfaceNormals = 6
+```                
+
+### DepthPlanner and DepthPerspective
 You normally want retrieve depth image as float (i.e. set `pixels_as_float = true`) and specify `ImageType = DepthPlanner` or `ImageType = DepthPerspective` in `ImageRequest`. For `ImageType = DepthPlanner`, you get depth in camera plan, i.e., all points that are in plan parallel to camera have same depth. For `ImageType = DepthPerspective`, you get depth from camera using a project ray that hits that pixel. Depending on your use case, planner depth or perspective depth may be the ground truth image that you want. For example, you may be able to feed perspective depth to ROS package such as `depth_image_proc` to generate point cloud. Or planner depth may be more compatible with estimated depth image generated by stereo algorithms such as SGM.
 
-### Depth Visualization Image
+### DepthVis
 When you specify `ImageType = DepthVis` in `ImageRequest`, you get image that helps depth visualization. In this case, each pixel value is interpolated from red to green depending on depth in camera plane in meters. The pixels with pure green means depth of 100m or more while pure red means depth of 0 meters.
 
-### Disparity Image
+### DisparityNormalized
 You normally want retrieve disparity image as float (i.e. set `pixels_as_float = true` and specify `ImageType = DisparityNormalized` in `ImageRequest`) in which case each pixel is `(Xl - Xr)/Xmax`, valued from 0 to 1.
 
-### Segmentation Image
+### Segmentation
 When you specify `ImageType = Segmentation` in `ImageRequest`, you get image that gives you ground truth segmentation of the scene. AirSim assigns value 0 to 255 to each mesh available in environment. This value is than mapped to a specific color in [the pallet](../Unreal/Plugins/AirSim/Content/HUDAssets/seg_color_pallet.png). Currently, if you had like to assign specific value to specific map, you will need to [change code](../Unreal/Plugins/AirSim/Source/FlyingPawn.cpp#L28). We are planning to enable APIs for this in future.
 
 ## Collision API

docs/settings.md

@@ -56,8 +56,9 @@ Below are complete list of settings available along with their default values. I
     }
   ],
   "SubWindows": [
-    {"Index": 1, "ImageType": 3},
-    {"Index": 2, "ImageType": 5}
+    {"WindowID": 0, "CameraID": 0, "ImageType": 3},
+    {"WindowID": 1, "CameraID": 0, "ImageType": 5},
+    {"WindowID": 2, "CameraID": 0, "ImageType": 0}    
   ] 
   "SimpleFlight": {
     "FirmwareName": "SimpleFlight",
@@ -151,6 +152,16 @@ The ViewMode determines how you will view the vehicle. For multirotors, the defa
 #### EngineSound
 To turn off the engine sound use [setting](settings.md) `"EngineSound": false`. Currently this setting applies only to car.
 
+#### SubWindows
+This setting determines what is shown in each of 3 subwindows which are visible when you press 0 key. The WindowsID can be 0 to 2, CameraID is integer identifying camera number on the vehicle. ImageType integer value determines what kind of image gets shown according to [ImageType enum](image_apis.md#available-imagetype). For example, for car vehicles below shows driver view, front bumper view and rear view as scene, depth ans surface normals respectively.
+```
+  "SubWindows": [
+    {"WindowID": 0, "ImageType": 0, "CameraID": 3},
+    {"WindowID": 1, "ImageType": 3, "CameraID": 0},
+    {"WindowID": 2, "ImageType": 6, "CameraID": 4}
+  ]
+```
+