diff --git a/Unreal/Plugins/AirSim/Source/Car/CarPawn.cpp b/Unreal/Plugins/AirSim/Source/Car/CarPawn.cpp index 1d95595019..e58be6b5f5 100644 --- a/Unreal/Plugins/AirSim/Source/Car/CarPawn.cpp +++ b/Unreal/Plugins/AirSim/Source/Car/CarPawn.cpp @@ -221,14 +221,21 @@ ACarPawn::ACarPawn() // Create In-Car camera component InternalCameraBase1 = CreateDefaultSubobject(TEXT("InternalCameraBase1")); - InternalCameraBase1->SetRelativeLocation(FVector(-34.0f, 0, 50.0f)); + InternalCameraBase1->SetRelativeLocation(FVector(36.0f, 0, 50.0f)); //center InternalCameraBase1->SetupAttachment(GetMesh()); InternalCameraBase2 = CreateDefaultSubobject(TEXT("InternalCameraBase2")); - InternalCameraBase2->SetRelativeLocation(FVector(-34.0f, -10, 50.0f)); + InternalCameraBase2->SetRelativeLocation(FVector(36.0f, -10, 50.0f)); //left InternalCameraBase2->SetupAttachment(GetMesh()); InternalCameraBase3 = CreateDefaultSubobject(TEXT("InternalCameraBase3")); - InternalCameraBase3->SetRelativeLocation(FVector(-34.0f, 10, 50.0f)); + InternalCameraBase3->SetRelativeLocation(FVector(36.0f, 10, 50.0f)); //right InternalCameraBase3->SetupAttachment(GetMesh()); + InternalCameraBase4 = CreateDefaultSubobject(TEXT("InternalCameraBase4")); + InternalCameraBase4->SetRelativeLocation(FVector(25, -10, 75.0f)); //driver + InternalCameraBase4->SetupAttachment(GetMesh()); + InternalCameraBase5 = CreateDefaultSubobject(TEXT("InternalCameraBase5")); + InternalCameraBase5->SetRelativeLocation(FVector(-36.0f, 0, 50.0f)); //rear + InternalCameraBase5->SetRelativeRotation(FRotator(0, 180, 0)); + InternalCameraBase5->SetupAttachment(GetMesh()); // In car HUD // Create text render component for in car speed display @@ -278,7 +285,7 @@ void ACarPawn::initializeForBeginPlay(bool enable_rpc, const std::string& api_se EngineSoundComponent->Deactivate(); //put camera little bit above vehicle - FTransform camera_transform(FVector(0, 0, 0)); + FTransform camera_transform(FVector::ZeroVector); FActorSpawnParameters camera_spawn_params; camera_spawn_params.SpawnCollisionHandlingOverride = ESpawnActorCollisionHandlingMethod::AdjustIfPossibleButAlwaysSpawn; InternalCamera1 = this->GetWorld()->SpawnActor(pip_camera_class_, camera_transform, camera_spawn_params); @@ -287,10 +294,15 @@ void ACarPawn::initializeForBeginPlay(bool enable_rpc, const std::string& api_se InternalCamera2->AttachToComponent(InternalCameraBase2, FAttachmentTransformRules::KeepRelativeTransform); InternalCamera3 = this->GetWorld()->SpawnActor(pip_camera_class_, camera_transform, camera_spawn_params); InternalCamera3->AttachToComponent(InternalCameraBase3, FAttachmentTransformRules::KeepRelativeTransform); + InternalCamera4 = this->GetWorld()->SpawnActor(pip_camera_class_, camera_transform, camera_spawn_params); + InternalCamera4->AttachToComponent(InternalCameraBase4, FAttachmentTransformRules::KeepRelativeTransform); + InternalCamera5 = this->GetWorld()->SpawnActor(pip_camera_class_, FTransform(FRotator(0, 180, 0), FVector::ZeroVector), camera_spawn_params); + InternalCamera5->AttachToComponent(InternalCameraBase4, FAttachmentTransformRules::KeepRelativeTransform); + setupInputBindings(); - std::vector cameras = { InternalCamera1, InternalCamera2, InternalCamera3 }; + std::vector cameras = { InternalCamera1, InternalCamera2, InternalCamera3, InternalCamera4, InternalCamera5 }; wrapper_->initialize(this, cameras); startApiServer(enable_rpc, api_server_address); diff --git a/Unreal/Plugins/AirSim/Source/Car/CarPawn.h b/Unreal/Plugins/AirSim/Source/Car/CarPawn.h index c82327150f..b6fd6a4443 100644 --- a/Unreal/Plugins/AirSim/Source/Car/CarPawn.h +++ b/Unreal/Plugins/AirSim/Source/Car/CarPawn.h @@ -31,6 +31,10 @@ class ACarPawn : public AWheeledVehicle class USceneComponent* InternalCameraBase2; UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true")) class USceneComponent* InternalCameraBase3; + UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true")) + class USceneComponent* InternalCameraBase4; + UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true")) + class USceneComponent* InternalCameraBase5; /** Camera component for the In-Car view */ UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true")) @@ -39,6 +43,10 @@ class ACarPawn : public AWheeledVehicle APIPCamera* InternalCamera2; UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true")) APIPCamera* InternalCamera3; + UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true")) + APIPCamera* InternalCamera4; + UPROPERTY(Category = Camera, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true")) + APIPCamera* InternalCamera5; /** Text component for the In-Car speed */ UPROPERTY(Category = Display, VisibleDefaultsOnly, BlueprintReadOnly, meta = (AllowPrivateAccess = "true")) diff --git a/Unreal/Plugins/AirSim/Source/SimHUD/SimHUD.cpp b/Unreal/Plugins/AirSim/Source/SimHUD/SimHUD.cpp index 1603f6d577..6807a747ac 100644 --- a/Unreal/Plugins/AirSim/Source/SimHUD/SimHUD.cpp +++ b/Unreal/Plugins/AirSim/Source/SimHUD/SimHUD.cpp @@ -222,6 +222,7 @@ void ASimHUD::initializeSubWindows() subwindow_cameras_[0] = subwindow_cameras_[1] = subwindow_cameras_[2] = simmode_->getFpvVehiclePawnWrapper()->getCamera(); else subwindow_cameras_[0] = subwindow_cameras_[1] = subwindow_cameras_[2] = nullptr; + subwindow_camera_types_[0] = ImageType::DepthVis; subwindow_camera_types_[1] = ImageType::Segmentation; subwindow_camera_types_[2] = ImageType::Scene; @@ -233,16 +234,24 @@ void ASimHUD::initializeSubWindows() for (size_t child_index = 0; child_index < json_settings_parent.size(); ++child_index) { Settings json_settings_child; if (json_settings_parent.getChild(child_index, json_settings_child)) { - int index = json_settings_child.getInt("Index", -1); + int index = json_settings_child.getInt("WindowID", -1); if (index == -1) { - UAirBlueprintLib::LogMessageString("Index not set in element(s) in settings.json", + UAirBlueprintLib::LogMessageString("WindowID not set in element(s) in settings.json", std::to_string(child_index), LogDebugLevel::Failure); continue; } subwindow_camera_types_[index] = Utils::toEnum(json_settings_child.getInt("ImageType", 0)); subwindow_visible_[index] = json_settings_child.getBool("Visible", false); + + int camera_id = json_settings_child.getInt("CameraID", 0); + if (camera_id >= 0 && camera_id < simmode_->getFpvVehiclePawnWrapper()->getCameraCount()) + subwindow_cameras_[index] = simmode_->getFpvVehiclePawnWrapper()->getCamera(camera_id); + else + + UAirBlueprintLib::LogMessageString("CameraID in element in settings.json is invalid", + std::to_string(child_index), LogDebugLevel::Failure); } } } diff --git a/docs/image_apis.md b/docs/image_apis.md index 42ffc3b34b..9ad3225c5b 100644 --- a/docs/image_apis.md +++ b/docs/image_apis.md @@ -165,16 +165,27 @@ To change resolution, FOV etc, you can use [settings.json](settings.md). For exa ``` ## What Does Pixel Values Mean in Different Image Types? -### Depth Image +### Available ImageType +``` + Scene = 0, + DepthPlanner = 1, + DepthPerspective = 2, + DepthVis = 3, + DisparityNormalized = 4, + Segmentation = 5, + SurfaceNormals = 6 +``` + +### DepthPlanner and DepthPerspective You normally want retrieve depth image as float (i.e. set `pixels_as_float = true`) and specify `ImageType = DepthPlanner` or `ImageType = DepthPerspective` in `ImageRequest`. For `ImageType = DepthPlanner`, you get depth in camera plan, i.e., all points that are in plan parallel to camera have same depth. For `ImageType = DepthPerspective`, you get depth from camera using a project ray that hits that pixel. Depending on your use case, planner depth or perspective depth may be the ground truth image that you want. For example, you may be able to feed perspective depth to ROS package such as `depth_image_proc` to generate point cloud. Or planner depth may be more compatible with estimated depth image generated by stereo algorithms such as SGM. -### Depth Visualization Image +### DepthVis When you specify `ImageType = DepthVis` in `ImageRequest`, you get image that helps depth visualization. In this case, each pixel value is interpolated from red to green depending on depth in camera plane in meters. The pixels with pure green means depth of 100m or more while pure red means depth of 0 meters. -### Disparity Image +### DisparityNormalized You normally want retrieve disparity image as float (i.e. set `pixels_as_float = true` and specify `ImageType = DisparityNormalized` in `ImageRequest`) in which case each pixel is `(Xl - Xr)/Xmax`, valued from 0 to 1. -### Segmentation Image +### Segmentation When you specify `ImageType = Segmentation` in `ImageRequest`, you get image that gives you ground truth segmentation of the scene. AirSim assigns value 0 to 255 to each mesh available in environment. This value is than mapped to a specific color in [the pallet](../Unreal/Plugins/AirSim/Content/HUDAssets/seg_color_pallet.png). Currently, if you had like to assign specific value to specific map, you will need to [change code](../Unreal/Plugins/AirSim/Source/FlyingPawn.cpp#L28). We are planning to enable APIs for this in future. ## Collision API diff --git a/docs/settings.md b/docs/settings.md index 30a436340a..966ee46467 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -56,8 +56,9 @@ Below are complete list of settings available along with their default values. I } ], "SubWindows": [ - {"Index": 1, "ImageType": 3}, - {"Index": 2, "ImageType": 5} + {"WindowID": 0, "CameraID": 0, "ImageType": 3}, + {"WindowID": 1, "CameraID": 0, "ImageType": 5}, + {"WindowID": 2, "CameraID": 0, "ImageType": 0} ] "SimpleFlight": { "FirmwareName": "SimpleFlight", @@ -151,6 +152,16 @@ The ViewMode determines how you will view the vehicle. For multirotors, the defa #### EngineSound To turn off the engine sound use [setting](settings.md) `"EngineSound": false`. Currently this setting applies only to car. +#### SubWindows +This setting determines what is shown in each of 3 subwindows which are visible when you press 0 key. The WindowsID can be 0 to 2, CameraID is integer identifying camera number on the vehicle. ImageType integer value determines what kind of image gets shown according to [ImageType enum](image_apis.md#available-imagetype). For example, for car vehicles below shows driver view, front bumper view and rear view as scene, depth ans surface normals respectively. +``` + "SubWindows": [ + {"WindowID": 0, "ImageType": 0, "CameraID": 3}, + {"WindowID": 1, "ImageType": 3, "CameraID": 0}, + {"WindowID": 2, "ImageType": 6, "CameraID": 4} + ] +``` +