Merge pull request microsoft#21 from Microsoft/feature/api_updates

Feature/api updates
mvaloriani · May 2, 2018 · 6223588 · 6223588
2 parents a0fe1f2 + cf138a6
commit 6223588
Show file tree

Hide file tree

Showing 55 changed files with 1,474 additions and 381 deletions.
diff --git a/Samples/BatchProcessing/BatchProcessing.vcxproj b/Samples/BatchProcessing/BatchProcessing.vcxproj
@@ -7,7 +7,7 @@
     <MinimumVisualStudioVersion>14.0</MinimumVisualStudioVersion>
     <AppContainerApplication>true</AppContainerApplication>
     <ApplicationType>Windows Store</ApplicationType>
-    <WindowsTargetPlatformVersion Condition=" '$(WindowsTargetPlatformVersion)' == '' ">10.0.14393.0</WindowsTargetPlatformVersion>
+    <WindowsTargetPlatformVersion Condition=" '$(WindowsTargetPlatformVersion)' == '' ">10.0.16299.0</WindowsTargetPlatformVersion>
     <WindowsTargetPlatformMinVersion>10.0.14393.0</WindowsTargetPlatformMinVersion>
     <ApplicationTypeRevision>10.0</ApplicationTypeRevision>
   </PropertyGroup>

diff --git a/Samples/BatchProcessing/CameraFrame.cpp b/Samples/BatchProcessing/CameraFrame.cpp
@@ -78,7 +78,9 @@ namespace BatchProcessing
             ASSERT(
                 1 /* Timestamp */ +
                 1 /* ImageFileName */ +
-                16 /* FrameToOrigin.m(1..4)(1..4) */ == tokens.size());
+                16 /* FrameToOrigin.m(1..4)(1..4) */ +
+                16 /* CameraViewTransform.m(1..4)(1..4) */ +
+                16 /* CameraProjectionTransform.m(1..4)(1..4) */ == tokens.size());
 
             //
             // Skip the CSV file header
@@ -88,6 +90,8 @@ namespace BatchProcessing
                 ASSERT(0 == strcmp(tokens[0].c_str(), "Timestamp"));
                 ASSERT(0 == strcmp(tokens[1].c_str(), "ImageFileName"));
                 ASSERT(0 == strcmp(tokens[17].c_str(), "FrameToOrigin.m44"));
+                ASSERT(0 == strcmp(tokens[33].c_str(), "CameraViewTransform.m44"));
+                ASSERT(0 == strcmp(tokens[49].c_str(), "CameraProjectionTransform.m44"));
 
                 csvFileHeaderSeen = true;
 
@@ -113,6 +117,18 @@ namespace BatchProcessing
                     4 /* cols */,
                     CV_32F /* type */);
 
+            cameraFrame.CameraViewTransform =
+                cv::Mat::eye(
+                    4 /* rows */,
+                    4 /* cols */,
+                    CV_32F /* type */);
+
+            cameraFrame.CameraProjectionTransform =
+                cv::Mat::eye(
+                    4 /* rows */,
+                    4 /* cols */,
+                    CV_32F /* type */);
+
             //
             // FrameToOrigin follows the timestamp and image file name fields:
             //
@@ -127,6 +143,28 @@ namespace BatchProcessing
                 }
             }
 
+            for (int32_t j = 0; j < 4; ++j)
+            {
+                for (int32_t i = 0; i < 4; ++i)
+                {
+                    cameraFrame.CameraViewTransform.at<float>(j, i) =
+                        static_cast<float>(
+                            std::atof(
+                                tokens[j * 4 + i + 2].c_str()));
+                }
+            }
+
+            for (int32_t j = 0; j < 4; ++j)
+            {
+                for (int32_t i = 0; i < 4; ++i)
+                {
+                    cameraFrame.CameraProjectionTransform.at<float>(j, i) =
+                        static_cast<float>(
+                            std::atof(
+                                tokens[j * 4 + i + 2].c_str()));
+                }
+            }
+
             //TODO: add the width/height/pixel format to the manifest or camera calibration CSV
             cameraFrame.Width = 1280;
             cameraFrame.Height = 720;

diff --git a/Samples/BatchProcessing/CameraFrame.h b/Samples/BatchProcessing/CameraFrame.h
@@ -22,6 +22,8 @@ namespace BatchProcessing
         Windows::Storage::StorageFolder^ RecordingFolder;
         std::wstring FileName;
         cv::Mat FrameToOrigin;
+        cv::Mat CameraViewTransform;
+        cv::Mat CameraProjectionTransform;
         int32_t Width;
         int32_t Height;
         int32_t PixelFormat;

diff --git a/Samples/BatchProcessing/MainPage.xaml.cpp b/Samples/BatchProcessing/MainPage.xaml.cpp
@@ -157,7 +157,7 @@ namespace BatchProcessing
                 uint32_t imageBufferDataLength = 0;
 
                 uint8_t* imageBufferData =
-                    Io::GetPointerToMemoryBuffer(
+                    Io::GetTypedPointerToMemoryBuffer<uint8_t>(
                         imageBufferReference,
                         imageBufferDataLength);
 

diff --git a/Samples/ComputeOnDesktop/ComputeOnDesktop.vcxproj b/Samples/ComputeOnDesktop/ComputeOnDesktop.vcxproj
@@ -8,7 +8,7 @@
     <MinimumVisualStudioVersion>14.0</MinimumVisualStudioVersion>
     <AppContainerApplication>true</AppContainerApplication>
     <ApplicationType>Windows Store</ApplicationType>
-    <WindowsTargetPlatformVersion Condition=" '$(WindowsTargetPlatformVersion)' == '' ">10.0.15063.0</WindowsTargetPlatformVersion>
+    <WindowsTargetPlatformVersion Condition=" '$(WindowsTargetPlatformVersion)' == '' ">10.0.16299.0</WindowsTargetPlatformVersion>
     <WindowsTargetPlatformMinVersion>10.0.14393.0</WindowsTargetPlatformMinVersion>
     <ApplicationTypeRevision>10.0</ApplicationTypeRevision>
   </PropertyGroup>

diff --git a/Samples/ComputeOnDevice/AppMain.cpp b/Samples/ComputeOnDevice/AppMain.cpp
@@ -124,52 +124,68 @@ namespace ComputeOnDevice
         if (!_undistortMapsInitialized)
         {
             Windows::Media::Devices::Core::CameraIntrinsics^ cameraIntrinsics =
-                latestFrame->CameraIntrinsics;
+                latestFrame->CoreCameraIntrinsics;
 
-            cv::Mat cameraMatrix(3, 3, CV_64FC1);
-
-            cv::setIdentity(cameraMatrix);
-
-            cameraMatrix.at<double>(0, 0) = cameraIntrinsics->FocalLength.x;
-            cameraMatrix.at<double>(1, 1) = cameraIntrinsics->FocalLength.y;
-            cameraMatrix.at<double>(0, 2) = cameraIntrinsics->PrincipalPoint.x;
-            cameraMatrix.at<double>(1, 2) = cameraIntrinsics->PrincipalPoint.y;
-
-            cv::Mat distCoeffs(5, 1, CV_64FC1);
-
-            distCoeffs.at<double>(0, 0) = cameraIntrinsics->RadialDistortion.x;
-            distCoeffs.at<double>(1, 0) = cameraIntrinsics->RadialDistortion.y;
-            distCoeffs.at<double>(2, 0) = cameraIntrinsics->TangentialDistortion.x;
-            distCoeffs.at<double>(3, 0) = cameraIntrinsics->TangentialDistortion.y;
-            distCoeffs.at<double>(4, 0) = cameraIntrinsics->RadialDistortion.z;
+            if (nullptr != cameraIntrinsics)
+            {
+                cv::Mat cameraMatrix(3, 3, CV_64FC1);
+
+                cv::setIdentity(cameraMatrix);
+
+                cameraMatrix.at<double>(0, 0) = cameraIntrinsics->FocalLength.x;
+                cameraMatrix.at<double>(1, 1) = cameraIntrinsics->FocalLength.y;
+                cameraMatrix.at<double>(0, 2) = cameraIntrinsics->PrincipalPoint.x;
+                cameraMatrix.at<double>(1, 2) = cameraIntrinsics->PrincipalPoint.y;
+
+                cv::Mat distCoeffs(5, 1, CV_64FC1);
+
+                distCoeffs.at<double>(0, 0) = cameraIntrinsics->RadialDistortion.x;
+                distCoeffs.at<double>(1, 0) = cameraIntrinsics->RadialDistortion.y;
+                distCoeffs.at<double>(2, 0) = cameraIntrinsics->TangentialDistortion.x;
+                distCoeffs.at<double>(3, 0) = cameraIntrinsics->TangentialDistortion.y;
+                distCoeffs.at<double>(4, 0) = cameraIntrinsics->RadialDistortion.z;
+
+                cv::initUndistortRectifyMap(
+                    cameraMatrix,
+                    distCoeffs,
+                    cv::Mat_<double>::eye(3, 3) /* R */,
+                    cameraMatrix,
+                    cv::Size(wrappedImage.cols, wrappedImage.rows),
+                    CV_32FC1 /* type */,
+                    _undistortMap1,
+                    _undistortMap2);
+
+                _undistortMapsInitialized = true;
+            }
+        }
 
-            cv::initUndistortRectifyMap(
-                cameraMatrix,
-                distCoeffs,
-                cv::Mat_<double>::eye(3, 3) /* R */,
-                cameraMatrix,
-                cv::Size(wrappedImage.cols, wrappedImage.rows),
-                CV_32FC1 /* type */,
+        if (_undistortMapsInitialized)
+        {
+            cv::remap(
+                wrappedImage,
+                _undistortedPVCameraImage,
                 _undistortMap1,
-                _undistortMap2);
-
-            _undistortMapsInitialized = true;
+                _undistortMap2,
+                cv::INTER_LINEAR);
+
+            cv::resize(
+                _undistortedPVCameraImage,
+                _resizedPVCameraImage,
+                cv::Size(),
+                0.5 /* fx */,
+                0.5 /* fy */,
+                cv::INTER_AREA);
+        }
+        else
+        {
+            cv::resize(
+                wrappedImage,
+                _resizedPVCameraImage,
+                cv::Size(),
+                0.5 /* fx */,
+                0.5 /* fy */,
+                cv::INTER_AREA);
         }
-
-        cv::remap(
-            wrappedImage,
-            _undistortedPVCameraImage,
-            _undistortMap1,
-            _undistortMap2,
-            cv::INTER_LINEAR);
-
-        cv::resize(
-            _undistortedPVCameraImage,
-            _resizedPVCameraImage,
-            cv::Size(),
-            0.5 /* fx */,
-            0.5 /* fy */,
-            cv::INTER_AREA);
 
         cv::medianBlur(
             _resizedPVCameraImage,
@@ -188,7 +204,7 @@ namespace ComputeOnDevice
             {
                 if (_cannyPVCameraImage.at<uint8_t>(y, x) > 64)
                 {
-                    _blurredPVCameraImage.at<uint32_t>(y, x) = 0xFFFF00FF;
+                    *(_blurredPVCameraImage.ptr<uint32_t>(y, x)) = 0xFFFF00FF;
                 }
             }
         }
@@ -265,6 +281,9 @@ namespace ComputeOnDevice
                 _spatialPerception,
                 _sensorFrameStreamer);
 
+        _holoLensMediaFrameSourceGroup->Enable(
+            HoloLensForCV::SensorType::PhotoVideo);
+
         concurrency::create_task(_holoLensMediaFrameSourceGroup->StartAsync()).then(
             [&]()
         {

diff --git a/Samples/ComputeOnDevice/ComputeOnDevice.vcxproj b/Samples/ComputeOnDevice/ComputeOnDevice.vcxproj
@@ -9,7 +9,7 @@
     <MinimumVisualStudioVersion>14.0</MinimumVisualStudioVersion>
     <AppContainerApplication>true</AppContainerApplication>
     <ApplicationType>Windows Store</ApplicationType>
-    <WindowsTargetPlatformVersion>10.0.14393.0</WindowsTargetPlatformVersion>
+    <WindowsTargetPlatformVersion>10.0.16299.0</WindowsTargetPlatformVersion>
     <WindowsTargetPlatformMinVersion>10.0.14393.0</WindowsTargetPlatformMinVersion>
     <ApplicationTypeRevision>10.0</ApplicationTypeRevision>
     <EnableDotNetNativeCompatibleProfile>true</EnableDotNetNativeCompatibleProfile>

diff --git a/Samples/ComputeOnDevice/Package.appxmanifest b/Samples/ComputeOnDevice/Package.appxmanifest
@@ -1,5 +1,10 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Package xmlns="http://schemas.microsoft.com/appx/manifest/foundation/windows10" xmlns:mp="http://schemas.microsoft.com/appx/2014/phone/manifest" xmlns:uap="http://schemas.microsoft.com/appx/manifest/uap/windows10" IgnorableNamespaces="uap mp">
+<Package
+    xmlns="http://schemas.microsoft.com/appx/manifest/foundation/windows10"
+    xmlns:mp="http://schemas.microsoft.com/appx/2014/phone/manifest"
+    xmlns:uap="http://schemas.microsoft.com/appx/manifest/uap/windows10"
+    xmlns:rescap="http://schemas.microsoft.com/appx/manifest/foundation/windows10/restrictedcapabilities"
+    IgnorableNamespaces="uap mp rescap">
   <Identity Name="72A4857D-CB19-4744-B055-444F6DE1CD8C" Publisher="CN=Microsoft Corporation, O=Microsoft Corporation, L=Redmond, S=Washington, C=US" Version="1.0.0.0" />
   <mp:PhoneIdentity PhoneProductId="72A4857D-CB19-4744-B055-444F6DE1CD8C" PhonePublisherId="00000000-0000-0000-0000-000000000000" />
   <Properties>
@@ -23,6 +28,7 @@
     </Application>
   </Applications>
   <Capabilities>
+    <rescap:Capability Name="perceptionSensorsExperimental" />
     <Capability Name="privateNetworkClientServer" />
     <Capability Name="internetClientServer" />
     <DeviceCapability Name="webcam" />

diff --git a/Shared/Audio/Audio.vcxproj b/Shared/Audio/Audio.vcxproj
@@ -34,7 +34,7 @@
     <MinimumVisualStudioVersion>14.0</MinimumVisualStudioVersion>
     <AppContainerApplication>true</AppContainerApplication>
     <ApplicationType>Windows Store</ApplicationType>
-    <WindowsTargetPlatformVersion>10.0.14393.0</WindowsTargetPlatformVersion>
+    <WindowsTargetPlatformVersion>10.0.16299.0</WindowsTargetPlatformVersion>
     <WindowsTargetPlatformMinVersion>10.0.14393.0</WindowsTargetPlatformMinVersion>
     <ApplicationTypeRevision>10.0</ApplicationTypeRevision>
   </PropertyGroup>

diff --git a/Shared/Debugging/Debugging.vcxproj b/Shared/Debugging/Debugging.vcxproj
@@ -34,7 +34,7 @@
     <MinimumVisualStudioVersion>14.0</MinimumVisualStudioVersion>
     <AppContainerApplication>true</AppContainerApplication>
     <ApplicationType>Windows Store</ApplicationType>
-    <WindowsTargetPlatformVersion>10.0.14393.0</WindowsTargetPlatformVersion>
+    <WindowsTargetPlatformVersion>10.0.16299.0</WindowsTargetPlatformVersion>
     <WindowsTargetPlatformMinVersion>10.0.14393.0</WindowsTargetPlatformMinVersion>
     <ApplicationTypeRevision>10.0</ApplicationTypeRevision>
   </PropertyGroup>

diff --git a/Shared/Graphics/Graphics.vcxproj b/Shared/Graphics/Graphics.vcxproj
@@ -35,7 +35,7 @@
     <MinimumVisualStudioVersion>14.0</MinimumVisualStudioVersion>
     <AppContainerApplication>true</AppContainerApplication>
     <ApplicationType>Windows Store</ApplicationType>
-    <WindowsTargetPlatformVersion>10.0.14393.0</WindowsTargetPlatformVersion>
+    <WindowsTargetPlatformVersion>10.0.16299.0</WindowsTargetPlatformVersion>
     <WindowsTargetPlatformMinVersion>10.0.14393.0</WindowsTargetPlatformMinVersion>
     <ApplicationTypeRevision>10.0</ApplicationTypeRevision>
   </PropertyGroup>

diff --git a/Shared/HoloLensForCV/CameraIntrinsics.cpp b/Shared/HoloLensForCV/CameraIntrinsics.cpp
@@ -0,0 +1,65 @@
+//*********************************************************
+//
+// Copyright (c) Microsoft. All rights reserved.
+// This code is licensed under the MIT License (MIT).
+// THIS CODE IS PROVIDED *AS IS* WITHOUT WARRANTY OF
+// ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING ANY
+// IMPLIED WARRANTIES OF FITNESS FOR A PARTICULAR
+// PURPOSE, MERCHANTABILITY, OR NON-INFRINGEMENT.
+//
+//*********************************************************
+
+#include "pch.h"
+
+namespace HoloLensForCV
+{
+    CameraIntrinsics::CameraIntrinsics(
+        _In_ Microsoft::WRL::ComPtr<SensorStreaming::ICameraIntrinsics> sensorStreamingCameraIntrinsics,
+        _In_ unsigned int imageWidth,
+        _In_ unsigned int imageHeight)
+        : _sensorStreamingCameraIntrinsics(sensorStreamingCameraIntrinsics)
+    {
+        ImageWidth = imageWidth;
+        ImageHeight = imageHeight;
+    }
+
+    bool CameraIntrinsics::MapImagePointToCameraUnitPlane(
+        _In_ Windows::Foundation::Point UV,
+        _Out_ Windows::Foundation::Point* XY)
+    {
+        float uv[2] = { UV.X, UV.Y };
+        float xy[2];
+
+        if (FAILED(_sensorStreamingCameraIntrinsics->MapImagePointToCameraUnitPlane(uv, xy)))
+        {
+            XY->X = XY->Y = std::numeric_limits<float>::infinity();
+
+            return false;
+        }
+
+        XY->X = xy[0];
+        XY->Y = xy[1];
+
+        return true;
+    }
+
+    bool CameraIntrinsics::MapCameraSpaceToImagePoint(
+        _In_ Windows::Foundation::Point XY,
+        _Out_ Windows::Foundation::Point* UV)
+    {
+        float xy[2] = { XY.X, XY.Y };
+        float uv[2];
+
+        if (FAILED(_sensorStreamingCameraIntrinsics->MapCameraSpaceToImagePoint(xy, uv)))
+        {
+            UV->X = UV->Y = std::numeric_limits<float>::infinity();
+
+            return false;
+        }
+
+        UV->X = uv[0];
+        UV->Y = uv[1];
+
+        return true;
+    }
+}