[openmvg/spherical] Add a new module for spherical/pinhole camera sam…

…pling#1754

- Add the new module
- Use it for sphere to cubic image conversion
- Add a 'Unit-test/samples' to demo how to use direct and indirect
sampling with OpenMVG cameras. (=> bearing vector projection between
camera models)

src/openMVG/CMakeLists.txt

@@ -15,6 +15,7 @@ add_subdirectory(numeric)
 add_subdirectory(robust_estimation)
 add_subdirectory(tracks)
 add_subdirectory(color_harmonization)
+add_subdirectory(spherical)
 add_subdirectory(system)
 add_subdirectory(sfm)
 add_subdirectory(stl)

src/openMVG/spherical/CMakeLists.txt

@@ -0,0 +1 @@
+UNIT_TEST(openMVG convert "openMVG_testing;openMVG_image;openMVG_camera")

src/openMVG/spherical/convert_test.cpp

@@ -0,0 +1,137 @@
+// This file is part of OpenMVG, an Open Multiple View Geometry C++ library.
+
+// Copyright (c) 2020 Pierre MOULON.
+
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include "openMVG/spherical/cubic_image_sampler.hpp"
+#include "openMVG/image/image_io.hpp"
+
+#include "testing/testing.h"
+
+using namespace openMVG;
+using namespace openMVG::image;
+using namespace openMVG::cameras;
+
+// Define some global variable that gonna be shared by the various demo sample
+static const int cubic_size = 256;
+static const std::vector<Image<RGBColor>> images =
+{
+  Image<RGBColor>(cubic_size, cubic_size, true, BLUE),
+  Image<RGBColor>(cubic_size, cubic_size, true, RED),
+  Image<RGBColor>(cubic_size, cubic_size, true, GREEN),
+  Image<RGBColor>(cubic_size, cubic_size, true, YELLOW),
+  Image<RGBColor>(cubic_size, cubic_size, true, CYAN),
+  Image<RGBColor>(cubic_size, cubic_size, true, MAGENTA)
+};
+static const openMVG::cameras::Pinhole_Intrinsic pinhole_camera =
+  spherical::ComputeCubicCameraIntrinsics(cubic_size);
+
+static const std::array<openMVG::Mat3,6> cubic_image_rotations = spherical::GetCubicRotations();
+// Project each pinhole on a sphere
+static const Intrinsic_Spherical spherical_camera(512, 256);
+
+void SphericalToPinholeIndirect
+(
+  Image<RGBColor> & spherical_image
+)
+{
+  // Compute the projection of the tiles on the pano (indirect)
+  // For each pixel of the spherical image, find back which tile pixel project on it
+  {
+    for (int x = 0; x < spherical_image.Width(); ++x)
+    for (int y = 0; y < spherical_image.Height(); ++y)
+    {
+      // Build a bearing vector
+      const Vec3 bearing_vector = spherical_camera(Vec2(x + .5,y + .5));
+      // See if the bearing_vector reproject on a pinhole images
+      for (const int i_rot : {0, 1, 2, 3, 4, 5}) // For every rotation
+      {
+        const Mat3 rotation_matrix = cubic_image_rotations[i_rot];
+        const Vec2 proj = pinhole_camera.project(rotation_matrix * bearing_vector);
+        const Vec3 pinhole_cam_look_dir = rotation_matrix.transpose() * pinhole_camera(Vec2(spherical_image.Width()/2,spherical_image.Height()/2));
+        const auto & pinhole_image = images[i_rot];
+        if (// Look if the projection is projecting into the image plane
+           proj.x() > 0 && proj.x() < pinhole_image.Width() &&
+           proj.y() > 0 && proj.y() < pinhole_image.Height() &&
+           // Look if the camera and the sampled sphere vector are looking in the same direction
+           pinhole_cam_look_dir.dot(bearing_vector) > 0.0
+          )
+        {
+          spherical_image(y, x) = pinhole_image((int)proj.y(), (int)proj.x());
+          break;
+        }
+      }
+    }
+  }
+}
+
+TEST(Ressampling, PinholeToSpherical_indirect_mapping)
+{
+  Image<RGBColor> spherical_image(512, 256);
+  SphericalToPinholeIndirect(spherical_image);
+  WriteImage("test_pinhole_to_spherical_indirect.png", spherical_image);
+}
+
+TEST(Ressampling, PinholeToSpherical_direct_mapping)
+{
+  Image<RGBColor> spherical_image(512, 256);
+
+  for (const int i_rot : {0, 1, 2, 3, 4, 5}) // For every rotation
+  {
+    const auto & pinhole_image = images[i_rot];
+    const Mat3 rotation_matrix = cubic_image_rotations[i_rot];
+    // Project each pixel
+    Mat2X xy_coords(2, static_cast<int>(cubic_size * cubic_size));
+    for (int x = 0; x < cubic_size; ++x)
+    for (int y = 0; y < cubic_size; ++y)
+      xy_coords.col(x + cubic_size * y ) << x + .5, y +.5;
+    // Compute bearing vectors
+    const Mat3X bearing_vectors = pinhole_camera(xy_coords);
+    // Compute rotated bearings
+    const Mat3X rotated_bearings = rotation_matrix.transpose() * bearing_vectors;
+    for (int it = 0; it < rotated_bearings.cols(); ++it)
+    {
+      // Project the bearing vector to the sphere
+      const Vec2 sphere_proj = spherical_camera.project(rotated_bearings.col(it));
+      // and use the corresponding pixel location in the panorama
+      const Vec2 xy = xy_coords.col(it);
+      if (spherical_image.Contains(sphere_proj.y(), sphere_proj.x()))
+      {
+        spherical_image(sphere_proj.y(), sphere_proj.x()) =
+          pinhole_image((int)xy.y(),(int)xy.x());
+      }
+    }
+  }
+  WriteImage("test_pinhole_to_spherical_direct.png", spherical_image);
+  // Notes:
+  // Since the projection is direct, we project only the pixel from the pinhole to the spherical domain
+  // -> It's gonna create hole in the spherical image for top and bottom image
+  // The indirect mapping is sampling from destimation to source
+  // Direct mapping is mapping from source to destination and so cannot fill all the pixel if the cubic image is small
+}
+
+TEST(Ressampling, PinholeToSpherical_SphericalToPinhole)
+{
+  Image<RGBColor> spherical_image(512, 256);
+  SphericalToPinholeIndirect(spherical_image);
+
+  openMVG::cameras::Pinhole_Intrinsic pinhole_camera = spherical::ComputeCubicCameraIntrinsics(64);
+  std::vector<Image<RGBColor>> cube_images;
+  spherical::SphericalToCubic(spherical_image, pinhole_camera, cube_images, image::Sampler2d<image::SamplerNearest>());
+
+  WriteImage("cube_0.png", cube_images[0]);
+  WriteImage("cube_1.png", cube_images[1]);
+  WriteImage("cube_2.png", cube_images[2]);
+  WriteImage("cube_3.png", cube_images[3]);
+  WriteImage("cube_4.png", cube_images[4]);
+  WriteImage("cube_5.png", cube_images[5]);
+
+  //WriteImage("test_pinhole_to_spherical_indirect.png", spherical_image);
+}
+
+/* ************************************************************************* */
+int main() { TestResult tr; return TestRegistry::runAllTests(tr);}
+/* ************************************************************************* */

src/openMVG/spherical/cubic_image_sampler.hpp

@@ -0,0 +1,76 @@
+// This file is part of OpenMVG, an Open Multiple View Geometry C++ library.
+
+// Copyright (c) 2020 Pierre MOULON.
+
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include "openMVG/spherical/image_resampling.hpp"
+
+#include <array>
+
+namespace openMVG
+{
+namespace spherical
+{
+
+/// Compute a rectilinear camera focal from a given angular desired FoV
+double FocalFromPinholeHeight
+(
+  int h,
+  double fov_radian = openMVG::D2R(45) // Camera FoV
+)
+{
+  return h / (2 * tan(fov_radian));
+}
+
+const static std::array<openMVG::Mat3,6> GetCubicRotations()
+{
+  using namespace openMVG;
+  return {
+    RotationAroundY(D2R(0)) ,   // front
+    RotationAroundY(D2R(-90)),   // right
+    RotationAroundY(D2R(-180)),  // behind
+    RotationAroundY(D2R(-270)),  // left
+    RotationAroundX(D2R(-90)),   // up
+    RotationAroundX(D2R(+90))    // down
+  };
+}
+
+openMVG::cameras::Pinhole_Intrinsic ComputeCubicCameraIntrinsics
+(
+  const int cubic_image_size,
+  const double fov = D2R(45)
+)
+{
+  const double focal = spherical::FocalFromPinholeHeight(cubic_image_size, fov);
+  const double principal_point_xy = cubic_image_size / 2;
+  return cameras::Pinhole_Intrinsic(cubic_image_size,
+                                     cubic_image_size,
+                                     focal,
+                                     principal_point_xy,
+                                     principal_point_xy);
+}
+
+template <typename ImageT, typename SamplerT>
+void SphericalToCubic
+(
+  const ImageT & equirectangular_image,
+  const openMVG::cameras::Pinhole_Intrinsic & pinhole_camera,
+  std::vector<ImageT> & cube_images,
+  const SamplerT sampler = image::Sampler2d<image::SamplerLinear>()
+)
+{
+  const std::array<Mat3, 6> rot_matrix = GetCubicRotations();
+  const std::vector<Mat3> rot_matrix_vec(rot_matrix.cbegin(), rot_matrix.cend());
+  SphericalToPinholes(
+      equirectangular_image,
+      pinhole_camera,
+      cube_images,
+      rot_matrix_vec,
+      sampler);
+}
+
+} // namespace spherical
+} // namespace openMVG

src/openMVG/spherical/image_resampling.hpp

@@ -0,0 +1,99 @@
+// This file is part of OpenMVG, an Open Multiple View Geometry C++ library.
+
+// Copyright (c) 2020 Pierre MOULON.
+
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include "openMVG/cameras/Camera_Pinhole.hpp"
+#include "openMVG/cameras/Camera_Spherical.hpp"
+#include "openMVG/image/sample.hpp"
+#include "openMVG/numeric/eigen_alias_definition.hpp"
+
+namespace openMVG
+{
+namespace spherical
+{
+
+// Backward rendering of a pinhole image for a given rotation in a panorama
+template <typename ImageT, typename SamplerT>
+ImageT SphericalToPinhole
+(
+  const ImageT & equirectangular_image,
+  const openMVG::cameras::Pinhole_Intrinsic & pinhole_camera,
+  const Mat3 & rot_matrix = Mat3::Identity(),
+  const SamplerT sampler = image::Sampler2d<image::SamplerLinear>()
+)
+{
+  using namespace openMVG;
+  using namespace openMVG::cameras;
+  //
+  // Initialize a camera model for each image domain
+  // - the equirectangular panorama
+  const Intrinsic_Spherical sphere_camera(equirectangular_image.Width()  - 1,
+                                          equirectangular_image.Height() - 1);
+
+  // Perform backward/inverse rendering:
+  // - For each destination pixel in the pinhole image,
+  //   compute where to pick the pixel in the panorama image.
+  // This is done by using bearing vector computation
+
+  const int renderer_image_size = pinhole_camera.h();
+  ImageT pinhole_image(renderer_image_size, renderer_image_size);
+
+  const int image_width = pinhole_image.Width();
+  const int image_height = pinhole_image.Height();
+
+  // Use image coordinate in a matrix to use OpenMVG camera bearing vector vectorization
+  Mat2X xy_coords(2, static_cast<int>(image_width * image_height));
+  for (int y = 0; y < image_height; ++y)
+    for (int x = 0; x < image_width; ++x)
+      xy_coords.col(x + image_width * y ) << x +.5 , y + .5;
+
+  // Compute bearing vectors
+  const Mat3X bearing_vectors = pinhole_camera(xy_coords);
+
+  // Compute rotation bearings
+  const Mat3X rotated_bearings = rot_matrix * bearing_vectors;
+  // For every pinhole image pixels
+  #pragma omp parallel for
+  for (int it = 0; it < rotated_bearings.cols(); ++it)
+  {
+    // Project the bearing vector to the sphere
+    const Vec2 sphere_proj = sphere_camera.project(rotated_bearings.col(it));
+    // and use the corresponding pixel location in the panorama
+    const Vec2 xy = xy_coords.col(it);
+    if (equirectangular_image.Contains(sphere_proj.y(), sphere_proj.x()))
+    {
+      pinhole_image(xy.y(), xy.x()) = sampler(equirectangular_image, sphere_proj.y(), sphere_proj.x());
+    }
+  }
+  return pinhole_image;
+}
+
+// Sample pinhole image from a panorama given some camera rotations
+template <typename ImageT, typename SamplerT>
+void SphericalToPinholes
+(
+  const ImageT & equirectangular_image,
+  const openMVG::cameras::Pinhole_Intrinsic & pinhole_camera,
+  std::vector<ImageT> & pinhole_images,
+  const std::vector<Mat3> & rotations,
+  const SamplerT sampler = image::Sampler2d<image::SamplerLinear>()
+)
+{
+  pinhole_images.resize(6);
+  // render each cube faces
+  for (int i_rot = 0; i_rot < rotations.size(); ++i_rot)
+  {
+    pinhole_images[i_rot] = SphericalToPinhole(
+        equirectangular_image,
+        pinhole_camera,
+        rotations[i_rot],
+        sampler);
+  }
+}
+
+} // namespace spherical
+} // namespace openMVG