Merge pull request #1244 from alicevision/dev/largePanoramaMemoryClean

Update panorama pipeline for very large panoramas

src/aliceVision/camera/IntrinsicBase.hpp

@@ -455,6 +455,20 @@ class IntrinsicBase
     return true;
   }
 
+  /**
+   * @brief Return true if these pixel coordinates should be visible in the image
+   * @param pix input pixel coordinates to check for visibility
+   * @return true if visible
+   */
+  virtual bool isVisible(const Vec2f & pix) const {
+
+    if (pix(0) < 0 || pix(0) >= _w || pix(1) < 0 || pix(1) >= _h) {
+      return false;
+    }
+
+    return true;
+  }
+
   /**
    * @brief Assuming the distortion is a function of radius, estimate the 
    * maximal undistorted radius for a range of distorted radius.

src/aliceVision/image/pixelTypes.hpp

@@ -392,6 +392,17 @@ namespace aliceVision
                      T( ( Z )( *this )( 2 ) * val ),
                      T( ( Z )( *this )( 3 ) * val ) );
       }
+
+      /**
+      * @brief Elementwise addition
+      * @param other the other element to substract
+      * @return Rgb color after addition
+      * @note This does not modify the Rgb value (ie: only return a modified copy)
+      */
+      inline Rgba operator +( const Rgba& other ) const
+      {
+        return Rgba( ((*this)(0) + other(0)), ((*this)(1) + other(1)), ((*this)(2) + other(2)), ((*this)(3) + other(3)));
+      }
     };
 
     typedef Rgba<unsigned char> RGBAColor;

src/aliceVision/panorama/coordinatesMap.cpp

@@ -10,9 +10,10 @@ bool CoordinatesMap::build(const std::pair<int, int>& panoramaSize, const geomet
 {
 
     /* Effectively compute the warping map */
-    _coordinates = aliceVision::image::Image<Eigen::Vector2d>(coarseBbox.width, coarseBbox.height, false);
+    _coordinates = aliceVision::image::Image<Eigen::Vector2f>(coarseBbox.width, coarseBbox.height, false);
     _mask = aliceVision::image::Image<unsigned char>(coarseBbox.width, coarseBbox.height, true, 0);
 
+
     int max_x = 0;
     int max_y = 0;
     int min_x = std::numeric_limits<int>::max();
@@ -47,7 +48,8 @@ bool CoordinatesMap::build(const std::pair<int, int>& panoramaSize, const geomet
             /**
              * Project this ray to camera pixel coordinates
              */
-            const Vec2 pix_disto = intrinsics.project(pose, ray.homogeneous(), true);
+            const Vec2 pix_disto_d = intrinsics.project(pose, ray.homogeneous(), true);
+            const Vec2f pix_disto = pix_disto_d.cast<float>();
 
             /**
              * Ignore invalid coordinates

src/aliceVision/panorama/coordinatesMap.hpp

@@ -28,15 +28,15 @@ class CoordinatesMap
 
     BoundingBox getBoundingBox() const { return _boundingBox; }
 
-    const aliceVision::image::Image<Eigen::Vector2d>& getCoordinates() const { return _coordinates; }
+    const aliceVision::image::Image<Eigen::Vector2f>& getCoordinates() const { return _coordinates; }
 
     const aliceVision::image::Image<unsigned char>& getMask() const { return _mask; }
 
 private:
     size_t _offset_x = 0;
     size_t _offset_y = 0;
 
-    aliceVision::image::Image<Eigen::Vector2d> _coordinates;
+    aliceVision::image::Image<Eigen::Vector2f> _coordinates;
     aliceVision::image::Image<unsigned char> _mask;
     BoundingBox _boundingBox;
 };

src/aliceVision/panorama/distance.cpp

@@ -6,7 +6,7 @@ namespace aliceVision
 bool distanceToCenter(aliceVision::image::Image<float>& _weights, const CoordinatesMap& map, int width, int height)
 {
 
-    const aliceVision::image::Image<Eigen::Vector2d>& coordinates = map.getCoordinates();
+    const aliceVision::image::Image<Eigen::Vector2f>& coordinates = map.getCoordinates();
     const aliceVision::image::Image<unsigned char>& mask = map.getMask();
 
     float cx = width / 2.0f;
@@ -27,7 +27,7 @@ bool distanceToCenter(aliceVision::image::Image<float>& _weights, const Coordina
                 continue;
             }
 
-            const Vec2& coords = coordinates(i, j);
+            const Vec2f& coords = coordinates(i, j);
 
             float x = coords(0);
             float y = coords(1);

src/aliceVision/panorama/gaussian.cpp

@@ -41,11 +41,6 @@ bool GaussianPyramidNoMask::process(const image::Image<image::RGBfColor>& input)
     if(input.Width() != _pyramid_color[0].Width())
         return false;
 
-    /**
-     * Kernel
-     */
-    oiio::ImageBuf K = oiio::ImageBufAlgo::make_kernel("gaussian", 5, 5);
-
     /**
      * Build pyramid
      */
@@ -56,12 +51,7 @@ bool GaussianPyramidNoMask::process(const image::Image<image::RGBfColor>& input)
         const image::Image<image::RGBfColor>& source = _pyramid_color[lvl];
         image::Image<image::RGBfColor>& dst = _filter_buffer[lvl];
 
-        oiio::ImageSpec spec(source.Width(), source.Height(), 3, oiio::TypeDesc::FLOAT);
-
-        const oiio::ImageBuf inBuf(spec, const_cast<image::RGBfColor*>(source.data()));
-        oiio::ImageBuf outBuf(spec, dst.data());
-        oiio::ImageBufAlgo::convolve(outBuf, inBuf, K);
-
+        convolveGaussian5x5(dst, source);
         downscale(_pyramid_color[lvl + 1], _filter_buffer[lvl]);
     }
 

src/aliceVision/panorama/warper.cpp

@@ -14,7 +14,7 @@ bool Warper::warp(const CoordinatesMap& map, const aliceVision::image::Image<ima
     _mask = map.getMask();
 
     const image::Sampler2d<image::SamplerLinear> sampler;
-    const aliceVision::image::Image<Eigen::Vector2d>& coordinates = map.getCoordinates();
+    const aliceVision::image::Image<Eigen::Vector2f>& coordinates = map.getCoordinates();
 
     /**
      * Create buffer
@@ -36,7 +36,7 @@ bool Warper::warp(const CoordinatesMap& map, const aliceVision::image::Image<ima
                 continue;
             }
 
-            const Eigen::Vector2d& coord = coordinates(i, j);
+            const Eigen::Vector2f& coord = coordinates(i, j);
             const image::RGBfColor pixel = sampler(source, coord(1), coord(0));
 
             _color(i, j) = pixel;
@@ -57,13 +57,13 @@ bool GaussianWarper::warp(const CoordinatesMap& map, const GaussianPyramidNoMask
     _mask = map.getMask();
 
     const image::Sampler2d<image::SamplerLinear> sampler;
-    const aliceVision::image::Image<Eigen::Vector2d>& coordinates = map.getCoordinates();
+    const aliceVision::image::Image<Eigen::Vector2f>& coordinates = map.getCoordinates();
 
     /**
      * Create a pyramid for input
      */
     const std::vector<image::Image<image::RGBfColor>>& mlsource = pyramid.getPyramidColor();
-    size_t max_level = pyramid.getScalesCount() - 1;
+    int max_level = pyramid.getScalesCount() - 1;
 
     /**
      * Create buffer
@@ -76,6 +76,14 @@ bool GaussianWarper::warp(const CoordinatesMap& map, const GaussianPyramidNoMask
      */
     for(size_t i = 0; i < _color.Height(); i++)
     {
+        int next_i = i + 1;
+
+        if (i == _color.Height() - 1)
+        {
+            next_i = i - 1;
+        }
+
+
         for(size_t j = 0; j < _color.Width(); j++)
         {
 
@@ -86,45 +94,42 @@ bool GaussianWarper::warp(const CoordinatesMap& map, const GaussianPyramidNoMask
             }
 
             int next_j = j + 1;
-            int next_i = i + 1;
+
 
             if (j == _color.Width() - 1)
             {
                 next_j = j - 1;
             }
 
-            if (i == _color.Height() - 1)
-            {
-                next_i = i - 1;
-            }
-
+
             if (!_mask(next_i, j) || !_mask(i, next_j))
             {
-                const Eigen::Vector2d& coord = coordinates(i, j);
+                const Eigen::Vector2f& coord = coordinates(i, j);
                 const image::RGBfColor pixel = sampler(mlsource[0], coord(1), coord(0));
                 _color(i, j) = pixel;
 
                 continue;
             }
 
-            const Eigen::Vector2d& coord_mm = coordinates(i, j);
-            const Eigen::Vector2d& coord_mp = coordinates(i, next_j);
-            const Eigen::Vector2d& coord_pm = coordinates(next_i, j);
+            const Eigen::Vector2f& coord_mm = coordinates(i, j);
+            const Eigen::Vector2f& coord_mp = coordinates(i, next_j);
+            const Eigen::Vector2f& coord_pm = coordinates(next_i, j);
 
-            double dxx = coord_pm(0) - coord_mm(0);
-            double dxy = coord_mp(0) - coord_mm(0);
-            double dyx = coord_pm(1) - coord_mm(1);
-            double dyy = coord_mp(1) - coord_mm(1);
-            double det = std::abs(dxx * dyy - dxy * dyx);
-            double scale = sqrt(det);
+            float dxx = coord_pm(0) - coord_mm(0);
+            float dxy = coord_mp(0) - coord_mm(0);
+            float dyx = coord_pm(1) - coord_mm(1);
+            float dyy = coord_mp(1) - coord_mm(1);
+            float det = std::abs(dxx * dyy - dxy * dyx);
+            float scale = /*sqrtf*/(det); //logsqrt = 0.5log
 
-            double flevel = std::max(0.0, log2(scale));
-            size_t blevel = std::min(max_level, size_t(floor(flevel)));
+            float flevel = std::max(0.0f,0.5f * log2f(scale));
+            int blevel = std::min(max_level, int(floor(flevel)));
 
-            double dscale, x, y;
-            dscale = 1.0 / pow(2.0, blevel);
+            float dscale, x, y;
+            dscale = pow(2.0, -blevel);
             x = coord_mm(0) * dscale;
             y = coord_mm(1) * dscale;
+
             /*Fallback to first level if outside*/
             if(x >= mlsource[blevel].Width() - 1 || y >= mlsource[blevel].Height() - 1)
             {
@@ -133,6 +138,7 @@ bool GaussianWarper::warp(const CoordinatesMap& map, const GaussianPyramidNoMask
             }
 
             _color(i, j) = sampler(mlsource[blevel], y, x);
+
             if (clamp)
             {
                 if (_color(i, j).r() > HALF_MAX) _color(i, j).r() = HALF_MAX;

src/aliceVision/sfmDataIO/viewIO.hpp

@@ -11,6 +11,7 @@
 #include <aliceVision/camera/IntrinsicBase.hpp>
 
 #include <boost/filesystem.hpp>
+#include <aliceVision/sfmData/uid.hpp>
 
 #include <memory>
 

src/software/pipeline/CMakeLists.txt

@@ -178,6 +178,13 @@ if(ALICEVISION_BUILD_SFM)
           aliceVision_panorama
           ${Boost_LIBRARIES}
   )
+  alicevision_add_software(aliceVision_panoramaPostProcessing
+    SOURCE main_panoramaPostProcessing.cpp
+    FOLDER ${FOLDER_SOFTWARE_PIPELINE}
+    LINKS aliceVision_system
+          aliceVision_image
+          ${Boost_LIBRARIES}
+  )
   alicevision_add_software(aliceVision_panoramaCompositing
     SOURCE main_panoramaCompositing.cpp
     FOLDER ${FOLDER_SOFTWARE_PIPELINE}