[untwine] Fix precision issues when indexing

Offset and scale of X/Y/Z dimensions were not written and
some datasets would simply fail to get indexed completely.
Other datasets may have had the data slightly misaligned
after indexing.

References:
hobuinc/untwine#28
hobuinc/untwine#46

external/untwine/bu/BuPyramid.cpp

@@ -8,7 +8,6 @@
 #include <pdal/util/ProgramArgs.hpp>
 
 #include "BuPyramid.hpp"
-#include "BuTypes.hpp"
 #include "FileInfo.hpp"
 #include "OctantInfo.hpp"
 #include "../untwine/Common.hpp"
@@ -21,7 +20,7 @@ namespace bu
 
 /// BuPyramid
 
-BuPyramid::BuPyramid() : m_manager(m_b)
+BuPyramid::BuPyramid(BaseInfo& common) : m_b(common), m_manager(m_b)
 {}
 
 
@@ -31,7 +30,6 @@ void BuPyramid::run(const Options& options, ProgressWriter& progress)
     m_b.outputDir = options.outputDir;
     m_b.stats = options.stats;
 
-    readBaseInfo();
     getInputFiles();
     size_t count = queueWork();
 
@@ -44,76 +42,6 @@ void BuPyramid::run(const Options& options, ProgressWriter& progress)
 }
 
 
-void BuPyramid::readBaseInfo()
-{
-    auto nextblock = [](std::istream& in)
-    {
-        std::string s;
-        bool firstnl = false;
-
-        while (in)
-        {
-            char c;
-            in.get(c);
-            if (c == '\n')
-            {
-                if (firstnl)
-                {
-                    // Remove trailing newline.
-                    s.resize(s.size() - 1);
-                    return s;
-                }
-                else
-                    firstnl = true;
-            }
-            else
-                firstnl = false;
-            s += c;
-        }
-        return s;
-    };
-
-    std::string baseFilename = m_b.inputDir + "/" + MetadataFilename;
-    std::ifstream in(baseFilename);
-
-    if (!in)
-        fatal("Can't open '" + MetadataFilename + "' in directory '" + m_b.inputDir + "'.");
-
-    std::stringstream ss(nextblock(in));
-    ss >> m_b.bounds.minx >> m_b.bounds.miny >> m_b.bounds.minz;
-    ss >> m_b.bounds.maxx >> m_b.bounds.maxy >> m_b.bounds.maxz;
-
-    ss.str(nextblock(in));
-    ss.clear();
-    ss >> m_b.trueBounds.minx >> m_b.trueBounds.miny >> m_b.trueBounds.minz;
-    ss >> m_b.trueBounds.maxx >> m_b.trueBounds.maxy >> m_b.trueBounds.maxz;
-
-    std::string srs = nextblock(in);
-    if (srs != "NONE")
-        m_b.srs.set(srs);
-
-    if (!in)
-        throw "Couldn't read info file.";
-
-    ss.str(nextblock(in));
-    ss.clear();
-    m_b.pointSize = 0;
-    while (true)
-    {
-        FileDimInfo fdi;
-        ss >> fdi;
-        if (!ss)
-            break;
-        if (fdi.name.empty())
-            fatal("Invalid dimension in info.txt.");
-        m_b.pointSize += pdal::Dimension::size(fdi.type);
-        m_b.dimInfo.push_back(fdi);
-    }
-    if (m_b.pointSize == 0)
-        throw "Couldn't read info file.";
-}
-
-
 void BuPyramid::writeInfo()
 {
     auto typeString = [](pdal::Dimension::BaseType b)
@@ -138,9 +66,9 @@ void BuPyramid::writeInfo()
     out << "{\n";
 
     pdal::BOX3D& b = m_b.bounds;
-    std::ios init(NULL);
-    init.copyfmt(out);
-    out << std::fixed << std::setw(12);
+
+    // Set fixed output for bounds output to get sufficient precision.
+    out << std::fixed;
     out << "\"bounds\": [" <<
         b.minx << ", " << b.miny << ", " << b.minz << ", " <<
         b.maxx << ", " << b.maxy << ", " << b.maxz << "],\n";
@@ -149,6 +77,8 @@ void BuPyramid::writeInfo()
     out << "\"boundsConforming\": [" <<
         tb.minx << ", " << tb.miny << ", " << tb.minz << ", " <<
         tb.maxx << ", " << tb.maxy << ", " << tb.maxz << "],\n";
+    // Reset to default float output to match PDAL option handling for now.
+    out << std::defaultfloat;
 
     out << "\"dataType\": \"laszip\",\n";
     out << "\"hierarchyType\": \"json\",\n";
@@ -163,8 +93,12 @@ void BuPyramid::writeInfo()
         out << "\t{";
             out << "\"name\": \"" << fdi.name << "\", ";
             out << "\"type\": \"" << typeString(pdal::Dimension::base(fdi.type)) << "\", ";
-            if (fdi.name == "X" || fdi.name == "Y" || fdi.name == "Z")
-                out << "\"scale\": 0.01, \"offset\": 0, ";
+            if (fdi.name == "X")
+                out << "\"scale\": " << m_b.scale[0] << ", \"offset\": " << m_b.offset[0] << ", ";
+            if (fdi.name == "Y")
+                out << "\"scale\": " << m_b.scale[1] << ", \"offset\": " << m_b.offset[1] << ", ";
+            if (fdi.name == "Z")
+                out << "\"scale\": " << m_b.scale[2] << ", \"offset\": " << m_b.offset[2] << ", ";
             out << "\"size\": " << pdal::Dimension::size(fdi.type);
             const Stats *stats = m_manager.stats(fdi.name);
             if (stats)
@@ -205,7 +139,6 @@ void BuPyramid::writeInfo()
     out << "}\n";
 
     out << "}\n";
-    out.copyfmt(init);
 }
 
 

external/untwine/bu/BuPyramid.hpp

@@ -18,7 +18,6 @@
 #include <vector>
 
 #include "PyramidManager.hpp"
-#include "BuTypes.hpp"
 
 namespace pdal
 {
@@ -39,7 +38,7 @@ class FileInfo;
 class BuPyramid
 {
 public:
-    BuPyramid();
+    BuPyramid(BaseInfo& common);
     void run(const Options& options, ProgressWriter& progress);
 
 private:
@@ -48,8 +47,8 @@ class BuPyramid
     size_t queueWork();
     void writeInfo();
 
-    PyramidManager m_manager;
     BaseInfo m_b;
+    PyramidManager m_manager;
     std::unordered_map<VoxelKey, FileInfo> m_allFiles;
 };
 

external/untwine/bu/PointAccessor.hpp

@@ -16,7 +16,6 @@
 #include "../untwine/MapFile.hpp"
 #include "../untwine/Point.hpp"
 
-#include "BuTypes.hpp"
 #include "FileInfo.hpp"
 
 namespace untwine

external/untwine/bu/Processor.cpp

@@ -368,7 +368,6 @@ void Processor::appendCompressed(pdal::PointViewPtr view, const DimInfoList& dim
     }
 }
 
-
 void Processor::flushCompressed(pdal::PointTableRef table, pdal::PointViewPtr view,
     const OctantInfo& oi, IndexedStats& stats)
 {
@@ -412,6 +411,12 @@ void Processor::flushCompressed(pdal::PointTableRef table, pdal::PointViewPtr vi
     wopts.add("software_id", "Entwine 1.0");
     wopts.add("compression", "laszip");
     wopts.add("filename", filename);
+    wopts.add("offset_x", m_b.offset[0]);
+    wopts.add("offset_y", m_b.offset[1]);
+    wopts.add("offset_z", m_b.offset[2]);
+    wopts.add("scale_x", m_b.scale[0]);
+    wopts.add("scale_y", m_b.scale[1]);
+    wopts.add("scale_z", m_b.scale[2]);
     w->setOptions(wopts);
     w->setInput(*prev);
     // Set dataformat ID based on time/rgb, but for now accept the default.

external/untwine/bu/Processor.hpp

@@ -15,7 +15,6 @@
 #include <pdal/PointTable.hpp>
 #include <pdal/PointView.hpp>
 
-#include "BuTypes.hpp"
 #include "PointAccessor.hpp"
 #include "Stats.hpp"
 #include "VoxelInfo.hpp"

external/untwine/bu/PyramidManager.hpp

@@ -17,7 +17,6 @@
 #include <unordered_map>
 #include <vector>
 
-#include "BuTypes.hpp"
 #include "OctantInfo.hpp"
 #include "Stats.hpp"
 #include "../untwine/ThreadPool.hpp"

external/untwine/bu/VoxelInfo.hpp

@@ -123,7 +123,8 @@ class VoxelInfo
         double y = p.y() - m_bounds.miny;
         double z = p.z() - m_bounds.minz;
 
-        return GridKey((int)(x / m_gridCellWidth), (int)(y / m_gridCellWidth), (int)(z / m_gridCellWidth));
+        return GridKey((int)(x / m_gridCellWidth), (int)(y / m_gridCellWidth),
+            (int)(z / m_gridCellWidth));
     }
 
     //ABELL - Really torn WRT making Grid its own thing.
@@ -140,6 +141,9 @@ class VoxelInfo
     int gridZCount() const
         { return m_gridZCount; }
 
+    pdal::BOX3D bounds() const
+        { return m_bounds; }
+
 private:
     pdal::BOX3D m_fullBounds;
     pdal::BOX3D m_bounds;