implement point format 6, 7, and 8 (only) support

bu/CopcSupport.hpp

@@ -20,16 +20,70 @@
 #include <lazperf/lazperf.hpp>
 #include <lazperf/vlr.hpp>
 
+
 namespace untwine
 {
 
-struct BaseInfo;
+class BaseInfo;
 struct FileDimInfo;
 using DimInfoList = std::vector<FileDimInfo>;
 
 namespace bu
 {
 
+
+struct copc_extents_vlr : public lazperf::vlr
+{
+public:
+
+    struct CopcExtent
+    {
+        double minimum;
+        double maximum;
+
+        CopcExtent(double minimum, double maximum);
+        CopcExtent() {};
+    };
+
+    std::vector<CopcExtent> items;
+
+    copc_extents_vlr(int itemCount);
+    copc_extents_vlr();
+    virtual ~copc_extents_vlr();
+
+    static copc_extents_vlr create(std::istream& in, int byteSize);
+    void read(std::istream& in, int byteSize);
+    void write(std::ostream& out) const;
+    virtual size_t size() const;
+    virtual lazperf::vlr_header header() const;
+};
+
+struct copc_info_vlr : public lazperf::vlr
+{
+public:
+    int64_t span {0};
+    uint64_t root_hier_offset {0};
+    uint64_t root_hier_size {0};
+    uint64_t laz_vlr_offset {0};
+    uint64_t laz_vlr_size {0};
+    uint64_t wkt_vlr_offset {0};
+    uint64_t wkt_vlr_size {0};
+    uint64_t eb_vlr_offset {0};
+    uint64_t eb_vlr_size {0};
+    uint64_t extent_vlr_offset {0};
+    uint64_t extent_vlr_size {0};
+    uint64_t reserved[9] {0};
+
+    copc_info_vlr();
+    virtual ~copc_info_vlr();
+
+    static copc_info_vlr create(std::istream& in);
+    void read(std::istream& in);
+    void write(std::ostream& out) const;
+    virtual size_t size() const;
+    virtual lazperf::vlr_header header() const;
+};
+
 class CopcSupport
 {
 public:
@@ -39,6 +93,7 @@ class CopcSupport
         int32_t byteSize;
         int32_t pointCount;
     };
+
     using CountMap = std::unordered_map<VoxelKey, int>;
     using Entries = std::vector<std::pair<VoxelKey, Hierarchy>>;
 
@@ -50,21 +105,42 @@ class CopcSupport
     void writeHierarchy(const CountMap& counts);
 
 private:
+
+
+    struct VLRInfo
+    {
+        int ebVLRSize {0};
+        int ebVLRCount {0};
+        int extentVLRCount {0};
+        DimInfoList ebDims;
+        pdal::Dimension::IdList statsDims;
+
+        VLRInfo();
+    };
+
+    BaseInfo m_b;
     std::ofstream m_f;
     lazperf::header14 m_header;
-    lazperf::copc_vlr m_copcVlr;
+    copc_info_vlr m_copcVlr;
     lazperf::laz_vlr m_lazVlr;
     lazperf::eb_vlr m_ebVlr;
     lazperf::wkt_vlr m_wktVlr;
+    copc_extents_vlr m_extentVlr;
     std::vector<lazperf::chunk> m_chunkTable;
     uint64_t m_chunkOffsetPos;
     uint64_t m_pointPos;
     std::unordered_map<VoxelKey, Hierarchy> m_hierarchy;
 
-    int extraByteSize(const DimInfoList& dims) const;
+    int ebVLRSize() const;
+    int ebVLRCount() const;
+    int extentVLRCount() const;
     Hierarchy emitRoot(const VoxelKey& root, const CountMap& counts);
     void emitChildren(const VoxelKey& root, const CountMap& counts,
         Entries& entries, int stopLevel);
+
+    VLRInfo computeVLRInfo() const;
+    void setEbVLR();
+
 };
 
 

bu/Processor.cpp

@@ -302,30 +302,53 @@ Processor::writeOctantCompressed(const OctantInfo& o, Index& index, IndexIter po
     // there's no reason why it should change. We should modify things to use a single
     // layout.
 
-    // PDRF 3 dim list
+    // Start with PDRF 6 dim list
     Dimension::IdList lasDims { Dimension::Id::X, Dimension::Id::Y, Dimension::Id::Z,
         Dimension::Id::Intensity, Dimension::Id::ReturnNumber, Dimension::Id::NumberOfReturns,
         Dimension::Id::ScanDirectionFlag, Dimension::Id::EdgeOfFlightLine,
         Dimension::Id::Classification, Dimension::Id::ScanAngleRank, Dimension::Id::UserData,
-        Dimension::Id::PointSourceId, Dimension::Id::GpsTime, Dimension::Id::Red,
-        Dimension::Id::Green, Dimension::Id::Blue };
+        Dimension::Id::PointSourceId, Dimension::Id::GpsTime };
+
+    if (m_b.opts.pointFormatId == 7)
+    {
+        lasDims.push_back(Dimension::Id::Red);
+        lasDims.push_back(Dimension::Id::Green);
+        lasDims.push_back(Dimension::Id::Blue);
+    }
+    if (m_b.opts.pointFormatId == 8)
+    {
+        lasDims.push_back(Dimension::Id::Red);
+        lasDims.push_back(Dimension::Id::Green);
+        lasDims.push_back(Dimension::Id::Blue);
+        lasDims.push_back(Dimension::Id::Infrared);
+    }
+
     DimInfoList dims = m_b.dimInfo;
     m_extraDims.clear();
     for (FileDimInfo& fdi : dims)
     {
-        fdi.dim = table.layout()->registerOrAssignDim(fdi.name, fdi.type);
-        if (m_b.opts.stats)
+        // register dimension if we are in lasDims
+        Dimension::Id candidate = pdal::Dimension::id(fdi.name);
+        if (Utils::contains(lasDims, candidate))
         {
-            // For single file output we need the counts by return number.
-            if (fdi.dim == pdal::Dimension::Id::Classification)
-                stats.push_back({fdi.dim, Stats(fdi.name, Stats::EnumType::Enumerate, false)});
-            else if (fdi.dim == pdal::Dimension::Id::ReturnNumber && m_b.opts.singleFile)
-                stats.push_back({fdi.dim, Stats(fdi.name, Stats::EnumType::Enumerate, false)});
-            else
-                stats.push_back({fdi.dim, Stats(fdi.name, Stats::EnumType::NoEnum, false)});
-        }
-        if (!Utils::contains(lasDims, fdi.dim))
+            // we add this one
+            fdi.dim = table.layout()->registerOrAssignDim(fdi.name, fdi.type);
+            if (m_b.opts.stats)
+            {
+                // For single file output we need the counts by return number.
+                if (fdi.dim == pdal::Dimension::Id::Classification)
+                    stats.push_back({fdi.dim, Stats(fdi.name, Stats::EnumType::Enumerate, false)});
+                else if (fdi.dim == pdal::Dimension::Id::ReturnNumber && m_b.opts.singleFile)
+                    stats.push_back({fdi.dim, Stats(fdi.name, Stats::EnumType::Enumerate, false)});
+                else
+                    stats.push_back({fdi.dim, Stats(fdi.name, Stats::EnumType::NoEnum, false)});
+            }
+        } else
+        {
+            fdi.dim = table.layout()->registerOrAssignDim(fdi.name, fdi.type);
             m_extraDims.push_back(DimType(fdi.dim, fdi.type));
+            stats.push_back({fdi.dim, Stats(fdi.name, Stats::EnumType::NoEnum, false)});
+        }
     }
     table.finalize();
 
@@ -398,7 +421,7 @@ void Processor::appendCompressed(pdal::PointViewPtr view, const DimInfoList& dim
 void Processor::flushCompressed(pdal::PointTableRef table, pdal::PointViewPtr view,
     const OctantInfo& oi, IndexedStats& stats)
 {
-    // For single file output we need the stats for 
+    // For single file output we need the stats for
     if (m_b.opts.stats)
     {
         for (pdal::PointId id = 0; id < view->size(); ++id)
@@ -419,7 +442,7 @@ void Processor::flushCompressed(pdal::PointTableRef table, pdal::PointViewPtr vi
     else
     {
         std::string filename = m_b.opts.outputName + "/ept-data/" + oi.key().toString() + ".laz";
-        writeEptFile(filename, table, view); 
+        writeEptFile(filename, table, view);
     }
 }
 
@@ -474,9 +497,9 @@ void Processor::createChunk(const VoxelKey& key, pdal::PointViewPtr view)
     for (DimType dim : m_extraDims)
         ebCount += layout->dimSize(dim.m_id);
 
-    std::vector<char> buf(lazperf::baseCount(3) + ebCount);
+    std::vector<char> buf(lazperf::baseCount(m_b.opts.pointFormatId) + ebCount);
 
-    lazperf::writer::chunk_compressor compressor(3, ebCount);
+    lazperf::writer::chunk_compressor compressor(m_b.opts.pointFormatId, ebCount);
     for (PointId idx = 0; idx < view->size(); ++idx)
     {
         PointRef point(*view, idx);
@@ -486,6 +509,7 @@ void Processor::createChunk(const VoxelKey& key, pdal::PointViewPtr view)
     std::vector<unsigned char> chunk = compressor.done();
 
     uint64_t location = m_manager.newChunk(key, chunk.size(), (uint32_t)view->size());
+//     std::cout << "made new chunk: " << location << " with size: " << chunk.size() << std::endl;
 
     std::ofstream out(m_b.opts.outputName, std::ios::out | std::ios::in | std::ios::binary);
     out.seekp(std::ofstream::pos_type(location));
@@ -501,11 +525,11 @@ void Processor::fillPointBuf(pdal::PointRef& point, std::vector<char>& buf)
 
     LeInserter ostream(buf.data(), buf.size());
 
-// We're currently only writing PDRF 3.
-    bool has14PointFormat = false;
+    // We're only write PDRF 6, 7, or 8.
+    bool has14PointFormat = true;
     bool hasTime = true; //  m_lasHeader.hasTime();
-    bool hasColor = true; // m_lasHeader.hasColor();
-    bool hasInfrared = false; // m_lasHeader.hasInfrared();
+    bool hasColor = m_b.opts.pointFormatId == 7 || m_b.opts.pointFormatId == 8; // m_lasHeader.hasColor();
+    bool hasInfrared = m_b.opts.pointFormatId == 8; // m_lasHeader.hasInfrared();
 
 //    static const size_t maxReturnCount = m_lasHeader.maxReturnCount();
 

cmake/unix_compiler_options.cmake

@@ -13,15 +13,14 @@ function(untwine_target_compile_settings target)
                 -Wno-noexcept-type
             )
         endif()
-        set(PDAL_COMPILER_GCC 1)
+        set(UNTWINE_COMPILER_GCC 1)
     elseif (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
-        set(PDAL_COMPILER_CLANG 1)
+        set(UNTWINE_COMPILER_CLANG 1)
     else()
         message(FATAL_ERROR "Unsupported C++ compiler")
     endif()
 
     target_compile_options(${target} PRIVATE
-        ${PDAL_CXX_STANDARD}
         -Wall
         -Wextra
         -Wpointer-arith
@@ -39,7 +38,7 @@ function(untwine_target_compile_settings target)
 
         -Werror
     )
-    if (PDAL_COMPILER_CLANG)
+    if (UNTWINE_COMPILER_CLANG)
         target_compile_options(${target} PRIVATE
             -Wno-unknown-warning-option
         )

epf/Epf.cpp

@@ -76,9 +76,15 @@ void Epf::run(ProgressWriter& progress)
     PointLayoutPtr layout(new PointLayout());
     for (const std::string& dimName : allDimNames)
     {
-        Dimension::Type type = Dimension::defaultType(Dimension::id(dimName));
-        if (type == Dimension::Type::None)
+        Dimension::Type type;
+        try
+        {
+            type = Dimension::defaultType(Dimension::id(dimName));
+        }
+        catch (pdal::pdal_error&)
+        {
             type = Dimension::Type::Double;
+        }
         layout->registerOrAssignDim(dimName, type);
     }
     layout->finalize();
@@ -156,18 +162,50 @@ void Epf::run(ProgressWriter& progress)
 
 void Epf::fillMetadata(const pdal::PointLayoutPtr layout)
 {
+    using namespace pdal;
+
     // Info to be passed to sampler.
     m_b.bounds = m_grid.processingBounds();
     m_b.trueBounds = m_grid.conformingBounds();
     if (m_srsFileInfo.valid())
         m_b.srs = m_srsFileInfo.srs;
     m_b.pointSize = 0;
+
+
+    auto check_dimension_exists = [] (pdal::Dimension::Id dim,  DimInfoList const& dimInfo)
+    {
+        auto it = std::find_if(dimInfo.begin(), dimInfo.end(),
+                               [&dim](const FileDimInfo& fdi) { return fdi.dim == dim; });
+
+        if (it == dimInfo.end()) return false;
+
+        return true;
+    };
+
+    // Set the pointFormatId based on whether or not colors exist in the
+    // file
+    int old (m_b.opts.pointFormatId);
+
+    if (check_dimension_exists(pdal::Dimension::Id::Infrared, m_b.dimInfo))
+    {
+        m_b.opts.pointFormatId = 8;
+    } else if (check_dimension_exists(pdal::Dimension::Id::Red, m_b.dimInfo) ||
+               check_dimension_exists(pdal::Dimension::Id::Green, m_b.dimInfo) ||
+               check_dimension_exists(pdal::Dimension::Id::Blue, m_b.dimInfo))
+    {
+        m_b.opts.pointFormatId = 7;
+    } else
+    {
+        m_b.opts.pointFormatId = 6;
+    }
+
     for (pdal::Dimension::Id id : layout->dims())
     {
         FileDimInfo di;
         di.name = layout->dimName(id);
         di.type = layout->dimType(id);
         di.offset = layout->dimOffset(id);
+        di.dim = id;
         m_b.pointSize += pdal::Dimension::size(di.type);
         m_b.dimInfo.push_back(di);
     }
@@ -192,6 +230,7 @@ void Epf::fillMetadata(const pdal::PointLayoutPtr layout)
     m_b.scale[0] = calcScale(m_b.scale[0], m_b.bounds.minx, m_b.bounds.maxx);
     m_b.scale[1] = calcScale(m_b.scale[1], m_b.bounds.miny, m_b.bounds.maxy);
     m_b.scale[2] = calcScale(m_b.scale[2], m_b.bounds.minz, m_b.bounds.maxz);
+
 }
 
 PointCount Epf::createFileInfo(const StringList& input, StringList dimNames,

untwine/Common.hpp

@@ -31,10 +31,14 @@ struct Options
     int progressFd;
     StringList dimNames;
     bool stats;
+    int pointFormatId;
 };
 
-struct BaseInfo
+class BaseInfo
 {
+public:
+    BaseInfo() {};
+
     Options opts;
     pdal::BOX3D bounds;
     pdal::BOX3D trueBounds;
@@ -46,6 +50,7 @@ struct BaseInfo
     using d3 = std::array<double, 3>;
     d3 scale { -1.0, -1.0, -1.0 };
     d3 offset {};
+
 };
 
 const std::string MetadataFilename {"info2.txt"};

untwine/FileDimInfo.hpp

@@ -29,6 +29,12 @@ struct FileDimInfo
     pdal::Dimension::Type type;
     int offset;
     pdal::Dimension::Id dim;
+
+    bool operator==(const FileDimInfo& r) const
+    {
+       return dim == r.dim;
+    }
+
 };
 using DimInfoList = std::vector<FileDimInfo>;
 

untwine/Untwine.cpp

@@ -51,6 +51,8 @@ void addArgs(pdal::ProgramArgs& programArgs, Options& options, pdal::Arg * &temp
         "loaded.", options.dimNames);
     programArgs.add("stats", "Generate statistics for dimensions in the manner of Entwine.",
         options.stats);
+    programArgs.add("format", "Point format to write (only 6, 7, or 8 allowed)",
+        options.pointFormatId, 6);
 }
 
 bool handleOptions(pdal::StringList& arglist, Options& options)