Catch up to recent updates in support of QGIS (#124)

* Use sort filter for more generic code.

* Use generic WKT function, rather than WKT1 function.

* Add console output on exception.

* Convert filenames to UTF-16 on Windows.

* Add Windows.h

* Fix more UTF-16 issues.

* provide special directoryList for OSX due to std::filesystem missing bits

* doh

* add gcc_impl_linux-64

* add libstdcxx-ng

* fix up epf too

* missing include for epf

* fix up OSX-special directoryList function to match other usage

* start on wmain

* fromNative

* need <algorithm> for std::max

* no WIN32_EXECUTABLE

* need conda compilers

Co-authored-by: Andrew Bell <andrew.bell.ia@gmail.com>

CMakeLists.txt

@@ -22,6 +22,12 @@ file(GLOB SRCS
     lazperf/detail/*.cpp
 )
 
+
+if(MSVC)
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \
+        /SUBSYSTEM:WINDOWS /ENTRY:wmainCRTStartup")
+endif()
+
 add_executable(untwine ${SRCS})
 
 untwine_target_compile_settings(untwine)

api/CMakeLists.txt

@@ -4,7 +4,7 @@ include(FeatureSummary)
 include(${PROJECT_SOURCE_DIR}/../cmake/compiler_options.cmake)
 
 add_executable(qgt
-    QgisTest.cpp
+    QgisTest2.cpp
     QgisUntwine.cpp
 )
 

api/QgisTest2.cpp

@@ -0,0 +1,43 @@
+#ifndef _WIN32
+#include <unistd.h>
+#endif
+
+#include <iostream>
+
+#include "QgisUntwine.hpp"
+
+int main()
+{
+    untwine::QgisUntwine::StringList files;
+    untwine::QgisUntwine::Options options;
+    std::string exe = "C:\\Users\\andre\\untwine\\build\\untwine.exe";
+
+    untwine::QgisUntwine api(exe);
+
+    std::vector<unsigned char> funnycVec = { 0xc4, 0x8d };
+    std::string funnyc(funnycVec.begin(), funnycVec.end());
+    std::string v8string { "C:\\Users\\andre\\untwine\\api\\" + funnyc + "\\" + funnyc + ".las" };
+    files.push_back(v8string);
+    std::string outDir { "./out_" + funnyc };
+    bool ok = api.start(files, outDir, options);
+    if (! ok)
+    {
+        std::cerr << "Couldn't start '" << exe << "!\n";
+        exit(-1);
+    }
+
+    while (true)
+    {
+#ifdef _WIN32
+    	Sleep(1000);
+#else
+        ::sleep(1);
+#endif
+        int percent = api.progressPercent();
+        std::string s = api.progressMessage();
+        std::cerr << "Percent/Msg = " << percent << " / " << s << "!\n";
+        if (!api.running())
+            break;
+    }
+    std::cerr << "Error = " << api.errorMessage() << "\n";
+}

bu/BuPyramid.cpp

@@ -3,6 +3,7 @@
 #include <set>
 #include <string>
 #include <vector>
+#include <filesystem>
 
 #include <pdal/util/FileUtils.hpp>
 #include <pdal/util/ProgramArgs.hpp>
@@ -13,6 +14,75 @@
 #include "../untwine/Common.hpp"
 #include "../untwine/ProgressWriter.hpp"
 
+namespace
+{
+
+// PDAL's directoryList had a bug, so we've imported a working
+// version here so that we can still use older PDAL releases.
+
+#ifndef __APPLE_CC__
+std::vector<std::string> directoryList(const std::string& dir)
+{
+    namespace fs = std::filesystem;
+
+    std::vector<std::string> files;
+
+    try
+    {
+        fs::directory_iterator it(untwine::toNative(dir));
+        fs::directory_iterator end;
+        while (it != end)
+        {
+            files.push_back(untwine::fromNative(it->path()));
+            it++;
+        }
+    }
+    catch (fs::filesystem_error&)
+    {
+        files.clear();
+    }
+    return files;
+}
+#else
+
+#include <dirent.h>
+
+// Provide simple opendir/readdir solution for OSX because directory_iterator is
+// not available until OSX 10.15
+std::vector<std::string> directoryList(const std::string& dir)
+{
+
+    DIR *dpdf;
+    struct dirent *epdf;
+
+    std::vector<std::string> files;
+    dpdf = opendir(dir.c_str());
+    if (dpdf != NULL){
+       while ((epdf = readdir(dpdf))){
+            std::string name = untwine::fromNative(epdf->d_name);
+            // Skip paths
+            if (pdal::Utils::iequals(name, ".") ||
+                pdal::Utils::iequals(name, ".."))
+            {
+                continue;
+            }
+            else
+            {
+                // we expect the path + name
+                std::string p = dir + "/" + untwine::fromNative(epdf->d_name);
+                files.push_back(p);
+           }
+       }
+    }
+    closedir(dpdf);
+
+    return files;
+
+}
+#endif
+
+} // unnamed namespace
+
 namespace untwine
 {
 namespace bu
@@ -28,7 +98,9 @@ void BuPyramid::run(ProgressWriter& progress)
 {
     getInputFiles();
     size_t count = queueWork();
-
+    if (!count)
+        throw FatalError("No temporary files to process. I/O or directory list error?");
+
     progress.setPercent(.6);
     progress.setIncrement(.4 / count);
     m_manager.setProgress(&progress);
@@ -63,7 +135,7 @@ void BuPyramid::writeInfo()
         }
     };
 
-    std::ofstream out(m_b.opts.outputName + "/ept.json");
+    std::ofstream out(toNative(m_b.opts.outputName + "/ept.json"));
     int maxdigits = std::numeric_limits<double>::max_digits10;
 
     out << "{\n";
@@ -164,7 +236,7 @@ void BuPyramid::getInputFiles()
         return std::make_pair(true, VoxelKey(x, y, z, level));
     };
 
-    std::vector<std::string> files = pdal::FileUtils::directoryList(m_b.opts.tempDir);
+    std::vector<std::string> files = directoryList(m_b.opts.tempDir);
 
     VoxelKey root;
     for (std::string file : files)

bu/CopcSupport.cpp

@@ -34,9 +34,9 @@ namespace bu
 CopcSupport::CopcSupport(const BaseInfo& b) : m_b(b),
     m_lazVlr(b.pointFormatId, extraByteSize(), lazperf::VariableChunkSize),
     m_ebVlr(),
-    m_wktVlr(b.srs.getWKT1())
+    m_wktVlr(b.srs.getWKT())
 {
-    m_f.open(b.opts.outputName, std::ios::out | std::ios::binary);
+    m_f.open(toNative(b.opts.outputName), std::ios::out | std::ios::binary);
 
     //ABELL
     m_header.file_source_id = 0;

bu/Processor.cpp

@@ -19,6 +19,7 @@
 #include <pdal/PDALUtils.hpp>
 #include <pdal/StageFactory.hpp>
 #include <pdal/io/BufferReader.hpp>
+#include <pdal/filters/SortFilter.hpp>
 #include <pdal/util/Algorithm.hpp>
 
 #include <lazperf/lazperf.hpp>
@@ -245,7 +246,7 @@ void Processor::writeBinOutput(Index& index)
     // pass.
     std::string filename = m_vi.key().toString() + ".bin";
     std::string fullFilename = m_b.opts.tempDir + "/" + filename;
-    std::ofstream out(fullFilename, std::ios::binary | std::ios::trunc);
+    std::ofstream out(toNative(fullFilename), std::ios::binary | std::ios::trunc);
     if (!out)
         throw FatalError("Couldn't open '" + fullFilename + "' for output.");
     for (size_t i = 0; i < index.size(); ++i)
@@ -416,8 +417,7 @@ void Processor::appendCompressed(pdal::PointViewPtr view, const DimInfoList& dim
     }
 }
 
-void Processor::flushCompressed(pdal::PointViewPtr view, const OctantInfo& oi,
-    IndexedStats& stats)
+void Processor::flushCompressed(pdal::PointViewPtr view, const OctantInfo& oi, IndexedStats& stats)
 {
     // For single file output we need the stats for
     if (m_b.opts.stats)
@@ -490,6 +490,21 @@ void Processor::writeEptFile(const std::string& filename, pdal::PointViewPtr vie
     w->execute(view->table());
 }
 
+void Processor::sortChunk(pdal::PointViewPtr view)
+{
+    pdal::BufferReader r;
+    r.addView(view);
+
+    pdal::SortFilter s;
+    s.setInput(r);
+    pdal::Options o;
+    o.add("dimension", "GpsTime");
+    s.setOptions(o);
+
+    s.prepare(view->table());
+    s.execute(view->table());
+}
+
 void Processor::createChunk(const VoxelKey& key, pdal::PointViewPtr view)
 {
     using namespace pdal;
@@ -502,9 +517,7 @@ void Processor::createChunk(const VoxelKey& key, pdal::PointViewPtr view)
 
     // Sort the chunk on GPS time.
     if (view->layout()->hasDim(Dimension::Id::GpsTime))
-        std::sort(view->begin(), view->end(),
-            [](const PointRef& p1, const PointRef& p2)
-                { return p1.compare(Dimension::Id::GpsTime, p2); });
+        sortChunk(view);
 
     PointLayoutPtr layout = view->layout();
 
@@ -524,7 +537,8 @@ void Processor::createChunk(const VoxelKey& key, pdal::PointViewPtr view)
 
     uint64_t location = m_manager.newChunk(key, chunk.size(), (uint32_t)view->size());
 
-    std::ofstream out(m_b.opts.outputName, std::ios::out | std::ios::in | std::ios::binary);
+    std::ofstream out(toNative(m_b.opts.outputName),
+        std::ios::out | std::ios::in | std::ios::binary);
     out.seekp(std::ofstream::pos_type(location));
     out.write(reinterpret_cast<const char *>(chunk.data()), chunk.size());
     out.close();

bu/Processor.hpp

@@ -58,6 +58,7 @@ class Processor
     void flushCompressed(pdal::PointViewPtr view, const OctantInfo& oi, IndexedStats& stats);
     void writeEptFile(const std::string& filename, pdal::PointViewPtr view);
     void createChunk(const VoxelKey& key, pdal::PointViewPtr view);
+    void sortChunk(pdal::PointViewPtr view);
     void fillPointBuf(pdal::PointRef& point, std::vector<char>& buf);
 
     VoxelInfo m_vi;

bu/PyramidManager.cpp

@@ -234,7 +234,8 @@ std::deque<VoxelKey> PyramidManager::emitRoot(const VoxelKey& root)
     entries.push_back({root, m_written[root]});
     std::deque<VoxelKey> roots = emit(root, stopLevel, entries);
 
-    std::ofstream out(m_b.opts.outputName + "/ept-hierarchy/" + root.toString() + ".json");
+    std::string filename = m_b.opts.outputName + "/ept-hierarchy/" + root.toString() + ".json";
+    std::ofstream out(toNative(filename));
 
     out << "{\n";
 

ci/linux/setup.sh

@@ -3,7 +3,6 @@
 mkdir build
 
 conda update -n base -c defaults conda
-#conda install -c conda-forge cmake ninja compilers -y
-conda install -c conda-forge cmake ninja -y
+conda install -c conda-forge cmake ninja compilers -y
 conda install -c conda-forge --yes --quiet pdal -y
 

epf/Epf.cpp

@@ -20,6 +20,7 @@
 #include "../untwine/Las.hpp"
 
 #include <unordered_set>
+#include <filesystem>
 
 #include <pdal/pdal_features.hpp>
 #include <pdal/Dimension.hpp>
@@ -30,6 +31,64 @@
 #include <pdal/util/FileUtils.hpp>
 #include <pdal/util/ProgramArgs.hpp>
 
+
+namespace
+{
+
+// PDAL's directoryList had a bug, so we've imported a working
+// version here so that we can still use older PDAL releases.
+
+#ifndef __APPLE_CC__
+std::vector<std::string> directoryList(const std::string& dir)
+{
+    namespace fs = std::filesystem;
+
+    std::vector<std::string> files;
+
+    try
+    {
+        fs::directory_iterator it(untwine::toNative(dir));
+        fs::directory_iterator end;
+        while (it != end)
+        {
+            files.push_back(untwine::fromNative(it->path()));
+            it++;
+        }
+    }
+    catch (fs::filesystem_error&)
+    {
+        files.clear();
+    }
+    return files;
+}
+#else
+
+#include <dirent.h>
+
+// Provide simple opendir/readdir solution for OSX because directory_iterator is
+// not available until OSX 10.15
+std::vector<std::string> directoryList(const std::string& dir)
+{
+
+    DIR *dpdf;
+    struct dirent *epdf;
+
+    std::vector<std::string> files;
+    dpdf = opendir(dir.c_str());
+    if (dpdf != NULL){
+       while ((epdf = readdir(dpdf))){
+           files.push_back(untwine::fromNative(epdf->d_name));
+       }
+    }
+    closedir(dpdf);
+
+    return files;
+
+}
+#endif
+
+} // unnamed namespace
+
 namespace untwine
 {
 namespace epf
@@ -278,7 +337,7 @@ PointCount Epf::createFileInfo(const StringList& input, StringList dimNames,
     {
         if (FileUtils::isDirectory(filename))
         {
-            std::vector<std::string> dirfiles = FileUtils::directoryList(filename);
+            std::vector<std::string> dirfiles = directoryList(filename);
             filenames.insert(filenames.end(), dirfiles.begin(), dirfiles.end());
         }
         else

epf/Writer.cpp

@@ -137,7 +137,7 @@ void Writer::run()
 
         // Open the file. Write the data. Stick the buffer back on the cache.
         // Remove the key from the active key list.
-        std::ofstream out(path(wd.key), std::ios::app | std::ios::binary);
+        std::ofstream out(toNative(path(wd.key)), std::ios::app | std::ios::binary);
         out.write(reinterpret_cast<const char *>(wd.data->data()), wd.dataSize);
         out.close();
         if (!out)