Add progress debug support

README.md

@@ -84,3 +84,12 @@ Options
   Generate a LAZ file with spatially arranged data and hierarchy information
   [(COPC)](https://github.com/copcio/copcio.github.io). [Default: false]
 
+- progress_fd
+
+  File descriptor number of a pipe using the Untwine API to send progress and error messages.
+  [Default: -1]
+
+- progress_debug
+
+  Set to true to have progress messages written to standard output. Disabled if 'progress_fd'
+  is set to '1'. [Default: false]

untwine/Common.hpp

@@ -37,6 +37,7 @@ struct Options
     size_t fileLimit;
     int level;
     int progressFd;
+    bool progressDebug;
     StringList dimNames;
     bool stats;
     std::string a_srs;

untwine/ProgressWriter.cpp

@@ -13,39 +13,31 @@
 namespace untwine
 {
 
-ProgressWriter::ProgressWriter() : m_progressFd(-1), m_percent(0.0), m_increment(.01)
+ProgressWriter::ProgressWriter() : m_fd(-1), m_debug(false), m_percent(0.0), m_increment(.01)
 {}
 
-ProgressWriter::ProgressWriter(int fd) : m_progressFd(fd), m_percent(0.0), m_increment(.01)
-{}
-
-void ProgressWriter::setFd(int fd)
+void ProgressWriter::init(int fd, bool debug)
 {
-    m_progressFd = fd;
+    m_fd = fd;
+    m_debug = debug;
 }
 
 void ProgressWriter::setIncrement(double increment)
 {
-    if (m_progressFd < 0)
-        return;
     std::unique_lock<std::mutex> lock(m_mutex);
 
     m_increment = increment;
 }
 
 void ProgressWriter::setPercent(double percent)
 {
-    if (m_progressFd < 0)
-        return;
     std::unique_lock<std::mutex> lock(m_mutex);
 
     m_percent = (std::max)(0.0, ((std::min)(1.0, percent)));
 }
 
 void ProgressWriter::writeIncrement(const std::string& message)
 {
-    if (m_progressFd < 0)
-        return;
     std::unique_lock<std::mutex> lock(m_mutex);
 
     m_percent += m_increment;
@@ -57,8 +49,6 @@ void ProgressWriter::writeIncrement(const std::string& message)
 
 void ProgressWriter::write(double percent, const std::string& message)
 {
-    if (m_progressFd < 0)
-        return;
     std::unique_lock<std::mutex> lock(m_mutex);
 
     m_percent = (std::min)(0.0, ((std::max)(1.0, percent)));
@@ -69,22 +59,27 @@ void ProgressWriter::write(double percent, const std::string& message)
 
 void ProgressWriter::writeMessage(uint32_t percent, const std::string& message)
 {
+    if (m_debug)
+        std::cout << "Untwine progress (" << percent << "% done): " << message << "\n";
+    if (m_fd < 0)
+        return;
+
     const int32_t msgId = 1000;
 #ifndef _WIN32
     bool err = false;
-    err = (::write(m_progressFd, &msgId, sizeof(msgId)) == -1);
-    err |= (::write(m_progressFd, &percent, sizeof(percent)) == -1);
+    err = (::write(m_fd, &msgId, sizeof(msgId)) == -1);
+    err |= (::write(m_fd, &percent, sizeof(percent)) == -1);
     uint32_t ssize = (uint32_t)message.size();
-    err |= (::write(m_progressFd, &ssize, sizeof(ssize)) == -1);
-    err |= (::write(m_progressFd, message.data(), ssize) == -1);
+    err |= (::write(m_fd, &ssize, sizeof(ssize)) == -1);
+    err |= (::write(m_fd, message.data(), ssize) == -1);
     if (err)
     {
-        ::close(m_progressFd);
-        m_progressFd = -1;
+        ::close(m_fd);
+        m_fd = -1;
     }
 #else
     DWORD numWritten;
-    HANDLE h = reinterpret_cast<HANDLE>((intptr_t)m_progressFd);
+    HANDLE h = reinterpret_cast<HANDLE>((intptr_t)m_fd);
     WriteFile(h, &msgId, sizeof(msgId), &numWritten, NULL);
     WriteFile(h, &percent, sizeof(percent), &numWritten, NULL);
     uint32_t ssize = (uint32_t)message.size();
@@ -95,27 +90,27 @@ void ProgressWriter::writeMessage(uint32_t percent, const std::string& message)
 
 void ProgressWriter::writeErrorMessage(const std::string& message)
 {
-    if (m_progressFd < 0)
+    if (m_fd < 0)
     {
-        std::cerr << message << "\n";
+        std::cerr << "Untwine Error: " << message << "\n";
         return;
     }
 
     const int32_t msgId = 1001;
 #ifndef _WIN32
     bool err = false;
-    err = (::write(m_progressFd, &msgId, sizeof(msgId)) == -1);
+    err = (::write(m_fd, &msgId, sizeof(msgId)) == -1);
     uint32_t ssize = (uint32_t)message.size();
-    err |= (::write(m_progressFd, &ssize, sizeof(ssize)) == -1);
-    err |= (::write(m_progressFd, message.data(), ssize) == -1);
+    err |= (::write(m_fd, &ssize, sizeof(ssize)) == -1);
+    err |= (::write(m_fd, message.data(), ssize) == -1);
     if (err)
     {
-        ::close(m_progressFd);
-        m_progressFd = -1;
+        ::close(m_fd);
+        m_fd = -1;
     }
 #else
     DWORD numWritten;
-    HANDLE h = reinterpret_cast<HANDLE>((intptr_t)m_progressFd);
+    HANDLE h = reinterpret_cast<HANDLE>((intptr_t)m_fd);
     WriteFile(h, &msgId, sizeof(msgId), &numWritten, NULL);
     uint32_t ssize = (uint32_t)message.size();
     WriteFile(h, &ssize, sizeof(ssize), &numWritten, NULL);

untwine/ProgressWriter.hpp

@@ -13,10 +13,9 @@ class ProgressWriter
     static const PointCount ChunkSize = 100'000;
 
     ProgressWriter();
-    ProgressWriter(int fd);
 
-    // Set the progress file descriptor.
-    void setFd(int fd);
+    // Set the progress config..
+    void init(int fd = -1, bool debug = false);
 
     /// Set the increment to use on the next call to setIncrement.
     void setIncrement(double increment);
@@ -36,7 +35,8 @@ class ProgressWriter
 
 private:
     std::mutex m_mutex;
-    int m_progressFd;
+    int m_fd;
+    bool m_debug;
     double m_percent; // Current percent.
     double m_increment; // Current increment.
 

untwine/Untwine.cpp

@@ -31,15 +31,14 @@ void addArgs(pdal::ProgramArgs& programArgs, Options& options, pdal::Arg * &temp
         options.outputName);
     programArgs.add("single_file,s", "Create a single output file", options.singleFile);
     tempArg = &(programArgs.add("temp_dir", "Temp directory", options.tempDir));
-    programArgs.add("clean_temp_dir", "Remove files from the temp directory",
-        options.cleanTempDir, true);
     programArgs.add("cube", "Make a cube, rather than a rectangular solid", options.doCube, true);
     programArgs.add("level", "Set an initial tree level, rather than guess based on data",
         options.level, -1);
     programArgs.add("file_limit", "Only load 'file_limit' files, even if more exist",
         options.fileLimit, (size_t)10000000);
     programArgs.add("progress_fd", "File descriptor on which to write progress messages.",
         options.progressFd, -1);
+    programArgs.add("progress_debug", "Send progress info to stdout.", options.progressDebug);
     programArgs.add("dims", "Dimensions to load. Note that X, Y and Z are always "
         "loaded.", options.dimNames);
     programArgs.add("stats", "Generate statistics for dimensions in the manner of Entwine.",
@@ -86,6 +85,13 @@ bool handleOptions(pdal::StringList& arglist, Options& options)
         }
         if (options.singleFile)
             options.stats = true;
+
+        //
+        if (options.progressFd == 1 && options.progressDebug)
+        {
+            std::cerr << "'--progress_fd' set to 1. Disabling '--progressDebug'.\n";
+            options.progressDebug = false;
+        }
     }
     catch (const pdal::arg_error& err)
     {
@@ -139,7 +145,7 @@ int main(int argc, char *argv[])
     {
         if (!handleOptions(arglist, options))
             return 0;
-        progress.setFd(options.progressFd);
+        progress.init(options.progressFd, options.progressDebug);
         createDirs(options);
 
         epf::Epf preflight(common);