From a441cfd8463f8d5c0dabee5026472b976e48f356 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Fri, 21 Jan 2022 13:27:02 -0500 Subject: [PATCH 1/4] Checkpoint. --- untwine/Common.hpp | 3 ++- untwine/ProgressWriter.cpp | 50 ++++++++++++++++++-------------------- untwine/ProgressWriter.hpp | 5 ++-- untwine/Untwine.cpp | 12 ++++++--- 4 files changed, 38 insertions(+), 32 deletions(-) diff --git a/untwine/Common.hpp b/untwine/Common.hpp index 4f63467..045df24 100644 --- a/untwine/Common.hpp +++ b/untwine/Common.hpp @@ -32,11 +32,12 @@ struct Options bool singleFile; StringList inputFiles; std::string tempDir; - bool cleanTempDir; + bool preserveTempDir; bool doCube; size_t fileLimit; int level; int progressFd; + bool progressDebug; StringList dimNames; bool stats; std::string a_srs; diff --git a/untwine/ProgressWriter.cpp b/untwine/ProgressWriter.cpp index f96fb27..ec601e8 100644 --- a/untwine/ProgressWriter.cpp +++ b/untwine/ProgressWriter.cpp @@ -13,21 +13,20 @@ namespace untwine { -ProgressWriter::ProgressWriter() : m_progressFd(-1), m_percent(0.0), m_increment(.01) +ProgressWriter::ProgressWriter() : m_fd(-1), m_debug(false), m_percent(0.0), m_increment(.01) {} -ProgressWriter::ProgressWriter(int fd) : m_progressFd(fd), m_percent(0.0), m_increment(.01) +ProgressWriter::ProgressWriter(int fd, bool debug) : + m_fd(fd), m_debug(debug), m_percent(0.0), m_increment(.01) {} void ProgressWriter::setFd(int fd) { - m_progressFd = fd; + m_fd = fd; } void ProgressWriter::setIncrement(double increment) { - if (m_progressFd < 0) - return; std::unique_lock lock(m_mutex); m_increment = increment; @@ -35,8 +34,6 @@ void ProgressWriter::setIncrement(double increment) void ProgressWriter::setPercent(double percent) { - if (m_progressFd < 0) - return; std::unique_lock lock(m_mutex); m_percent = (std::max)(0.0, ((std::min)(1.0, percent))); @@ -44,8 +41,6 @@ void ProgressWriter::setPercent(double percent) void ProgressWriter::writeIncrement(const std::string& message) { - if (m_progressFd < 0) - return; std::unique_lock lock(m_mutex); m_percent += m_increment; @@ -57,8 +52,6 @@ void ProgressWriter::writeIncrement(const std::string& message) void ProgressWriter::write(double percent, const std::string& message) { - if (m_progressFd < 0) - return; std::unique_lock lock(m_mutex); m_percent = (std::min)(0.0, ((std::max)(1.0, percent))); @@ -69,22 +62,27 @@ void ProgressWriter::write(double percent, const std::string& message) void ProgressWriter::writeMessage(uint32_t percent, const std::string& message) { + if (m_debug) + std::cout << "Progress (" << percent << ") " << message << "\n"; + if (m_fd < 0) + return; + const int32_t msgId = 1000; #ifndef _WIN32 bool err = false; - err = (::write(m_progressFd, &msgId, sizeof(msgId)) == -1); - err |= (::write(m_progressFd, &percent, sizeof(percent)) == -1); + err = (::write(m_fd, &msgId, sizeof(msgId)) == -1); + err |= (::write(m_fd, &percent, sizeof(percent)) == -1); uint32_t ssize = (uint32_t)message.size(); - err |= (::write(m_progressFd, &ssize, sizeof(ssize)) == -1); - err |= (::write(m_progressFd, message.data(), ssize) == -1); + err |= (::write(m_fd, &ssize, sizeof(ssize)) == -1); + err |= (::write(m_fd, message.data(), ssize) == -1); if (err) { - ::close(m_progressFd); - m_progressFd = -1; + ::close(m_fd); + m_fd = -1; } #else DWORD numWritten; - HANDLE h = reinterpret_cast((intptr_t)m_progressFd); + HANDLE h = reinterpret_cast((intptr_t)m_fd); WriteFile(h, &msgId, sizeof(msgId), &numWritten, NULL); WriteFile(h, &percent, sizeof(percent), &numWritten, NULL); uint32_t ssize = (uint32_t)message.size(); @@ -95,27 +93,27 @@ void ProgressWriter::writeMessage(uint32_t percent, const std::string& message) void ProgressWriter::writeErrorMessage(const std::string& message) { - if (m_progressFd < 0) + if (m_fd < 0) { - std::cerr << message << "\n"; + std::cerr << "Untwine Error: " << message << "\n"; return; } const int32_t msgId = 1001; #ifndef _WIN32 bool err = false; - err = (::write(m_progressFd, &msgId, sizeof(msgId)) == -1); + err = (::write(m_fd, &msgId, sizeof(msgId)) == -1); uint32_t ssize = (uint32_t)message.size(); - err |= (::write(m_progressFd, &ssize, sizeof(ssize)) == -1); - err |= (::write(m_progressFd, message.data(), ssize) == -1); + err |= (::write(m_fd, &ssize, sizeof(ssize)) == -1); + err |= (::write(m_fd, message.data(), ssize) == -1); if (err) { - ::close(m_progressFd); - m_progressFd = -1; + ::close(m_fd); + m_fd = -1; } #else DWORD numWritten; - HANDLE h = reinterpret_cast((intptr_t)m_progressFd); + HANDLE h = reinterpret_cast((intptr_t)m_fd); WriteFile(h, &msgId, sizeof(msgId), &numWritten, NULL); uint32_t ssize = (uint32_t)message.size(); WriteFile(h, &ssize, sizeof(ssize), &numWritten, NULL); diff --git a/untwine/ProgressWriter.hpp b/untwine/ProgressWriter.hpp index 2b6b59b..0d1d571 100644 --- a/untwine/ProgressWriter.hpp +++ b/untwine/ProgressWriter.hpp @@ -13,7 +13,7 @@ class ProgressWriter static const PointCount ChunkSize = 100'000; ProgressWriter(); - ProgressWriter(int fd); + ProgressWriter(int fd, bool debug); // Set the progress file descriptor. void setFd(int fd); @@ -36,7 +36,8 @@ class ProgressWriter private: std::mutex m_mutex; - int m_progressFd; + int m_fd; + bool m_debug; double m_percent; // Current percent. double m_increment; // Current increment. diff --git a/untwine/Untwine.cpp b/untwine/Untwine.cpp index 30c5b91..f3faefc 100644 --- a/untwine/Untwine.cpp +++ b/untwine/Untwine.cpp @@ -31,8 +31,6 @@ void addArgs(pdal::ProgramArgs& programArgs, Options& options, pdal::Arg * &temp options.outputName); programArgs.add("single_file,s", "Create a single output file", options.singleFile); tempArg = &(programArgs.add("temp_dir", "Temp directory", options.tempDir)); - programArgs.add("clean_temp_dir", "Remove files from the temp directory", - options.cleanTempDir, true); programArgs.add("cube", "Make a cube, rather than a rectangular solid", options.doCube, true); programArgs.add("level", "Set an initial tree level, rather than guess based on data", options.level, -1); @@ -40,6 +38,7 @@ void addArgs(pdal::ProgramArgs& programArgs, Options& options, pdal::Arg * &temp options.fileLimit, (size_t)10000000); programArgs.add("progress_fd", "File descriptor on which to write progress messages.", options.progressFd, -1); + programArgs.add("progress_debug", "Send progress info to stdout.", options.progressDebug); programArgs.add("dims", "Dimensions to load. Note that X, Y and Z are always " "loaded.", options.dimNames); programArgs.add("stats", "Generate statistics for dimensions in the manner of Entwine.", @@ -86,6 +85,13 @@ bool handleOptions(pdal::StringList& arglist, Options& options) } if (options.singleFile) options.stats = true; + + // + if (options.progressFd == 1 && options.progressDebug) + { + std::cerr << "'--progress_fd' set to 1. Disabling '--progressDebug'.\n"; + options.progressDebug = false; + } } catch (const pdal::arg_error& err) { @@ -110,7 +116,7 @@ void createDirs(const Options& options) if (pdal::FileUtils::fileExists(options.tempDir) && !pdal::FileUtils::isDirectory(options.tempDir)) throw FatalError("Can't use temp directory - exists as a regular or special file."); - if (options.cleanTempDir) + if (!options.preserveTempDir) pdal::FileUtils::deleteDirectory(options.tempDir); if (!pdal::FileUtils::createDirectory(options.tempDir)) throw FatalError("Couldn't create temp directory: '" + options.tempDir + "'."); From c12e5e976263bd8a6beb399c5bd04adac101af02 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Fri, 21 Jan 2022 13:48:24 -0500 Subject: [PATCH 2/4] In progress. --- untwine/Common.hpp | 2 +- untwine/Untwine.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/untwine/Common.hpp b/untwine/Common.hpp index 045df24..316280e 100644 --- a/untwine/Common.hpp +++ b/untwine/Common.hpp @@ -32,7 +32,7 @@ struct Options bool singleFile; StringList inputFiles; std::string tempDir; - bool preserveTempDir; + bool cleanTempDir; bool doCube; size_t fileLimit; int level; diff --git a/untwine/Untwine.cpp b/untwine/Untwine.cpp index f3faefc..32b6424 100644 --- a/untwine/Untwine.cpp +++ b/untwine/Untwine.cpp @@ -116,7 +116,7 @@ void createDirs(const Options& options) if (pdal::FileUtils::fileExists(options.tempDir) && !pdal::FileUtils::isDirectory(options.tempDir)) throw FatalError("Can't use temp directory - exists as a regular or special file."); - if (!options.preserveTempDir) + if (options.cleanTempDir) pdal::FileUtils::deleteDirectory(options.tempDir); if (!pdal::FileUtils::createDirectory(options.tempDir)) throw FatalError("Couldn't create temp directory: '" + options.tempDir + "'."); From a843558d4ebe0ba51257355267a312e3d26d715d Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Fri, 21 Jan 2022 14:27:55 -0500 Subject: [PATCH 3/4] Fix initialization. --- untwine/ProgressWriter.cpp | 9 +++------ untwine/ProgressWriter.hpp | 5 ++--- untwine/Untwine.cpp | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/untwine/ProgressWriter.cpp b/untwine/ProgressWriter.cpp index ec601e8..450f40c 100644 --- a/untwine/ProgressWriter.cpp +++ b/untwine/ProgressWriter.cpp @@ -16,13 +16,10 @@ namespace untwine ProgressWriter::ProgressWriter() : m_fd(-1), m_debug(false), m_percent(0.0), m_increment(.01) {} -ProgressWriter::ProgressWriter(int fd, bool debug) : - m_fd(fd), m_debug(debug), m_percent(0.0), m_increment(.01) -{} - -void ProgressWriter::setFd(int fd) +void ProgressWriter::init(int fd, bool debug) { m_fd = fd; + m_debug = debug; } void ProgressWriter::setIncrement(double increment) @@ -63,7 +60,7 @@ void ProgressWriter::write(double percent, const std::string& message) void ProgressWriter::writeMessage(uint32_t percent, const std::string& message) { if (m_debug) - std::cout << "Progress (" << percent << ") " << message << "\n"; + std::cout << "Untwine progress (" << percent << "% done): " << message << "\n"; if (m_fd < 0) return; diff --git a/untwine/ProgressWriter.hpp b/untwine/ProgressWriter.hpp index 0d1d571..cf521b5 100644 --- a/untwine/ProgressWriter.hpp +++ b/untwine/ProgressWriter.hpp @@ -13,10 +13,9 @@ class ProgressWriter static const PointCount ChunkSize = 100'000; ProgressWriter(); - ProgressWriter(int fd, bool debug); - // Set the progress file descriptor. - void setFd(int fd); + // Set the progress config.. + void init(int fd = -1, bool debug = false); /// Set the increment to use on the next call to setIncrement. void setIncrement(double increment); diff --git a/untwine/Untwine.cpp b/untwine/Untwine.cpp index 32b6424..2ae7161 100644 --- a/untwine/Untwine.cpp +++ b/untwine/Untwine.cpp @@ -145,7 +145,7 @@ int main(int argc, char *argv[]) { if (!handleOptions(arglist, options)) return 0; - progress.setFd(options.progressFd); + progress.init(options.progressFd, options.progressDebug); createDirs(options); epf::Epf preflight(common); From bfadd6a9974aa590fbe97a21e02a9fc2eb6514e9 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Fri, 21 Jan 2022 14:47:56 -0500 Subject: [PATCH 4/4] Add progress options to readme. --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index b1772bc..6c2d1f7 100644 --- a/README.md +++ b/README.md @@ -84,3 +84,12 @@ Options Generate a LAZ file with spatially arranged data and hierarchy information [(COPC)](https://github.com/copcio/copcio.github.io). [Default: false] +- progress_fd + + File descriptor number of a pipe using the Untwine API to send progress and error messages. + [Default: -1] + +- progress_debug + + Set to true to have progress messages written to standard output. Disabled if 'progress_fd' + is set to '1'. [Default: false]