Pass errors back through API.

api/QgisTest.cpp

@@ -10,18 +10,22 @@ int main()
 {
     untwine::QgisUntwine::StringList files;
     untwine::QgisUntwine::Options options;
-//    std::string exe = "C:\\Users\\andre\\untwine\\build\\untwine.exe";
-    std::string exe = "/Users/acbell/untwine/build/untwine";
+    std::string exe = "C:\\Users\\andre\\untwine\\build\\untwine.exe";
+//    std::string exe = "/Users/acbell/untwine/build/untwine";
 
     untwine::QgisUntwine api(exe);
 
 //    files.push_back("C:\\Users\\andre\\nyc2");
-//    files.push_back("C:\\Users\\andre\\nyc2\\18TXL075075.las.laz");
+    std::vector<unsigned char> funnycVec = { 0xc4, 0x8d };
+    std::string funnyc(funnycVec.begin(), funnycVec.end());
+    std::string v8string { "C:\\Users\\andre\\untwine\\api\\build\\" + funnyc + "\\" + funnyc + ".las" };
+    files.push_back(v8string);
+//   files.push_back("C:\\Users\\andre\\nyc2\\18TXL075075.las.laz");
 //    files.push_back("/Users/acbell/nyc/18TXL075075.las.laz");
 //    files.push_back("/Users/acbell/nyc/18TXL075090.las.laz");
-    files.push_back("/Users/acbell/nyc2");
+//    files.push_back("/Users/acbell/nyc2");
 
-    options.push_back({"dims", "X, Y, Z, Red, Green, Blue, Intensity"});
+//    options.push_back({"dims", "X, Y, Z, Red, Green, Blue, Intensity"});
 //    book ok = api.start(files, ".\\out", options);
     bool ok = api.start(files, "./out", options);
     if (! ok)
@@ -51,4 +55,5 @@ int main()
         if (!api.running())
             break;
     }
+    std::cerr << "Error = " << api.errorMessage() << "\n";
 }

api/QgisUntwine.cpp

@@ -52,4 +52,11 @@ std::string QgisUntwine::progressMessage() const
     return m_progressMsg;
 }
 
+std::string QgisUntwine::errorMessage() const
+{
+    readPipe();
+
+    return m_errorMsg;
+}
+
 } // namespace untwine

api/QgisUntwine.hpp

@@ -26,12 +26,16 @@ class QgisUntwine
     bool running();
     int progressPercent() const;
     std::string progressMessage() const;
+    std::string errorMessage() const;
 
 private:
+    const int32_t ProgressMsg = 1000;
+    const int32_t ErrorMsg = 1001;
     std::string m_path;
     mutable bool m_running;
-    mutable int m_percent;
+    mutable uint32_t m_percent;
     mutable std::string m_progressMsg;
+    mutable std::string m_errorMsg;
 #ifndef _WIN32
     pid_t m_pid;
     int m_progressFd;

api/QgisUntwine_unix.cpp

@@ -125,16 +125,34 @@ uint32_t readString(int fd, std::string& s)
 
 void QgisUntwine::readPipe() const
 {
+    int32_t msgId;
+
     // Read messages until the pipe has been drained.
     while (true)
     {
-        ssize_t size = read(m_progressFd, &m_percent, sizeof(m_percent));
+        ssize_t size = read(m_progressFd, &msgId, sizeof(msgId));
         // If we didn't read the full size, just return.
         if (size != sizeof(m_percent))
             return;
 
-        // Read the string, waiting as necessary.
-        if (readString(m_progressFd, m_progressMsg) != 0)
+        if (msgId == ProgressMsg)
+        {
+            ssize_t size = read(m_progressFd, &m_percent, sizeof(m_percent));
+            // If we didn't read the full size, just return.
+            if (size != sizeof(m_percent))
+                break;
+
+            // Read the string, waiting as necessary.
+            if (readString(m_progressFd, m_progressMsg) != 0)
+                break;
+        }
+        else if (msgId == ErrorMsg)
+        {
+            // Read the error string, waiting as necessary.
+            if (readString(m_progressFd, m_errorMsg) != 0)
+                break;
+        }
+        else
             break;
     }
 }

api/QgisUntwine_win.cpp

@@ -136,13 +136,29 @@ void QgisUntwine::readPipe() const
     while (true)
     {
         DWORD numRead;
-        ReadFile(m_progressFd, &m_percent, sizeof(m_percent), &numRead, NULL);
-        if (numRead != sizeof(m_percent))
+	uint32_t msgId;
+
+        ReadFile(m_progressFd, &msgId, sizeof(msgId), &numRead, NULL);
+        if (numRead != sizeof(msgId))
             return;
 
-        // Read the string, waiting as necessary.
-        if (readString(m_progressFd, m_progressMsg) != 0)
-            break;
+        if (msgId == ProgressMsg)
+        {
+            // Read the percent value.
+            ReadFile(m_progressFd, &m_percent, sizeof(m_percent), &numRead, NULL);
+            if (numRead != sizeof(m_percent))
+                return;
+
+            // Read the string, waiting as necessary.
+            if (readString(m_progressFd, m_progressMsg) != 0)
+                break;
+        }
+        else if (msgId == ErrorMsg)
+        {
+            // Read the string, waiting as necessary.
+            if (readString(m_progressFd, m_errorMsg) != 0)
+                break;
+        }
     }
 }
 

bu/BuPyramid.cpp

@@ -32,8 +32,13 @@ void BuPyramid::run(ProgressWriter& progress)
     progress.setPercent(.6);
     progress.setIncrement(.4 / count);
     m_manager.setProgress(&progress);
+    //ABELL - Not sure why this was being run in a separate thread. The current thread
+    // would block in join() anyway.
+    /**
     std::thread runner(&PyramidManager::run, &m_manager);
     runner.join();
+    **/
+    m_manager.run();
     if (!m_b.opts.singleFile)
         writeInfo();
 }

bu/OctantInfo.cpp

@@ -28,7 +28,7 @@ void OctantInfo::mergeSmallFiles(const std::string tempDir, size_t pointSize)
 
     std::ofstream out(filename, std::ios::binary | std::ios::trunc);
     if (!out)
-        fatal("Couldn't open temporary merge file '" + filename + "'.");
+        throw FatalError("Couldn't open temporary merge file '" + filename + "'.");
 
     int totalPoints = 0;
     auto it = m_fileInfos.begin();
@@ -43,7 +43,7 @@ void OctantInfo::mergeSmallFiles(const std::string tempDir, size_t pointSize)
             filename = tempDir + "/" + fi.filename();
             std::ifstream in(filename, std::ios::binary);
             if (!in)
-                fatal("Couldn't open file '" + filename + "' to merge.");
+                throw FatalError("Couldn't open file '" + filename + "' to merge.");
             in.read(buf.data(), bytes);
             out.write(buf.data(), bytes);
             totalPoints += numPoints;

bu/PointAccessor.hpp

@@ -40,7 +40,7 @@ class PointAccessor
         std::string filename = m_b.opts.tempDir + "/" + fi.filename();
         auto ctx = mapFile(filename, true, 0, fi.numPoints() * m_b.pointSize);
         if (ctx.m_addr == nullptr)
-            fatal(filename + ": " + ctx.m_error);
+            throw FatalError(filename + ": " + ctx.m_error);
         fi.setContext(ctx);
         fi.setStart(size());
         m_fileInfos.push_back(&fi);

bu/Processor.cpp

@@ -42,6 +42,27 @@ Processor::Processor(PyramidManager& manager, const VoxelInfo& v, const BaseInfo
 
 
 void Processor::run()
+{
+    // Don't let any exception sneak out of here.
+    try
+    {
+        runLocal();
+    }
+    catch (const std::exception& ex)
+    {
+        m_manager.queueWithError(m_vi.octant(), ex.what());
+        return;
+    }
+    catch (...)
+    {
+        std::string msg = std::string("Unexpected error processing ") + m_vi.key().toString() + ".";
+        m_manager.queueWithError(m_vi.octant(), msg);
+        return;
+    }
+    m_manager.queue(m_vi.octant());
+}
+
+void Processor::runLocal()
 {
     // If we don't merge small files into one, we'll end up trying to deal with too many
     // open files later and run out of file descriptors.
@@ -78,8 +99,6 @@ void Processor::run()
         sample(accepted, rejected);
 
     write(accepted, rejected);
-
-    m_manager.queue(m_vi.octant());
 }
 
 
@@ -226,7 +245,7 @@ void Processor::writeBinOutput(Index& index)
     std::string fullFilename = m_b.opts.tempDir + "/" + filename;
     std::ofstream out(fullFilename, std::ios::binary | std::ios::trunc);
     if (!out)
-        fatal("Couldn't open '" + fullFilename + "' for output.");
+        throw FatalError("Couldn't open '" + fullFilename + "' for output.");
     for (size_t i = 0; i < index.size(); ++i)
         out.write(m_points[index[i]].cdata(), m_b.pointSize);
     m_vi.octant().appendFileInfo(FileInfo(filename, index.size()));
@@ -371,9 +390,8 @@ Processor::writeOctantCompressed(const OctantInfo& o, Index& index, IndexIter po
     }
     catch (pdal_error& err)
     {
-        fatal(err.what());
+        throw FatalError(err.what());
     }
-
     m_manager.logOctant(o.key(), count, stats);
     return pos;
 }
@@ -450,21 +468,22 @@ void Processor::writeEptFile(const std::string& filename, pdal::PointTableRef ta
     pdal::Options wopts;
     wopts.add("extra_dims", "all");
     wopts.add("software_id", "Entwine 1.0");
-    wopts.add("compression", "lazperf");
+    wopts.add("compression", "laszip");
     wopts.add("filename", filename);
     wopts.add("offset_x", m_b.offset[0]);
     wopts.add("offset_y", m_b.offset[1]);
     wopts.add("offset_z", m_b.offset[2]);
     wopts.add("scale_x", m_b.scale[0]);
     wopts.add("scale_y", m_b.scale[1]);
     wopts.add("scale_z", m_b.scale[2]);
+    wopts.add("minor_version", 4);
+    wopts.add("dataformat_id", m_b.pointFormatId);
     if (m_b.opts.a_srs.size())
         wopts.add("a_srs", m_b.opts.a_srs);
     if (m_b.opts.metadata)
         wopts.add("pdal_metadata", m_b.opts.metadata);
     w->setOptions(wopts);
     w->setInput(*prev);
-    // Set dataformat ID based on time/rgb, but for now accept the default.
 
     w->prepare(table);
     w->execute(table);
@@ -519,7 +538,7 @@ void Processor::createChunk(const VoxelKey& key, pdal::PointViewPtr view)
     out.write(reinterpret_cast<const char *>(chunk.data()), chunk.size());
     out.close();
     if (!out)
-        fatal("Failure writing to '" + m_b.opts.outputName + "'.");
+        throw FatalError("Failure writing to '" + m_b.opts.outputName + "'.");
 }
 
 void Processor::fillPointBuf(pdal::PointRef& point, std::vector<char>& buf)
@@ -549,7 +568,7 @@ void Processor::fillPointBuf(pdal::PointRef& point, std::vector<char>& buf)
         int32_t i(0);
 
         if (!Utils::numericCast(d, i))
-            fatal("Unable to convert scaled value (" +
+            throw FatalError("Unable to convert scaled value (" +
                 Utils::toString(d) + ") to "
                 "int32 for dimension '" + Dimension::name(dim) +
                 "' when writing LAS/LAZ file.");

bu/Processor.hpp

@@ -42,6 +42,7 @@ class Processor
     using Index = std::deque<int>;
     using IndexIter = Index::const_iterator;
 
+    void runLocal();
     void sample(Index& accepted, Index& rejected);
     void write(Index& accepted, Index& rejected);
     bool acceptable(int pointId, GridKey key);

bu/PyramidManager.cpp

@@ -32,7 +32,6 @@ PyramidManager::PyramidManager(const BaseInfo& b) : m_b(b), m_pool(10), m_totalP
     m_copc(m_b)
 {}
 
-
 PyramidManager::~PyramidManager()
 {}
 
@@ -54,6 +53,18 @@ void PyramidManager::queue(const OctantInfo& o)
 }
 
 
+void PyramidManager::queueWithError(const OctantInfo& o, const std::string& error)
+{
+    {
+        std::lock_guard<std::mutex> lock(m_mutex);
+
+        m_queue.push(o);
+        m_error = error;
+    }
+    m_cv.notify_one();
+}
+
+
 void PyramidManager::run()
 {
     while (true)
@@ -65,6 +76,9 @@ void PyramidManager::run()
             m_cv.wait(lock, [this](){return m_queue.size();});
             o = m_queue.front();
             m_queue.pop();
+
+            if (m_error.size())
+                throw FatalError(m_error);
         }
 
         if (o.key() == VoxelKey(0, 0, 0, 0))

bu/PyramidManager.hpp

@@ -42,6 +42,7 @@ class PyramidManager
 
     void setProgress(ProgressWriter *progress);
     void queue(const OctantInfo& o);
+    void queueWithError(const OctantInfo& o, const std::string& error);
     void run();
     void logOctant(const VoxelKey& k, int cnt, const IndexedStats& istats);
     uint64_t totalPoints() const
@@ -62,6 +63,7 @@ class PyramidManager
     StatsMap m_stats;
     ProgressWriter *m_progress;
     CopcSupport m_copc;
+    std::string m_error;
     //
     std::unordered_map<VoxelKey, int> m_written;
     std::unordered_map<VoxelKey, int> m_childCounts;