diff --git a/CMakeLists.txt b/CMakeLists.txt index a5273c9..6f67d54 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,12 @@ file(GLOB SRCS lazperf/detail/*.cpp ) + +if(MSVC) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \ + /SUBSYSTEM:WINDOWS /ENTRY:wmainCRTStartup") +endif() + add_executable(untwine ${SRCS}) untwine_target_compile_settings(untwine) diff --git a/api/CMakeLists.txt b/api/CMakeLists.txt index 84b33ee..9031116 100644 --- a/api/CMakeLists.txt +++ b/api/CMakeLists.txt @@ -4,7 +4,7 @@ include(FeatureSummary) include(${PROJECT_SOURCE_DIR}/../cmake/compiler_options.cmake) add_executable(qgt - QgisTest.cpp + QgisTest2.cpp QgisUntwine.cpp ) diff --git a/api/QgisTest2.cpp b/api/QgisTest2.cpp new file mode 100644 index 0000000..b3593f7 --- /dev/null +++ b/api/QgisTest2.cpp @@ -0,0 +1,43 @@ +#ifndef _WIN32 +#include +#endif + +#include + +#include "QgisUntwine.hpp" + +int main() +{ + untwine::QgisUntwine::StringList files; + untwine::QgisUntwine::Options options; + std::string exe = "C:\\Users\\andre\\untwine\\build\\untwine.exe"; + + untwine::QgisUntwine api(exe); + + std::vector funnycVec = { 0xc4, 0x8d }; + std::string funnyc(funnycVec.begin(), funnycVec.end()); + std::string v8string { "C:\\Users\\andre\\untwine\\api\\" + funnyc + "\\" + funnyc + ".las" }; + files.push_back(v8string); + std::string outDir { "./out_" + funnyc }; + bool ok = api.start(files, outDir, options); + if (! ok) + { + std::cerr << "Couldn't start '" << exe << "!\n"; + exit(-1); + } + + while (true) + { +#ifdef _WIN32 + Sleep(1000); +#else + ::sleep(1); +#endif + int percent = api.progressPercent(); + std::string s = api.progressMessage(); + std::cerr << "Percent/Msg = " << percent << " / " << s << "!\n"; + if (!api.running()) + break; + } + std::cerr << "Error = " << api.errorMessage() << "\n"; +} diff --git a/bu/BuPyramid.cpp b/bu/BuPyramid.cpp index 3740c80..e1efd87 100644 --- a/bu/BuPyramid.cpp +++ b/bu/BuPyramid.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -13,6 +14,75 @@ #include "../untwine/Common.hpp" #include "../untwine/ProgressWriter.hpp" +namespace +{ + +// PDAL's directoryList had a bug, so we've imported a working +// version here so that we can still use older PDAL releases. + +#ifndef __APPLE_CC__ +std::vector directoryList(const std::string& dir) +{ + namespace fs = std::filesystem; + + std::vector files; + + try + { + fs::directory_iterator it(untwine::toNative(dir)); + fs::directory_iterator end; + while (it != end) + { + files.push_back(untwine::fromNative(it->path())); + it++; + } + } + catch (fs::filesystem_error&) + { + files.clear(); + } + return files; +} +#else + +#include + +// Provide simple opendir/readdir solution for OSX because directory_iterator is +// not available until OSX 10.15 +std::vector directoryList(const std::string& dir) +{ + + DIR *dpdf; + struct dirent *epdf; + + std::vector files; + dpdf = opendir(dir.c_str()); + if (dpdf != NULL){ + while ((epdf = readdir(dpdf))){ + std::string name = untwine::fromNative(epdf->d_name); + // Skip paths + if (pdal::Utils::iequals(name, ".") || + pdal::Utils::iequals(name, "..")) + { + continue; + } + else + { + // we expect the path + name + std::string p = dir + "/" + untwine::fromNative(epdf->d_name); + files.push_back(p); + } + } + } + closedir(dpdf); + + return files; + +} +#endif + +} // unnamed namespace + namespace untwine { namespace bu @@ -28,7 +98,9 @@ void BuPyramid::run(ProgressWriter& progress) { getInputFiles(); size_t count = queueWork(); - + if (!count) + throw FatalError("No temporary files to process. I/O or directory list error?"); + progress.setPercent(.6); progress.setIncrement(.4 / count); m_manager.setProgress(&progress); @@ -63,7 +135,7 @@ void BuPyramid::writeInfo() } }; - std::ofstream out(m_b.opts.outputName + "/ept.json"); + std::ofstream out(toNative(m_b.opts.outputName + "/ept.json")); int maxdigits = std::numeric_limits::max_digits10; out << "{\n"; @@ -164,7 +236,7 @@ void BuPyramid::getInputFiles() return std::make_pair(true, VoxelKey(x, y, z, level)); }; - std::vector files = pdal::FileUtils::directoryList(m_b.opts.tempDir); + std::vector files = directoryList(m_b.opts.tempDir); VoxelKey root; for (std::string file : files) diff --git a/bu/CopcSupport.cpp b/bu/CopcSupport.cpp index e6c788d..a630372 100644 --- a/bu/CopcSupport.cpp +++ b/bu/CopcSupport.cpp @@ -34,9 +34,9 @@ namespace bu CopcSupport::CopcSupport(const BaseInfo& b) : m_b(b), m_lazVlr(b.pointFormatId, extraByteSize(), lazperf::VariableChunkSize), m_ebVlr(), - m_wktVlr(b.srs.getWKT1()) + m_wktVlr(b.srs.getWKT()) { - m_f.open(b.opts.outputName, std::ios::out | std::ios::binary); + m_f.open(toNative(b.opts.outputName), std::ios::out | std::ios::binary); //ABELL m_header.file_source_id = 0; diff --git a/bu/Processor.cpp b/bu/Processor.cpp index c780f8a..e7ccd78 100644 --- a/bu/Processor.cpp +++ b/bu/Processor.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -245,7 +246,7 @@ void Processor::writeBinOutput(Index& index) // pass. std::string filename = m_vi.key().toString() + ".bin"; std::string fullFilename = m_b.opts.tempDir + "/" + filename; - std::ofstream out(fullFilename, std::ios::binary | std::ios::trunc); + std::ofstream out(toNative(fullFilename), std::ios::binary | std::ios::trunc); if (!out) throw FatalError("Couldn't open '" + fullFilename + "' for output."); for (size_t i = 0; i < index.size(); ++i) @@ -416,8 +417,7 @@ void Processor::appendCompressed(pdal::PointViewPtr view, const DimInfoList& dim } } -void Processor::flushCompressed(pdal::PointViewPtr view, const OctantInfo& oi, - IndexedStats& stats) +void Processor::flushCompressed(pdal::PointViewPtr view, const OctantInfo& oi, IndexedStats& stats) { // For single file output we need the stats for if (m_b.opts.stats) @@ -490,6 +490,21 @@ void Processor::writeEptFile(const std::string& filename, pdal::PointViewPtr vie w->execute(view->table()); } +void Processor::sortChunk(pdal::PointViewPtr view) +{ + pdal::BufferReader r; + r.addView(view); + + pdal::SortFilter s; + s.setInput(r); + pdal::Options o; + o.add("dimension", "GpsTime"); + s.setOptions(o); + + s.prepare(view->table()); + s.execute(view->table()); +} + void Processor::createChunk(const VoxelKey& key, pdal::PointViewPtr view) { using namespace pdal; @@ -502,9 +517,7 @@ void Processor::createChunk(const VoxelKey& key, pdal::PointViewPtr view) // Sort the chunk on GPS time. if (view->layout()->hasDim(Dimension::Id::GpsTime)) - std::sort(view->begin(), view->end(), - [](const PointRef& p1, const PointRef& p2) - { return p1.compare(Dimension::Id::GpsTime, p2); }); + sortChunk(view); PointLayoutPtr layout = view->layout(); @@ -524,7 +537,8 @@ void Processor::createChunk(const VoxelKey& key, pdal::PointViewPtr view) uint64_t location = m_manager.newChunk(key, chunk.size(), (uint32_t)view->size()); - std::ofstream out(m_b.opts.outputName, std::ios::out | std::ios::in | std::ios::binary); + std::ofstream out(toNative(m_b.opts.outputName), + std::ios::out | std::ios::in | std::ios::binary); out.seekp(std::ofstream::pos_type(location)); out.write(reinterpret_cast(chunk.data()), chunk.size()); out.close(); diff --git a/bu/Processor.hpp b/bu/Processor.hpp index 8714aa3..d5d7dce 100644 --- a/bu/Processor.hpp +++ b/bu/Processor.hpp @@ -58,6 +58,7 @@ class Processor void flushCompressed(pdal::PointViewPtr view, const OctantInfo& oi, IndexedStats& stats); void writeEptFile(const std::string& filename, pdal::PointViewPtr view); void createChunk(const VoxelKey& key, pdal::PointViewPtr view); + void sortChunk(pdal::PointViewPtr view); void fillPointBuf(pdal::PointRef& point, std::vector& buf); VoxelInfo m_vi; diff --git a/bu/PyramidManager.cpp b/bu/PyramidManager.cpp index cf78790..f874340 100644 --- a/bu/PyramidManager.cpp +++ b/bu/PyramidManager.cpp @@ -234,7 +234,8 @@ std::deque PyramidManager::emitRoot(const VoxelKey& root) entries.push_back({root, m_written[root]}); std::deque roots = emit(root, stopLevel, entries); - std::ofstream out(m_b.opts.outputName + "/ept-hierarchy/" + root.toString() + ".json"); + std::string filename = m_b.opts.outputName + "/ept-hierarchy/" + root.toString() + ".json"; + std::ofstream out(toNative(filename)); out << "{\n"; diff --git a/ci/linux/setup.sh b/ci/linux/setup.sh index 4a72325..cff7ef4 100755 --- a/ci/linux/setup.sh +++ b/ci/linux/setup.sh @@ -3,7 +3,6 @@ mkdir build conda update -n base -c defaults conda -#conda install -c conda-forge cmake ninja compilers -y -conda install -c conda-forge cmake ninja -y +conda install -c conda-forge cmake ninja compilers -y conda install -c conda-forge --yes --quiet pdal -y diff --git a/epf/Epf.cpp b/epf/Epf.cpp index 5265f5c..cd45089 100644 --- a/epf/Epf.cpp +++ b/epf/Epf.cpp @@ -20,6 +20,7 @@ #include "../untwine/Las.hpp" #include +#include #include #include @@ -30,6 +31,64 @@ #include #include + +namespace +{ + +// PDAL's directoryList had a bug, so we've imported a working +// version here so that we can still use older PDAL releases. + +#ifndef __APPLE_CC__ +std::vector directoryList(const std::string& dir) +{ + namespace fs = std::filesystem; + + std::vector files; + + try + { + fs::directory_iterator it(untwine::toNative(dir)); + fs::directory_iterator end; + while (it != end) + { + files.push_back(untwine::fromNative(it->path())); + it++; + } + } + catch (fs::filesystem_error&) + { + files.clear(); + } + return files; +} +#else + +#include + +// Provide simple opendir/readdir solution for OSX because directory_iterator is +// not available until OSX 10.15 +std::vector directoryList(const std::string& dir) +{ + + DIR *dpdf; + struct dirent *epdf; + + std::vector files; + dpdf = opendir(dir.c_str()); + if (dpdf != NULL){ + while ((epdf = readdir(dpdf))){ + files.push_back(untwine::fromNative(epdf->d_name)); + } + } + closedir(dpdf); + + return files; + +} +#endif + +} // unnamed namespace + namespace untwine { namespace epf @@ -278,7 +337,7 @@ PointCount Epf::createFileInfo(const StringList& input, StringList dimNames, { if (FileUtils::isDirectory(filename)) { - std::vector dirfiles = FileUtils::directoryList(filename); + std::vector dirfiles = directoryList(filename); filenames.insert(filenames.end(), dirfiles.begin(), dirfiles.end()); } else diff --git a/epf/Writer.cpp b/epf/Writer.cpp index 210ed28..50f7343 100644 --- a/epf/Writer.cpp +++ b/epf/Writer.cpp @@ -137,7 +137,7 @@ void Writer::run() // Open the file. Write the data. Stick the buffer back on the cache. // Remove the key from the active key list. - std::ofstream out(path(wd.key), std::ios::app | std::ios::binary); + std::ofstream out(toNative(path(wd.key)), std::ios::app | std::ios::binary); out.write(reinterpret_cast(wd.data->data()), wd.dataSize); out.close(); if (!out) diff --git a/untwine/Common.hpp b/untwine/Common.hpp index 045df24..56a681a 100644 --- a/untwine/Common.hpp +++ b/untwine/Common.hpp @@ -1,5 +1,9 @@ #pragma once +#ifdef _WIN32 +#include +#endif + #include #include #include @@ -66,4 +70,50 @@ struct BaseInfo const std::string MetadataFilename {"info2.txt"}; +// We check both _WIN32 and _MSC_VER to deal with MinGW, which doesn't support the special +// Windows wide character interfaces for streams. +#if defined(_WIN32) && defined(_MSC_VER) +inline std::wstring toNative(const std::string& in) +{ + if (in.empty()) + return std::wstring(); + + int len = MultiByteToWideChar(CP_UTF8, 0, in.data(), in.length(), nullptr, 0); + std::wstring out(len, 0); + if (MultiByteToWideChar(CP_UTF8, 0, in.data(), in.length(), out.data(), len) == 0) + { + char buf[200] {}; + len = FormatMessageA(0, 0, GetLastError(), 0, buf, 199, 0); + throw FatalError("Can't convert UTF8 to UTF16: " + std::string(buf, len)); + } + return out; +} + +inline std::string fromNative(const std::wstring& in) +{ + if (in.empty()) + return std::string(); + + int len = WideCharToMultiByte(CP_UTF8, 0, in.data(), in.length(), nullptr, 0, nullptr, nullptr); + std::string out(len, 0); + if (WideCharToMultiByte(CP_UTF8, 0, in.data(), in.length(), out.data(), len, nullptr, nullptr) == 0) + { + char buf[200] {}; + len = FormatMessageA(0, 0, GetLastError(), 0, buf, 199, 0); + throw FatalError("Can't convert UTF16 to UTF8: " + std::string(buf, len)); + } + return out; +} +#else +inline std::string toNative(const std::string& in) +{ + return in; +} + +inline std::string fromNative(const std::string& in) +{ + return in; +} +#endif + } // namespace untwine diff --git a/untwine/MapFile.cpp b/untwine/MapFile.cpp index 5f517de..90d7e2e 100644 --- a/untwine/MapFile.cpp +++ b/untwine/MapFile.cpp @@ -28,13 +28,15 @@ * OF SUCH DAMAGE. ****************************************************************************/ +// This is here so that things will work with older version of PDAL. + +#include "Common.hpp" #include "MapFile.hpp" namespace untwine { -MapContext mapFile(const std::string& filename, bool readOnly, - size_t pos, size_t size) +MapContext mapFile(const std::string& filename, bool readOnly, size_t pos, size_t size) { MapContext ctx; @@ -45,9 +47,9 @@ MapContext mapFile(const std::string& filename, bool readOnly, } #ifndef _WIN32 - ctx.m_fd = ::open(filename.c_str(), readOnly ? O_RDONLY : O_RDWR); + ctx.m_fd = ::open(filename.data(), readOnly ? O_RDONLY : O_RDWR); #else - ctx.m_fd = ::_open(filename.c_str(), readOnly ? _O_RDONLY : _O_RDWR); + ctx.m_fd = ::_wopen(toNative(filename).data(), readOnly ? _O_RDONLY : _O_RDWR); #endif if (ctx.m_fd == -1) diff --git a/untwine/MapFile.hpp b/untwine/MapFile.hpp index 5a73cfa..854907d 100644 --- a/untwine/MapFile.hpp +++ b/untwine/MapFile.hpp @@ -28,6 +28,9 @@ * OF SUCH DAMAGE. ****************************************************************************/ +//ABELL - This exists here because older version of PDAL don't have it and the QGIS +// crew wanted things to work with older versions of PDAL. + #pragma once #include diff --git a/untwine/ThreadPool.cpp b/untwine/ThreadPool.cpp index dd7fcc1..d607d43 100644 --- a/untwine/ThreadPool.cpp +++ b/untwine/ThreadPool.cpp @@ -32,8 +32,6 @@ * OF SUCH DAMAGE. ****************************************************************************/ -#include - #include "ThreadPool.hpp" namespace untwine diff --git a/untwine/Untwine.cpp b/untwine/Untwine.cpp index 940207b..cd6c0cf 100644 --- a/untwine/Untwine.cpp +++ b/untwine/Untwine.cpp @@ -127,7 +127,11 @@ void createDirs(const Options& options) } // namespace untwine +#ifdef _WIN32 +int wmain( int argc, wchar_t *argv[ ], wchar_t *envp[ ] ) +#else int main(int argc, char *argv[]) +#endif { std::vector arglist; @@ -135,7 +139,7 @@ int main(int argc, char *argv[]) argv++; argc--; while (argc--) - arglist.push_back(*argv++); + arglist.push_back(untwine::fromNative(*argv++)); using namespace untwine; diff --git a/untwine/VoxelKey.hpp b/untwine/VoxelKey.hpp index 8eeef2e..a29a802 100644 --- a/untwine/VoxelKey.hpp +++ b/untwine/VoxelKey.hpp @@ -11,6 +11,7 @@ ****************************************************************************/ #include +#include #pragma once