Add flag to avoid sorting captured files

.github/workflows/build_and_test_mac.yml

@@ -12,17 +12,18 @@ env:
 jobs:
   Build_and_Run_GTest:
     name: Build and Run GoogleTest
-    runs-on: macos-12
+    runs-on: macos-13
     defaults:
       run:
         shell: bash -l {0}
 
     steps:
     - uses: actions/checkout@v3
 
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: anaconda-client-env
+        miniconda-version: "latest"
         python-version: 3.8
         channels: conda-forge
         auto-activate-base: false
@@ -45,18 +46,19 @@ jobs:
 
   Build_and_Run_PyTest:
     name: Build and Run PyTest
-    runs-on: macos-12
+    runs-on: macos-13
     defaults:
       run:
         shell: bash -l {0}
 
     steps:
     - uses: actions/checkout@v3
 
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: anaconda-client-env
         python-version: 3.8
+        miniconda-version: "latest"
         channels: conda-forge
         auto-activate-base: false
 

.github/workflows/build_wheels.yml

@@ -12,7 +12,7 @@ jobs:
       MACOSX_DEPLOYMENT_TARGET: "10.15"
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-12, windows-latest]
+        os: [ubuntu-20.04, macos-13, windows-latest]
         cibw_archs: ["auto64"]
         cibw_build: ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"]
 

.github/workflows/publish_pypi.yml

@@ -14,7 +14,7 @@ jobs:
       MACOSX_DEPLOYMENT_TARGET: "10.15"
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-12, windows-latest]
+        os: [ubuntu-20.04, macos-13, windows-latest]
         cibw_archs: ["auto64"]
         cibw_build: ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"]
 

src/filepattern/cpp/bindings.cpp

@@ -22,10 +22,12 @@ PYBIND11_MODULE(backend, m){
                       const std::string&, 
                       const std::string&,  
                       bool, 
+                      bool,
                       bool>())
         .def(py::init<const std::vector<std::string>&,
                       const std::string&, 
                       bool, 
+                      bool,
                       bool>())
         .def("getMatching", &FilePattern::getMatching)
         .def("getOccurrences", &FilePattern::getOccurrences)

src/filepattern/cpp/external/external_filepattern.cpp

@@ -2,7 +2,7 @@
 
 using namespace std;
 
-ExternalFilePattern::ExternalFilePattern(const string& path, const string& filePattern, const string& block_size, bool recursive, bool suppressWarnings):
+ExternalFilePattern::ExternalFilePattern(const string& path, const string& filePattern, const string& block_size, bool recursive, bool suppressWarnings, bool sorted):
 ExternalPattern(path, block_size, recursive) {
 
     this->setSuppressWarnings(suppressWarnings);
@@ -23,13 +23,17 @@ ExternalPattern(path, block_size, recursive) {
     this->setFirstCall(true); // first call to next() has not occurred
 
     this->matchFiles(); // match files to pattern
-
-    ExternalMergeSort sort = ExternalMergeSort(std_map,
-                                               this->getValidFilesPath(),
-                                               this->getValidFilesPath(),
-                                               this->stream_.getBlockSizeStr(),
-                                               "",
-                                               this->stream_.map_size_);
+
+    this->setIsSorted(sorted);
+
+    if (isSorted()) {
+        ExternalMergeSort sort = ExternalMergeSort(std_map,
+                                                this->getValidFilesPath(),
+                                                this->getValidFilesPath(),
+                                                this->stream_.getBlockSizeStr(),
+                                                "",
+                                                this->stream_.map_size_);
+    }
 
     this->group_stream_.open(this->stream_.getValidFilesPath());
     this->infile_.open(this->getValidFilesPath()); // open temp file for the valid files

src/filepattern/cpp/external/external_filepattern.hpp

@@ -33,7 +33,7 @@ class ExternalFilePattern : public ExternalPattern {
          * @param recursive Iterate over all subdirectories if true
          */
 
-        ExternalFilePattern(const std::string& path, const std::string& file_pattern, const std::string& block_size="50 MB", bool recursive=false, bool suppress_warnings=false);
+        ExternalFilePattern(const std::string& path, const std::string& file_pattern, const std::string& block_size="50 MB", bool recursive=false, bool suppress_warnings=false, bool sorted=true);
 
         ExternalFilePattern(){}
 

src/filepattern/cpp/external/external_pattern.cpp

@@ -185,9 +185,12 @@ void ExternalPattern::groupByHelper(){
             grouped_variables.clear();
             for(auto& g: vec.first) grouped_variables.push_back(g);
             // Sort the matched files by the group_by parameter
-            sort(vec.second.begin(), vec.second.end(), [&group_by = as_const(group_by)](Tuple& p1, Tuple& p2){
-                return get<0>(p1)[group_by] < get<0>(p2)[group_by];
-            });
+
+            if (isSorted()) {
+                sort(vec.second.begin(), vec.second.end(), [&group_by = as_const(group_by)](Tuple& p1, Tuple& p2){
+                    return get<0>(p1)[group_by] < get<0>(p2)[group_by];
+                });
+            }
 
             Types current_value = get<0>(vec.second[0])[group_by]; // get the value of variable
             vector<Tuple> empty_vec;
@@ -207,9 +210,12 @@ void ExternalPattern::groupByHelper(){
 
                 grouped_variables.push_back(make_pair(group_by, current_value));
                 temp_group.push_back(make_pair(grouped_variables, temp_vec));
-                sort(temp_group[group_ptr].second.begin(), temp_group[group_ptr].second.end(), [](Tuple& m1, Tuple& m2){
-                    return get<1>(m1)[0] < get<1>(m2)[0];
-                });
+
+                if (isSorted()) {
+                    sort(temp_group[group_ptr].second.begin(), temp_group[group_ptr].second.end(), [](Tuple& m1, Tuple& m2){
+                        return get<1>(m1)[0] < get<1>(m2)[0];
+                    });
+                }
                 temp_vec.clear();
 
                 if (i < vec.second.size()){
@@ -290,10 +296,12 @@ void ExternalPattern::nextGroup(){
             } else {
 
                 // update variable value and end loop on variable value change
-                        // sort block by basename
-                sort(this->current_group_[0].second.begin(), this->current_group_[0].second.end(), [](Tuple& m1, Tuple& m2){
-                    return get<1>(m1)[0] < get<1>(m2)[0];
-                });
+                // sort block by basename
+                if (isSorted()) {
+                    sort(this->current_group_[0].second.begin(), this->current_group_[0].second.end(), [](Tuple& m1, Tuple& m2){
+                        return get<1>(m1)[0] < get<1>(m2)[0];
+                    });
+                }
                 this->current_value_ = get<0>(this->temp_)[this->group_[0]];
                 value_added = false;
 
@@ -302,9 +310,12 @@ void ExternalPattern::nextGroup(){
             };
         }
     }
-    sort(this->current_group_[0].second.begin(), this->current_group_[0].second.end(), [](Tuple& m1, Tuple& m2){
-        return get<1>(m1)[0] < get<1>(m2)[0];
-    });
+
+    if (isSorted()) {
+        sort(this->current_group_[0].second.begin(), this->current_group_[0].second.end(), [](Tuple& m1, Tuple& m2){
+            return get<1>(m1)[0] < get<1>(m2)[0];
+        });
+    }
     this->groupByHelper();
 }
 
@@ -334,12 +345,15 @@ void ExternalPattern::groupBy(vector<string>& group_by) {
     // sort valid files externally
     string path = this->stream_.getValidFilesPath();
     this->tmp_directories_.push_back(path);
-    ExternalMergeSort sort = ExternalMergeSort(std_map,
-                                               path,
-                                               path,
-                                               this->stream_.getBlockSizeStr(),
-                                               group_by[0],
-                                               this->stream_.map_size_);
+
+    if (isSorted()) {
+        ExternalMergeSort sort = ExternalMergeSort(std_map,
+                                                path,
+                                                path,
+                                                this->stream_.getBlockSizeStr(),
+                                                group_by[0],
+                                                this->stream_.map_size_);
+    }
 }
 
 string ExternalPattern::externalOutPutName(){

src/filepattern/cpp/external/external_stringpattern.cpp

@@ -2,7 +2,7 @@
 
 using namespace std;
 
-ExternalStringPattern::ExternalStringPattern(const string& path, const string& file_pattern, const string& block_size, bool suppress_warnings):
+ExternalStringPattern::ExternalStringPattern(const string& path, const string& file_pattern, const string& block_size, bool suppress_warnings, bool sorted):
 ExternalPattern(path, block_size, false) {
     this->setSuppressWarnings(suppress_warnings);
     this->setPath(path); // store path to target directory
@@ -23,6 +23,8 @@ ExternalPattern(path, block_size, false) {
 
     this->setFirstCall(true); // first call to next() has not occurred
 
+    this->setIsSorted(sorted);
+
     this->matchFiles(); // match files to pattern
     this->group_stream_.open(this->stream_.getValidFilesPath());
     this->infile_.open(this->getValidFilesPath()); // open temp file for the valid files

src/filepattern/cpp/external/external_stringpattern.hpp

@@ -34,7 +34,7 @@ class ExternalStringPattern : public ExternalPattern {
          * @param recursive Iterate over all subdirectories if true
          */
 
-        ExternalStringPattern(const std::string& path, const std::string& file_pattern, const std::string& block_size="50 MB", bool suppress_warnings=false);
+        ExternalStringPattern(const std::string& path, const std::string& file_pattern, const std::string& block_size="50 MB", bool suppress_warnings=false, bool sorted=true);
 
         ~ExternalStringPattern();
 

src/filepattern/cpp/external/external_vectorpattern.cpp

@@ -6,7 +6,7 @@ const std::regex ExternalVectorPattern::STITCH_REGEX_ = std::regex("(corr): (.*)
 const std::vector<std::regex> ExternalVectorPattern::STITCH_REGEX_VECTOR_ = {std::regex("(corr):\\s*(.*?);"), std::regex("(position):\\s*\\((.*?),\\s*(.*?)\\);"), std::regex("(grid):\\s*\\((.*),\\s*(.*)\\);")};
 const std::vector<std::string> ExternalVectorPattern::STITCH_VARIABLES_ = {"correlation","posX","posY","gridX","gridY"}; // stitching vector variables
 
-ExternalVectorPattern::ExternalVectorPattern(const string& path, const string& file_pattern, const string& block_size, bool suppress_warnings):
+ExternalVectorPattern::ExternalVectorPattern(const string& path, const string& file_pattern, const string& block_size, bool suppress_warnings, bool sorted):
 ExternalPattern(path, block_size, false){
     this->setSuppressWarnings(suppress_warnings);
     this->path_ = path; // store path to target directory
@@ -26,6 +26,7 @@ ExternalPattern(path, block_size, false){
     this->setFirstCall(true); // first call to next() has not occurred
 
     this->matchFiles();
+    this->setIsSorted(sorted);
 
     this->group_stream_.open(this->stream_.getValidFilesPath());
     this->infile_.open(this->getValidFilesPath()); // open temp file for the valid files

src/filepattern/cpp/external/external_vectorpattern.hpp

@@ -25,7 +25,7 @@ class ExternalVectorPattern: public ExternalPattern {
          * @param pattern Pattern to match filenames to
          * @param block_size Maximum amount of memory to use
          */
-        ExternalVectorPattern(const std::string& path, const std::string& pattern, const std::string& block_size, bool suppress_warnings=false);
+        ExternalVectorPattern(const std::string& path, const std::string& pattern, const std::string& block_size, bool suppress_warnings=false, bool sorted=true);
 
         /**
          * @brief Deconstructor of ExternalVectorPattern. Removes temporary directories and files.

src/filepattern/cpp/include/filepattern.h

@@ -38,9 +38,9 @@ class FILEPATTERN_EXPORT FilePattern {
 
     public:
 
-        FilePattern(const std::string& path, const std::string& filePattern="", const std::string& block_size="", bool recursive=false, bool suppressWarnings=false);
+        FilePattern(const std::string& path, const std::string& filePattern="", const std::string& block_size="", bool recursive=false, bool suppressWarnings=false, bool sorted=true);
 
-        FilePattern(const std::vector<std::string>& file_array, const std::string& filePattern, bool recursive=false, bool suppressWarnings=false);
+        FilePattern(const std::vector<std::string>& file_array, const std::string& filePattern, bool recursive=false, bool suppressWarnings=false, bool sorted=true);
 
         ~FilePattern();
 

src/filepattern/cpp/interface/filepattern.cpp

@@ -4,11 +4,11 @@
 
 #include <tuple>
 
-FilePattern::FilePattern(const std::string& path, const std::string& filePattern, const std::string& block_size, bool recursive, bool suppressWarnings) {
+FilePattern::FilePattern(const std::string& path, const std::string& filePattern, const std::string& block_size, bool recursive, bool suppressWarnings, bool sorted) {
 
     FilePatternFactory fpf = FilePatternFactory();
 
-    this->fp_ = std::unique_ptr<PatternObject>(fpf.getObject(path, filePattern, block_size, recursive, suppressWarnings));
+    this->fp_ = std::unique_ptr<PatternObject>(fpf.getObject(path, filePattern, block_size, recursive, suppressWarnings, sorted));
 
     if (block_size != "") {
         this->fp_->external = true;
@@ -18,11 +18,11 @@ FilePattern::FilePattern(const std::string& path, const std::string& filePattern
 
 }
 
-FilePattern::FilePattern(const std::vector<std::string>& file_array, const std::string& filePattern, bool recursive, bool suppressWarnings) {
+FilePattern::FilePattern(const std::vector<std::string>& file_array, const std::string& filePattern, bool recursive, bool suppressWarnings, bool sorted) {
 
     FilePatternFactory fpf = FilePatternFactory();
 
-    this->fp_ = std::unique_ptr<PatternObject>(fpf.getObject(file_array, filePattern, suppressWarnings));
+    this->fp_ = std::unique_ptr<PatternObject>(fpf.getObject(file_array, filePattern, suppressWarnings, sorted));
 
     this->fp_->external = false;
 
@@ -136,10 +136,10 @@ std::string FilePattern::inferPattern(const std::string& path, std::string& vari
     std::unique_ptr<PatternObject> fp;
 
     if (block_size == "") {
-        fp = std::unique_ptr<PatternObject>(fpf.getObject(path, "", block_size, false, true));
+        fp = std::unique_ptr<PatternObject>(fpf.getObject(path, "", block_size, false, true, true));
     } else {
 
-        fp = std::unique_ptr<PatternObject>(fpf.getObject(path, "", block_size, false, true));
+        fp = std::unique_ptr<PatternObject>(fpf.getObject(path, "", block_size, false, true, true));
     }
 
 
@@ -150,7 +150,7 @@ std::string FilePattern::inferPattern(std::vector<std::string>& vec, std::string
 
     FilePatternFactory fpf = FilePatternFactory();
 
-    std::unique_ptr<PatternObject> fp = std::unique_ptr<PatternObject>(fpf.getObject(".", "dummy_pattern", "", false, true));
+    std::unique_ptr<PatternObject> fp = std::unique_ptr<PatternObject>(fpf.getObject(".", "dummy_pattern", "", false, true, true));
 
     return fp->inferPattern(vec, variables);
 }

src/filepattern/cpp/interface/filepattern_factory.h

@@ -19,9 +19,10 @@ class FilePatternFactory {
         std::unique_ptr<PatternObject> getObject(
             const std::vector<std::string>& file_array, 
             const std::string& file_pattern, 
-            bool suppressWarnings) {
+            bool suppressWarnings,
+            bool sorted) {
 
-            return std::make_unique<ArrayPattern>(file_array, file_pattern, suppressWarnings);
+            return std::make_unique<ArrayPattern>(file_array, file_pattern, suppressWarnings, sorted);
 
         }
 
@@ -30,7 +31,8 @@ class FilePatternFactory {
             const std::string& file_pattern, 
             const std::string& block_size, 
             bool recursive, 
-            bool suppressWarnings) {
+            bool suppressWarnings,
+            bool sorted) {
 
             if (block_size == "") {
                 if(fs::is_regular_file(path)) {
@@ -40,13 +42,13 @@ class FilePatternFactory {
                     std::getline(infile, str);
 
                     if(VectorParser::isStitchingVector(str)) {
-                        return std::make_unique<VectorPattern>(path, file_pattern, suppressWarnings); // need to add builder to FPOjbect
+                        return std::make_unique<VectorPattern>(path, file_pattern, suppressWarnings, sorted); // need to add builder to FPOjbect
                     }
 
-                    return std::make_unique<StringPattern>(path, file_pattern, suppressWarnings); // need to add builder to FPOjbect
+                    return std::make_unique<StringPattern>(path, file_pattern, suppressWarnings, sorted); // need to add builder to FPOjbect
                 }
 
-                return std::make_unique<FilePatternObject>(path, file_pattern, recursive, suppressWarnings); // need to add builder to FPOjbect
+                return std::make_unique<FilePatternObject>(path, file_pattern, recursive, suppressWarnings, sorted); // need to add builder to FPOjbect
             }
 
             if(fs::is_regular_file(path)) {
@@ -56,12 +58,12 @@ class FilePatternFactory {
                     std::getline(infile, str);
 
                 if(VectorParser::isStitchingVector(str)) {
-                    return std::make_unique<ExternalVectorPattern>(path, file_pattern, block_size, suppressWarnings); // need to add builder to FPOjbect
+                    return std::make_unique<ExternalVectorPattern>(path, file_pattern, block_size, suppressWarnings, sorted); // need to add builder to FPOjbect
                 }
 
-                return std::make_unique<ExternalStringPattern>(path, file_pattern, block_size, suppressWarnings); // need to add builder to FPOjbect
+                return std::make_unique<ExternalStringPattern>(path, file_pattern, block_size, suppressWarnings, sorted); // need to add builder to FPOjbect
             }
 
-            return std::make_unique<ExternalFilePattern>(path, file_pattern, block_size, recursive, suppressWarnings); // need to add builder to FPOjbect
+            return std::make_unique<ExternalFilePattern>(path, file_pattern, block_size, recursive, suppressWarnings, sorted); // need to add builder to FPOjbect
         }
 };