ci: find branchs in chrono order

.gitlab/config/generate_pipelines.py

@@ -5,11 +5,11 @@
 #
 # generate_pipeline.py
 #
-#  Created: May 19, 2023
 #   Author: Vicente Adolfo Bolea Sanchez <vicente.bolea@kitware.com>
 
 from datetime import datetime
 import argparse
+import itertools
 import requests
 import time
 import re
@@ -18,18 +18,43 @@
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 
-def is_date_after(date, days):
-    deadline_sec = int(time.time()) - (days * 86400)
-    utc_dt = datetime.strptime(date, '%Y-%m-%dT%H:%M:%SZ')
-    timestamp_sec = (utc_dt - datetime(1970, 1, 1)).total_seconds()
-    return timestamp_sec > deadline_sec
-
-
-def request_dict(url):
+def request_as_dict(url):
     r = requests.get(url + '?per_page=100', verify=False)
     return r.json()
 
 
+def add_timestamp(branch):
+    date_str = branch['commit']['committed_date']
+    # We ignore the TZ since Gitlab/GitHub always reports in UTC
+    branch['dt'] = int(
+        datetime.strptime(date_str.split(".")[0],
+                          '%Y-%m-%dT%H:%M:%S').timestamp())
+    return branch
+
+
+def is_recent(branch):
+    deadline_sec = int(time.time()) - (args.days * 86400)
+    return branch['dt'] > deadline_sec
+
+
+def has_no_status(branch):
+    gh_commit_sha = branch['commit']['id']
+    # Backported branches use the merge head
+    if re.fullmatch(r'^pr\d+_.*$', branch['name']):
+        gh_commit_sha = branch['commit']['parent_ids'][1]
+
+    # Query GitHub for the status of this commit
+    commit = request_as_dict(gh_url + '/commits/' + gh_commit_sha + '/status')
+    if commit is None or 'sha' not in commit:
+        return False
+
+    for status in commit['statuses']:
+        if status['context'] == args.gh_context:
+            return False
+
+    return True
+
+
 parser = argparse.ArgumentParser(
     prog='generate_pipeline.py',
     description='Generate Dynamic pipelines for Gitlab')
@@ -39,12 +64,12 @@ def request_dict(url):
 parser.add_argument(
     '-n', '--gh-name', required=True,
     help='Full name of the GitHub project. Ex: ornladios/ADIOS2')
-parser.add_argument(
-    '-c', '--gh-context', default='OLCF Crusher (Frontier)',
-    help='Name of the status in GitHub (A.K.A context)')
 parser.add_argument(
     '-p', '--project_id', required=True,
     help='Gitlab internal project ID of the project.')
+parser.add_argument(
+    '-c', '--gh-context', default='OLCF Crusher (Frontier)',
+    help='Name of the status in GitHub (A.K.A context)')
 parser.add_argument(
     '-d', '--days', type=int, default=1,
     help='How many days back to search for commits')
@@ -57,34 +82,21 @@ def request_dict(url):
 args = parser.parse_args()
 
 
-with open(args.template_file, "r") as fd:
+gl_url = args.gl_url + '/api/v4/projects/' + str(args.project_id)
+gh_url = 'https://api.github.com/repos/' + args.gh_name
+
+with open(args.template_file, 'r') as fd:
     template_str = fd.read()
-    gl_url = args.gl_url + "/api/v4/projects/" + str(args.project_id)
-    gh_url = 'https://api.github.com/repos/' + args.gh_name
-    branches = request_dict(gl_url + "/repository/branches")
-    num_pipeline = 0
+
+    branches = request_as_dict(gl_url + '/repository/branches')
+    branches = map(add_timestamp, branches)
+    branches = filter(is_recent, branches)
+    branches = filter(has_no_status, branches)
+
+    # Select the arg.max most least recent branches
+    branches = sorted(branches, key=lambda x: x['dt'])
+    branches = itertools.islice(branches, args.max)
+
     for branch in branches:
-        # Convert to ISO 8601 date format.
-        date_stamp = branch['commit']['committed_date'].split('.')[0] + "Z"
-        if num_pipeline < args.max and is_date_after(date_stamp, args.days):
-            commit_sha = branch['commit']['id']
-            # Backported branches use the merge head
-            gh_commit_sha = commit_sha
-            if re.fullmatch(r'^pr\d+_.*$', branch['name']):
-                gh_commit_sha = branch['commit']['parent_ids'][1]
-
-            # Quit if GitHub does not have the commit
-            if 'sha' not in request_dict(gh_url + "/commits/" + gh_commit_sha):
-                continue
-
-            # Query GitHub for the status of this commit
-            commit = request_dict(gh_url + "/commits/" +
-                                  gh_commit_sha + "/status")
-            status_found = False
-            for status in commit['statuses']:
-                if status['context'] == args.gh_context:
-                    status_found = True
-            if not status_found:
-                num_pipeline += 1
-                print(template_str.format(
-                    branch=branch['name'], commit=commit_sha))
+        print(template_str.format(
+            branch=branch['name'], commit=branch['commit']['id']))

cmake/DetectOptions.cmake

@@ -451,7 +451,7 @@ if(DAOS_FOUND)
 endif()
 
 # BP5
-if(ADIOS2_USE_BP5 AND NOT WIN32)
+if(ADIOS2_USE_BP5)
   set(ADIOS2_HAVE_BP5 TRUE)
 endif()
 

source/adios2/core/Engine.tcc

@@ -27,8 +27,9 @@ template <class T>
 typename Variable<T>::Span &Engine::Put(Variable<T> &variable,
                                         const bool initialize, const T &value)
 {
-    CheckOpenModes({{Mode::Write}}, " for variable " + variable.m_Name +
-                                        ", in call to Variable<T>::Span Put");
+    CheckOpenModes({{Mode::Write, Mode::Append}},
+                   " for variable " + variable.m_Name +
+                       ", in call to Variable<T>::Span Put");
     if (!variable.m_Operations.empty())
     {
         helper::Throw<std::invalid_argument>(

source/adios2/core/Variable.tcc

@@ -113,9 +113,13 @@ std::pair<T, T> Variable<T>::DoMinMax(const size_t step) const
         MinMaxStruct MM;
         if (m_Engine->VariableMinMax(*this, step, MM))
         {
+        if (std::is_same<T, std::string>::value) {
+            return minMax;
+        } else {
             minMax.first = *(T *)&MM.MinUnion;
             minMax.second = *(T *)&MM.MaxUnion;
             return minMax;
+            }
         }
     }
     if (m_Engine != nullptr && !m_FirstStreamingStep)

source/adios2/engine/bp5/BP5Reader.cpp

@@ -368,7 +368,7 @@ void BP5Reader::PerformGets()
         // then main thread process the last subset
         for (size_t tid = 0; tid < nThreads - 1; ++tid)
         {
-            futures[tid] = std::async(std::launch::async, lf_Reader, tid + 1,
+            futures[tid] = std::async(std::launch::async, lf_Reader, (int)(tid + 1),
                                       maxOpenFiles);
         }
         // main thread runs last subset of reads
@@ -513,9 +513,9 @@ void BP5Reader::InitParameters()
     }
 
     size_t limit = helper::RaiseLimitNoFile();
-    if (m_Parameters.MaxOpenFilesAtOnce > limit - 8)
+    if (m_Parameters.MaxOpenFilesAtOnce > (unsigned int) limit - 8)
     {
-        m_Parameters.MaxOpenFilesAtOnce = limit - 8;
+        m_Parameters.MaxOpenFilesAtOnce = (unsigned int) limit - 8;
     }
 }
 
@@ -986,11 +986,11 @@ size_t BP5Reader::ParseMetadataIndex(format::BufferSTL &bufferSTL,
         {
             auto p = m_WriterMap.emplace(m_StepsCount, WriterMapStruct());
             auto &s = p.first->second;
-            s.WriterCount = helper::ReadValue<uint64_t>(
+            s.WriterCount = (uint32_t)helper::ReadValue<uint64_t>(
                 buffer, position, m_Minifooter.IsLittleEndian);
-            s.AggregatorCount = helper::ReadValue<uint64_t>(
+            s.AggregatorCount = (uint32_t)helper::ReadValue<uint64_t>(
                 buffer, position, m_Minifooter.IsLittleEndian);
-            s.SubfileCount = helper::ReadValue<uint64_t>(
+            s.SubfileCount = (uint32_t)helper::ReadValue<uint64_t>(
                 buffer, position, m_Minifooter.IsLittleEndian);
             // Get the process -> subfile map
             s.RankToSubfile.reserve(s.WriterCount);

source/adios2/engine/bp5/BP5Writer.cpp

@@ -160,6 +160,7 @@ void BP5Writer::WriteMetaMetadata(
         m_FileMetaMetadataManager.WriteFiles((char *)b.MetaMetaInfo,
                                              b.MetaMetaInfoLen);
     }
+    m_FileMetaMetadataManager.FlushFiles();
 }
 
 uint64_t
@@ -206,6 +207,8 @@ BP5Writer::WriteMetadata(const std::vector<core::iovec> &MetaDataBlocks,
         MetaDataSize += b.iov_len;
     }
 
+    m_FileMetadataManager.FlushFiles();
+
     m_MetaDataPos += MetaDataSize;
     return MetaDataSize;
 }
@@ -272,6 +275,7 @@ void BP5Writer::WriteData(format::BufferV *Data)
                     std::to_string(m_Parameters.AggregationType) +
                     "is not supported in BP5");
         }
+        m_FileDataManager.FlushFiles();
         delete Data;
     }
 }
@@ -337,8 +341,6 @@ void BP5Writer::WriteData_EveryoneWrites(format::BufferV *Data,
 void BP5Writer::WriteMetadataFileIndex(uint64_t MetaDataPos,
                                        uint64_t MetaDataSize)
 {
-    m_FileMetadataManager.FlushFiles();
-
     // bufsize: Step record
     size_t bufsize =
         1 + (4 + ((FlushPosSizeInfo.size() * 2) + 1) * m_Comm.Size()) *
@@ -407,7 +409,6 @@ void BP5Writer::WriteMetadataFileIndex(uint64_t MetaDataPos,
     }
 
     m_FileMetadataIndexManager.WriteFiles((char *)buf.data(), buf.size());
-
 #ifdef DUMPDATALOCINFO
     std::cout << "Flush count is :" << FlushPosSizeInfo.size() << std::endl;
     std::cout << "Write Index positions = {" << std::endl;
@@ -427,6 +428,8 @@ void BP5Writer::WriteMetadataFileIndex(uint64_t MetaDataPos,
     }
     std::cout << "}" << std::endl;
 #endif
+    m_FileMetadataIndexManager.FlushFiles();
+
     /* reset for next timestep */
     FlushPosSizeInfo.clear();
 }
@@ -472,7 +475,7 @@ void BP5Writer::MarshalAttributes()
 
         if (!attributePair.second->m_IsSingleValue)
         {
-            element_count = (*baseAttr)->m_Elements;
+            element_count = (int)(*baseAttr)->m_Elements;
         }
 
         if (type == DataType::None)
@@ -511,7 +514,7 @@ void BP5Writer::MarshalAttributes()
         void *data_addr = &attribute.m_DataSingleValue;                        \
         if (!attribute.m_IsSingleValue)                                        \
         {                                                                      \
-            element_count = attribute.m_Elements;                              \
+            element_count = (int)attribute.m_Elements;                         \
             data_addr = attribute.m_DataArray.data();                          \
         }                                                                      \
         m_BP5Serializer.MarshalAttribute(attribute.m_Name.c_str(), type,       \
@@ -536,7 +539,7 @@ void BP5Writer::EndStep()
 
     // true: advances step
     auto TSInfo = m_BP5Serializer.CloseTimestep(
-        m_WriterStep, m_Parameters.AsyncWrite || m_Parameters.DirectIO);
+        (int)m_WriterStep, m_Parameters.AsyncWrite || m_Parameters.DirectIO);
 
     /* TSInfo includes NewMetaMetaBlocks, the MetaEncodeBuffer, the
      * AttributeEncodeBuffer and the data encode Vector */
@@ -684,6 +687,10 @@ void BP5Writer::EndStep()
             m_AsyncWriteLock.unlock();
         }
     }
+    m_FileMetadataIndexManager.FlushFiles();
+    m_FileMetadataManager.FlushFiles();
+    m_FileMetaMetadataManager.FlushFiles();
+    m_FileDataManager.FlushFiles();
 
     m_Profiler.Stop("ES");
     m_WriterStep++;
@@ -752,7 +759,8 @@ void BP5Writer::InitParameters()
         {
             size_t k =
                 m_Parameters.StripeSize / m_Parameters.DirectIOAlignOffset + 1;
-            m_Parameters.StripeSize = k * m_Parameters.DirectIOAlignOffset;
+            m_Parameters.StripeSize =
+                (unsigned int)(k * m_Parameters.DirectIOAlignOffset);
         }
         if (m_Parameters.BufferChunkSize % m_Parameters.DirectIOAlignOffset)
         {
@@ -852,12 +860,12 @@ uint64_t BP5Writer::CountStepsInMetadataIndex(format::BufferSTL &bufferSTL)
         {
         case IndexRecord::WriterMapRecord:
         {
-            m_AppendWriterCount =
-                helper::ReadValue<uint64_t>(buffer, position, IsLittleEndian);
-            m_AppendAggregatorCount =
-                helper::ReadValue<uint64_t>(buffer, position, IsLittleEndian);
-            m_AppendSubfileCount =
-                helper::ReadValue<uint64_t>(buffer, position, IsLittleEndian);
+            m_AppendWriterCount = (uint32_t)helper::ReadValue<uint64_t>(
+                buffer, position, IsLittleEndian);
+            m_AppendAggregatorCount = (uint32_t)helper::ReadValue<uint64_t>(
+                buffer, position, IsLittleEndian);
+            m_AppendSubfileCount = (uint32_t)helper::ReadValue<uint64_t>(
+                buffer, position, IsLittleEndian);
             if (m_AppendSubfileCount > nDataFiles)
             {
                 nDataFiles = m_AppendSubfileCount;
@@ -939,12 +947,12 @@ uint64_t BP5Writer::CountStepsInMetadataIndex(format::BufferSTL &bufferSTL)
         {
         case IndexRecord::WriterMapRecord:
         {
-            m_AppendWriterCount =
-                helper::ReadValue<uint64_t>(buffer, position, IsLittleEndian);
-            m_AppendAggregatorCount =
-                helper::ReadValue<uint64_t>(buffer, position, IsLittleEndian);
-            m_AppendSubfileCount =
-                helper::ReadValue<uint64_t>(buffer, position, IsLittleEndian);
+            m_AppendWriterCount = (uint32_t)helper::ReadValue<uint64_t>(
+                buffer, position, IsLittleEndian);
+            m_AppendAggregatorCount = (uint32_t)helper::ReadValue<uint64_t>(
+                buffer, position, IsLittleEndian);
+            m_AppendSubfileCount = (uint32_t)helper::ReadValue<uint64_t>(
+                buffer, position, IsLittleEndian);
 
             // Get the process -> subfile map
             writerToFileMap.clear();
@@ -1796,8 +1804,8 @@ void BP5Writer::PutCommon(VariableBase &variable, const void *values, bool sync)
         helper::DimsArray MemoryCount(variable.m_MemoryCount);
         helper::DimsArray varCount(variable.m_Count);
 
-        int DimCount = variable.m_Count.size();
-        std::vector<size_t> ZeroDims(DimCount);
+        int DimCount = (int)variable.m_Count.size();
+        helper::DimsArray ZeroDims(DimCount, (size_t)0);
         // get a temporary span then fill with memselection now
         format::BufferV::BufferPos bp5span(0, 0, 0);
 
@@ -1816,8 +1824,8 @@ void BP5Writer::PutCommon(VariableBase &variable, const void *values, bool sync)
         }
         helper::NdCopy((const char *)values, helper::CoreDims(ZeroDims),
                        MemoryCount, sourceRowMajor, false, (char *)ptr,
-                       MemoryStart, varCount, sourceRowMajor, false, ObjSize,
-                       helper::CoreDims(), helper::CoreDims(),
+                       MemoryStart, varCount, sourceRowMajor, false,
+                       (int)ObjSize, helper::CoreDims(), helper::CoreDims(),
                        helper::CoreDims(), helper::CoreDims(),
                        false /* safemode */, variable.m_MemSpace);
     }

source/adios2/engine/bp5/BP5Writer_EveryoneWrites_Async.cpp

@@ -88,7 +88,7 @@ void BP5Writer::AsyncWriteOwnData(AsyncWriteInfo *info,
     size_t wrote = 0;
     size_t block = 0;
     size_t temp_offset = 0;
-    size_t max_size = std::max(1024 * 1024UL, totalsize / 100UL);
+    size_t max_size = std::max((size_t)1024 * 1024UL, totalsize / 100UL);
 
     bool firstWrite = seekOnFirstWrite;
     while (block < nBlocks)

source/adios2/operator/compress/CompressBlosc.cpp

@@ -181,8 +181,8 @@ size_t CompressBlosc::Operate(const char *dataIn, const Dims &blockStart,
         for (; inputOffset < sizeIn; ++chunk)
         {
             size_t inputChunkSize =
-                std::min(sizeIn - inputOffset,
-                         static_cast<size_t>(BLOSC2_MAX_BUFFERSIZE));
+                std::min<size_t>(sizeIn - inputOffset,
+                                 static_cast<size_t>(BLOSC2_MAX_BUFFERSIZE));
             bloscSize_t maxIntputSize =
                 static_cast<bloscSize_t>(inputChunkSize);
 
@@ -375,8 +375,8 @@ size_t CompressBlosc::DecompressChunkedFormat(const char *bufferIn,
             char *out_ptr = dataOut + currentOutputSize;
 
             size_t outputChunkSize =
-                std::min(uncompressedSize - currentOutputSize,
-                         static_cast<size_t>(BLOSC2_MAX_BUFFERSIZE));
+                std::min<size_t>(uncompressedSize - currentOutputSize,
+                                 static_cast<size_t>(BLOSC2_MAX_BUFFERSIZE));
             bloscSize_t max_output_size =
                 static_cast<bloscSize_t>(outputChunkSize);