Merge pull request ornladios#2763 from eisenhauer/ArrayOrder

Restructure to allow direct specification of array order

bindings/C/adios2/c/adios2_c_adios.cpp

@@ -17,6 +17,30 @@
 extern "C" {
 #endif
 
+adios2::ArrayOrdering adios2_ToArrayOrdering(const adios2_arrayordering Corder)
+{
+    adios2::ArrayOrdering order = adios2::ArrayOrdering::Auto;
+    switch (Corder)
+    {
+
+    case adios2_arrayordering_rowmajor:
+        order = adios2::ArrayOrdering::RowMajor;
+        break;
+
+    case adios2_arrayordering_columnmajor:
+        order = adios2::ArrayOrdering::ColumnMajor;
+        break;
+
+    case adios2_arrayordering_auto:
+        order = adios2::ArrayOrdering::Auto;
+        break;
+
+    default:
+        break;
+    }
+    return order;
+}
+
 adios2_adios *adios2_init_config_glue_serial(const char *config_file,
                                              const adios2_debug_mode debug_mode,
                                              const char *host_language)
@@ -68,6 +92,25 @@ adios2_io *adios2_declare_io(adios2_adios *adios, const char *name)
     return io;
 }
 
+adios2_io *adios2_declare_io_order(adios2_adios *adios, const char *name,
+                                   adios2_arrayordering order)
+{
+    adios2_io *io = nullptr;
+    try
+    {
+        adios2::helper::CheckForNullptr(
+            adios, "for adios2_adios, in call to adios2_declare_io");
+        io = reinterpret_cast<adios2_io *>(
+            &reinterpret_cast<adios2::core::ADIOS *>(adios)->DeclareIO(
+                name, adios2_ToArrayOrdering(order)));
+    }
+    catch (...)
+    {
+        adios2::helper::ExceptionToError("adios2_declare_io");
+    }
+    return io;
+}
+
 adios2_io *adios2_at_io(adios2_adios *adios, const char *name)
 {
     adios2_io *io = nullptr;

bindings/C/adios2/c/adios2_c_adios.h

@@ -72,6 +72,16 @@ adios2_adios *adios2_init_config_serial(const char *config_file);
  */
 adios2_io *adios2_declare_io(adios2_adios *adios, const char *name);
 
+/**
+ * Declares a new io handler with specific array ordering
+ * @param adios owner the io handler
+ * @param name unique io identifier within current adios handler
+ * @param order array ordering
+ * @return success: handler, failure: NULL
+ */
+adios2_io *adios2_declare_io_order(adios2_adios *adios, const char *name,
+                                   adios2_arrayordering order);
+
 /**
  * Retrieves a previously declared io handler by name
  * @param adios owner the io handler

bindings/C/adios2/c/adios2_c_types.h

@@ -138,6 +138,13 @@ typedef enum
     adios2_shapeid_local_array = 4
 } adios2_shapeid;
 
+typedef enum
+{
+    adios2_arrayordering_rowmajor,
+    adios2_arrayordering_columnmajor,
+    adios2_arrayordering_auto
+} adios2_arrayordering;
+
 static const size_t adios2_string_array_element_max_size = 4096;
 
 static const size_t adios2_local_value_dim = SIZE_MAX - 2;

bindings/CXX11/adios2/cxx11/ADIOS.cpp

@@ -31,10 +31,10 @@ ADIOS::ADIOS(const std::string &configFile, const std::string &hostLanguage,
 
 ADIOS::operator bool() const noexcept { return m_ADIOS ? true : false; }
 
-IO ADIOS::DeclareIO(const std::string name)
+IO ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder)
 {
     CheckPointer("for io name " + name + ", in call to ADIOS::DeclareIO");
-    return IO(&m_ADIOS->DeclareIO(name));
+    return IO(&m_ADIOS->DeclareIO(name, ArrayOrder));
 }
 
 IO ADIOS::AtIO(const std::string name)

bindings/CXX11/adios2/cxx11/ADIOS.h

@@ -164,7 +164,8 @@ class ADIOS
      * @exception std::invalid_argument if IO with unique name is already
      * declared
      */
-    IO DeclareIO(const std::string name);
+    IO DeclareIO(const std::string name,
+                 const ArrayOrdering ArrayOrder = ArrayOrdering::Auto);
 
     /**
      * Retrieve an existing IO object previously created with DeclareIO.

source/adios2/common/ADIOSTypes.h

@@ -141,6 +141,14 @@ enum class DataType
     Compound
 };
 
+/** Type of ArrayOrdering */
+enum class ArrayOrdering
+{
+    RowMajor,    /// Contiguous elements of a row lie together in memory
+    ColumnMajor, /// Contiguous elements of a column lie together in memory
+    Auto         /// Default based on language type
+};
+
 // Types
 using std::size_t;
 

source/adios2/core/ADIOS.cpp

@@ -121,7 +121,7 @@ ADIOS::ADIOS(const std::string hostLanguage)
 
 ADIOS::~ADIOS() = default;
 
-IO &ADIOS::DeclareIO(const std::string name)
+IO &ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder)
 {
     auto itIO = m_IOs.find(name);
 
@@ -132,6 +132,7 @@ IO &ADIOS::DeclareIO(const std::string name)
         if (!io.IsDeclared()) // exists from config xml
         {
             io.SetDeclared();
+            io.SetArrayOrder(ArrayOrder);
             return io;
         }
         else
@@ -149,6 +150,7 @@ IO &ADIOS::DeclareIO(const std::string name)
         std::forward_as_tuple(*this, name, false, m_HostLanguage));
     IO &io = ioPair.first->second;
     io.SetDeclared();
+    io.SetArrayOrder(ArrayOrder);
     return io;
 }
 

source/adios2/core/ADIOS.h

@@ -86,7 +86,8 @@ class ADIOS
      * @exception std::invalid_argument if IO with unique name is already
      * declared
      */
-    IO &DeclareIO(const std::string name);
+    IO &DeclareIO(const std::string name,
+                  const ArrayOrdering ArrayOrder = ArrayOrdering::Auto);
 
     /**
      * Retrieve a reference to an existing IO object created with DeclareIO.

source/adios2/core/IO.cpp

@@ -322,6 +322,19 @@ bool IO::InConfigFile() const noexcept { return m_InConfigFile; }
 
 void IO::SetDeclared() noexcept { m_IsDeclared = true; }
 
+void IO::SetArrayOrder(const ArrayOrdering ArrayOrder) noexcept
+{
+    if (ArrayOrder == ArrayOrdering::Auto)
+    {
+        if (helper::IsRowMajor(m_HostLanguage))
+            m_ArrayOrder = ArrayOrdering::RowMajor;
+        else
+            m_ArrayOrder = ArrayOrdering::ColumnMajor;
+    }
+    else
+        m_ArrayOrder = ArrayOrder;
+}
+
 bool IO::IsDeclared() const noexcept { return m_IsDeclared; }
 
 bool IO::RemoveVariable(const std::string &name) noexcept

source/adios2/core/IO.h

@@ -59,6 +59,8 @@ class IO
     /** from ADIOS class passed to Engine created with Open */
     const std::string m_HostLanguage = "C++";
 
+    ArrayOrdering m_ArrayOrder;
+
     /** From SetParameter, parameters for a particular engine from m_Type */
     Params m_Parameters;
 
@@ -351,6 +353,8 @@ class IO
      */
     void SetDeclared() noexcept;
 
+    void SetArrayOrder(const ArrayOrdering ArrayOrder) noexcept;
+
     /**
      * Check if declared in code
      * @return true: created with ADIOS DeclareIO, false: dummy from config file

source/adios2/engine/bp3/BP3Reader.tcc

@@ -105,7 +105,7 @@ void BP3Reader::ReadVariableBlocks(Variable<T> &variable)
 
                 m_BP3Deserializer.PostDataRead(
                     variable, blockInfo, subStreamBoxInfo,
-                    helper::IsRowMajor(m_IO.m_HostLanguage), 0);
+                    m_IO.m_ArrayOrder == ArrayOrdering::RowMajor, 0);
             } // substreams loop
             // advance pointer to next step
             blockInfo.Data += helper::GetTotalSize(blockInfo.Count);

source/adios2/engine/bp3/BP3Writer.cpp

@@ -222,7 +222,8 @@ void BP3Writer::InitBPBuffer()
     else
     {
         m_BP3Serializer.PutProcessGroupIndex(
-            m_IO.m_Name, m_IO.m_HostLanguage,
+            m_IO.m_Name,
+            (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran",
             m_FileDataManager.GetTransportsTypes());
     }
 }

source/adios2/engine/bp3/BP3Writer.tcc

@@ -40,7 +40,8 @@ void BP3Writer::PutCommon(Variable<T> &variable,
     if (!m_BP3Serializer.m_MetadataSet.DataPGIsOpen)
     {
         m_BP3Serializer.PutProcessGroupIndex(
-            m_IO.m_Name, m_IO.m_HostLanguage,
+            m_IO.m_Name,
+            (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran",
             m_FileDataManager.GetTransportsTypes());
     }
 
@@ -53,7 +54,7 @@ void BP3Writer::PutCommon(Variable<T> &variable,
     }
 
     // WRITE INDEX to data buffer and metadata structure (in memory)//
-    const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage);
+    const bool sourceRowMajor = (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor);
     m_BP3Serializer.PutVariableMetadata(variable, blockInfo, sourceRowMajor,
                                         &span);
     span.m_Value = value;
@@ -83,7 +84,8 @@ void BP3Writer::PutSyncCommon(Variable<T> &variable,
     if (!m_BP3Serializer.m_MetadataSet.DataPGIsOpen)
     {
         m_BP3Serializer.PutProcessGroupIndex(
-            m_IO.m_Name, m_IO.m_HostLanguage,
+            m_IO.m_Name,
+            (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran",
             m_FileDataManager.GetTransportsTypes());
     }
 
@@ -94,12 +96,13 @@ void BP3Writer::PutSyncCommon(Variable<T> &variable,
 
         // new group index for incoming variable
         m_BP3Serializer.PutProcessGroupIndex(
-            m_IO.m_Name, m_IO.m_HostLanguage,
+            m_IO.m_Name,
+            (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran",
             m_FileDataManager.GetTransportsTypes());
     }
 
     // WRITE INDEX to data buffer and metadata structure (in memory)//
-    const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage);
+    const bool sourceRowMajor = (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor);
     m_BP3Serializer.PutVariableMetadata(variable, blockInfo, sourceRowMajor);
     m_BP3Serializer.PutVariablePayload(variable, blockInfo, sourceRowMajor);
 }

source/adios2/engine/bp4/BP4Reader.tcc

@@ -105,7 +105,7 @@ void BP4Reader::ReadVariableBlocks(Variable<T> &variable)
 
                 m_BP4Deserializer.PostDataRead(
                     variable, blockInfo, subStreamBoxInfo,
-                    helper::IsRowMajor(m_IO.m_HostLanguage), 0);
+                    m_IO.m_ArrayOrder == ArrayOrdering::RowMajor, 0);
             } // substreams loop
             // advance pointer to next step
             blockInfo.Data += helper::GetTotalSize(blockInfo.Count);

source/adios2/engine/bp4/BP4Writer.cpp

@@ -393,7 +393,8 @@ void BP4Writer::InitBPBuffer()
     }
 
     m_BP4Serializer.PutProcessGroupIndex(
-        m_IO.m_Name, m_IO.m_HostLanguage,
+        m_IO.m_Name,
+        (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran",
         m_FileDataManager.GetTransportsTypes());
 }
 

source/adios2/engine/bp4/BP4Writer.tcc

@@ -40,7 +40,8 @@ void BP4Writer::PutCommon(Variable<T> &variable,
     if (!m_BP4Serializer.m_MetadataSet.DataPGIsOpen)
     {
         m_BP4Serializer.PutProcessGroupIndex(
-            m_IO.m_Name, m_IO.m_HostLanguage,
+            m_IO.m_Name,
+            (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran",
             m_FileDataManager.GetTransportsTypes());
     }
 
@@ -53,7 +54,7 @@ void BP4Writer::PutCommon(Variable<T> &variable,
     }
 
     // WRITE INDEX to data buffer and metadata structure (in memory)//
-    const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage);
+    const bool sourceRowMajor = m_IO.m_ArrayOrder == ArrayOrdering::RowMajor;
     m_BP4Serializer.PutVariableMetadata(variable, blockInfo, sourceRowMajor,
                                         &span);
     span.m_Value = value;
@@ -82,7 +83,8 @@ void BP4Writer::PutSyncCommon(Variable<T> &variable,
     if (!m_BP4Serializer.m_MetadataSet.DataPGIsOpen)
     {
         m_BP4Serializer.PutProcessGroupIndex(
-            m_IO.m_Name, m_IO.m_HostLanguage,
+            m_IO.m_Name,
+            (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran",
             m_FileDataManager.GetTransportsTypes());
     }
 
@@ -93,12 +95,13 @@ void BP4Writer::PutSyncCommon(Variable<T> &variable,
 
         // new group index for incoming variable
         m_BP4Serializer.PutProcessGroupIndex(
-            m_IO.m_Name, m_IO.m_HostLanguage,
+            m_IO.m_Name,
+            (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran",
             m_FileDataManager.GetTransportsTypes());
     }
 
     // WRITE INDEX to data buffer and metadata structure (in memory)//
-    const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage);
+    const bool sourceRowMajor = m_IO.m_ArrayOrder == ArrayOrdering::RowMajor;
     m_BP4Serializer.PutVariableMetadata(variable, blockInfo, sourceRowMajor);
     m_BP4Serializer.PutVariablePayload(variable, blockInfo, sourceRowMajor);
 }

source/adios2/engine/bp5/BP5Reader.cpp

@@ -211,7 +211,7 @@ void BP5Reader::Init()
     m_IO.m_ReadStreaming = false;
 
     ParseParams(m_IO, m_Parameters);
-    m_ReaderIsRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage);
+    m_ReaderIsRowMajor = (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor);
     InitTransports();
 
     /* Do a collective wait for the file(s) to appear within timeout.

source/adios2/engine/bp5/BP5Writer.cpp

@@ -794,7 +794,7 @@ void BP5Writer::MakeHeader(format::BufferSTL &b, const std::string fileType,
     // byte 48 columnMajor
     // write if data is column major in metadata and data
     const uint8_t columnMajor =
-        (helper::IsRowMajor(m_IO.m_HostLanguage) == false) ? 'y' : 'n';
+        (m_IO.m_ArrayOrder == ArrayOrdering::ColumnMajor) ? 'y' : 'n';
     helper::CopyToBuffer(buffer, position, &columnMajor);
 
     // byte 49-63: unused

source/adios2/engine/dataman/DataManReader.cpp

@@ -22,7 +22,7 @@ DataManReader::DataManReader(IO &io, const std::string &name,
                              const Mode openMode, helper::Comm comm)
 : Engine("DataManReader", io, name, openMode, std::move(comm)),
   m_FinalStep(std::numeric_limits<signed long int>::max()),
-  m_Serializer(m_Comm, helper::IsRowMajor(io.m_HostLanguage)),
+  m_Serializer(m_Comm, (io.m_ArrayOrder == ArrayOrdering::RowMajor)),
   m_RequesterThreadActive(true), m_SubscriberThreadActive(true)
 {
     m_MpiRank = m_Comm.Rank();

source/adios2/engine/dataman/DataManReader.tcc

@@ -31,7 +31,7 @@ void DataManReader::GetSyncCommon(Variable<T> &variable, T *data)
 template <class T>
 void DataManReader::GetDeferredCommon(Variable<T> &variable, T *data)
 {
-    if (helper::IsRowMajor(m_IO.m_HostLanguage))
+    if (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor)
     {
         while (true)
         {

source/adios2/engine/dataman/DataManWriter.cpp

@@ -20,7 +20,7 @@ namespace engine
 DataManWriter::DataManWriter(IO &io, const std::string &name,
                              const Mode openMode, helper::Comm comm)
 : Engine("DataManWriter", io, name, openMode, std::move(comm)), m_SentSteps(0),
-  m_Serializer(m_Comm, helper::IsRowMajor(io.m_HostLanguage)),
+  m_Serializer(m_Comm, (io.m_ArrayOrder == ArrayOrdering::RowMajor)),
   m_ReplyThreadActive(true), m_PublishThreadActive(true)
 {
 

source/adios2/engine/dataman/DataManWriter.tcc

@@ -32,7 +32,7 @@ template <class T>
 void DataManWriter::PutDeferredCommon(Variable<T> &variable, const T *values)
 {
     variable.SetData(values);
-    if (helper::IsRowMajor(m_IO.m_HostLanguage))
+    if (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor)
     {
         m_Serializer.PutData(variable, m_Name, CurrentStep(), m_MpiRank, "");
     }

source/adios2/engine/dataspaces/DataSpacesReader.cpp

@@ -179,7 +179,7 @@ StepStatus DataSpacesReader::BeginStep(StepMode mode, const float timeout_sec)
         shape.resize(var_dim_size);
         if (var_dim_size > 0)
         {
-            bool isOrderC = helper::IsRowMajor(m_IO.m_HostLanguage);
+            bool isOrderC = io.m_ArrayOrder == RowMajor;
             for (int i = 0; i < var_dim_size; i++)
             {
                 if (isOrderC)

source/adios2/engine/dataspaces/DataSpacesReader.tcc

@@ -60,7 +60,7 @@ void DataSpacesReader::ReadDsData(Variable<T> &variable, T *data, int version)
 {
     uint64_t lb_in[MAX_DS_NDIM], ub_in[MAX_DS_NDIM], gdims_in[MAX_DS_NDIM];
     int ndims = std::max(variable.m_Shape.size(), variable.m_Count.size());
-    bool isOrderC = helper::IsRowMajor(m_IO.m_HostLanguage);
+    bool isOrderC = io.m_ArrayOrder == RowMajor;
     /* Order of dimensions: in DataSpaces: fast --> slow --> slowest
            For example:
            Fortran: i,j,k --> i, j, k  = lb[0], lb[1], lb[2]