First cut at reader-side derived variables

ornladios · Jul 16, 2024 · 1b5d544 · 1b5d544
1 parent ab1518d
commit 1b5d544
Show file tree

Hide file tree

Showing 11 changed files with 334 additions and 14 deletions.
diff --git a/source/adios2/common/ADIOSTypes.cpp b/source/adios2/common/ADIOSTypes.cpp
@@ -35,6 +35,23 @@ std::string ToString(MemorySpace value)
     }
 }
 
+#ifdef ADIOS2_HAVE_DERIVED_VARIABLE
+std::string ToString(DerivedVarType value)
+{
+    switch (value)
+    {
+    case DerivedVarType::MetadataOnly:
+        return "DerivedVarType::MetadataOnly";
+    case DerivedVarType::ExpressionString:
+        return "DerivedVarType::ExpressionString";
+    case DerivedVarType::StoreData:
+        return "DerivedVarType::StoreData";
+    default:
+        return "ToString:  Unknown DerivedVarType";
+    }
+};
+#endif
+
 std::string ToString(ShapeID value)
 {
     switch (value)

diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h
@@ -344,6 +344,9 @@ bool TypeHasMinMax(DataType adiosvartype);
  */
 
 std::string ToString(ShapeID value);
+#ifdef ADIOS2_HAVE_DERIVED_VARIABLE
+std::string ToString(DerivedVarType value);
+#endif
 std::string ToString(IOMode value);
 std::string ToString(Mode value);
 std::string ToString(ReadMultiplexPattern value);

diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h
@@ -471,6 +471,13 @@ class Engine
         return nullptr;
     }
 
+    //  in this call, Step is RELATIVE, not absolute
+    virtual MinVarInfo *MinBlocksInfo(const VariableBase &, const size_t Step,
+                                      const size_t WriterID, const size_t BlockID) const
+    {
+        return nullptr;
+    }
+
     //  in this call, Step is RELATIVE, not absolute
     virtual bool VarShape(const VariableBase &, const size_t Step, Dims &Shape) const
     {

diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp
@@ -349,6 +349,22 @@ bool IO::RemoveVariable(const std::string &name) noexcept
     return isRemoved;
 }
 
+#ifdef ADIOS2_HAVE_DERIVED_VARIABLE
+bool IO::RemoveDerivedVariable(const std::string &name) noexcept
+{
+    PERFSTUBS_SCOPED_TIMER("IO::RemoveVariable");
+    bool isRemoved = false;
+    auto itVariable = m_VariablesDerived.find(name);
+    // variable exists
+    if (itVariable != m_VariablesDerived.end())
+    {
+        m_VariablesDerived.erase(itVariable);
+        isRemoved = true;
+    }
+    return isRemoved;
+}
+#endif
+
 void IO::RemoveAllVariables() noexcept
 {
     PERFSTUBS_SCOPED_TIMER("IO::RemoveAllVariables");

diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h
@@ -235,6 +235,18 @@ class IO
      */
     bool RemoveVariable(const std::string &name) noexcept;
 
+#ifdef ADIOS2_HAVE_DERIVED_VARIABLE
+    /**
+     * @brief Removes an existing DerivedVariable in current IO object.
+     * Dangerous function since references and
+     * pointers can be dangling after this call.
+     * @param name unique identifier input
+     * @return true: found and removed variable, false: not found, nothing to
+     * remove
+     */
+    bool RemoveDerivedVariable(const std::string &name) noexcept;
+#endif
+
     /**
      * @brief Removes all existing variables in current IO object.
      * Dangerous function since references and

diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp
@@ -355,10 +355,7 @@ void BP5Reader::PerformGets()
     }
 
     // clear pending requests inside deserializer
-    {
-        std::vector<adios2::format::BP5Deserializer::ReadRequest> empty;
-        m_BP5Deserializer->FinalizeGets(empty);
-    }
+    m_BP5Deserializer->ClearGetState();
 }
 
 void BP5Reader::PerformRemoteGetsWithKVCache()
@@ -661,6 +658,8 @@ void BP5Reader::PerformLocalGets()
             m_BP5Deserializer->FinalizeGet(Req, false);
         }
     }
+    m_BP5Deserializer->FinalizeDerivedGets(ReadRequests);
+    m_BP5Deserializer->ClearGetState();
     m_JSONProfiler.Stop("DataRead");
     /*TP end = NOW();
     double t1 = DURATION(start, end);
@@ -914,6 +913,12 @@ MinVarInfo *BP5Reader::MinBlocksInfo(const VariableBase &Var, const size_t Step)
     return m_BP5Deserializer->MinBlocksInfo(Var, Step);
 }
 
+MinVarInfo *BP5Reader::MinBlocksInfo(const VariableBase &Var, const size_t Step,
+                                     const size_t WriterID, const size_t BlockID) const
+{
+    return m_BP5Deserializer->MinBlocksInfo(Var, Step, WriterID, BlockID);
+}
+
 bool BP5Reader::VarShape(const VariableBase &Var, const size_t Step, Dims &Shape) const
 {
     return m_BP5Deserializer->VarShape(Var, Step, Shape);

diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h
@@ -56,6 +56,8 @@ class BP5Reader : public BP5Engine, public Engine
     void PerformGets() final;
 
     MinVarInfo *MinBlocksInfo(const VariableBase &, const size_t Step) const;
+    MinVarInfo *MinBlocksInfo(const VariableBase &, const size_t Step, const size_t WriterID,
+                              const size_t BlockID) const;
     bool VarShape(const VariableBase &Var, const size_t Step, Dims &Shape) const;
     bool VariableMinMax(const VariableBase &, const size_t Step, MinMaxStruct &MinMax);
     std::string VariableExprStr(const VariableBase &Var);

diff --git a/source/adios2/engine/sst/SstReader.cpp b/source/adios2/engine/sst/SstReader.cpp
@@ -695,6 +695,7 @@ void SstReader::BP5PerformGets()
     }
 
     m_BP5Deserializer->FinalizeGets(ReadRequests);
+    m_BP5Deserializer->ClearGetState();
 }
 
 void SstReader::PerformGets()