Merge pull request #8 from williamfgc/bpindataman

Implemented new function to reset metadata in BP3Deserializer
ornladios · Jan 29, 2018 · 4a0da85 · 4a0da85
2 parents cf0d2fd + f78c520
commit 4a0da85
Show file tree

Hide file tree

Showing 6 changed files with 36 additions and 24 deletions.
diff --git a/examples/hello/datamanReader/helloDataManReader.cpp b/examples/hello/datamanReader/helloDataManReader.cpp
@@ -49,29 +49,30 @@ int main(int argc, char *argv[])
 
         for (unsigned int i = 0; i < 3; ++i)
         {
-            std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+            std::this_thread::sleep_for(std::chrono::milliseconds(10000));
         }
 
-        adios2::Variable<float> *bpFloats;
-
-        std::vector<float> myFloats(10);
-
-        bpFloats = dataManIO.InquireVariable<float>("bpFloats");
-        if (bpFloats == nullptr)
-        {
-            std::cout << "Variable bpFloats not read...yet\n";
-        }
-
-        else
-        {
-            for (int i = 0; i < 1000; ++i)
-            {
-                dataManReader.BeginStep();
-                dataManReader.GetSync<float>(*bpFloats, myFloats.data());
-                Dump(myFloats);
-                dataManReader.EndStep();
-            }
-        }
+        //        adios2::Variable<float> *bpFloats;
+        //
+        //        std::vector<float> myFloats(10);
+        //
+        //        bpFloats = dataManIO.InquireVariable<float>("bpFloats");
+        //        if (bpFloats == nullptr)
+        //        {
+        //            std::cout << "Variable bpFloats not read...yet\n";
+        //        }
+        //
+        //        else
+        //        {
+        //            for (int i = 0; i < 1000; ++i)
+        //            {
+        //                dataManReader.BeginStep();
+        //                dataManReader.GetSync<float>(*bpFloats,
+        //                myFloats.data());
+        //                Dump(myFloats);
+        //                dataManReader.EndStep();
+        //            }
+        //        }
 
         dataManReader.Close();
     }

diff --git a/examples/hello/datamanWriter/helloDataManWriter.cpp b/examples/hello/datamanWriter/helloDataManWriter.cpp
@@ -49,7 +49,6 @@ int main(int argc, char *argv[])
                        {"Library", "ZMQ"}, {"IPAddress", "127.0.0.1"},
                    });
 
-        // Define variable and local size
         auto bpFloats =
             dataManIO.DefineVariable<float>("bpFloats", {}, {}, {Nx});
 
@@ -60,6 +59,7 @@ int main(int argc, char *argv[])
 
         for (int i = 0; i < 1000; ++i)
         {
+            // Define variable and local size
             dataManWriter.BeginStep();
             for (auto &j : myFloats)
             {

diff --git a/source/adios2/engine/dataman/DataManReader.cpp b/source/adios2/engine/dataman/DataManReader.cpp
@@ -67,7 +67,7 @@ void DataManReader::ReadThread(std::shared_ptr<transportman::DataMan> man)
             // implementation needs multiple threads then this has to be
             // rewriten by manually removing variables one by one and it
             // has to be protected by mutex.
-            //                                m_IO.RemoveAllVariables();
+
             std::shared_ptr<std::vector<char>> buffer = man->ReadWAN();
             if (buffer != nullptr)
             {
@@ -80,6 +80,8 @@ void DataManReader::ReadThread(std::shared_ptr<transportman::DataMan> man)
                                 buffer->data(), buffer->size());
 
                     m_MutexIO.lock();
+                    m_IO.RemoveAllVariables();
+                    m_IO.RemoveAllAttributes();
                     std::cout << "ParseMetadata ===\n";
                     deserializer.ParseMetadata(deserializer.m_Data, m_IO);
                     m_MutexIO.unlock();

diff --git a/source/adios2/engine/dataman/DataManWriter.cpp b/source/adios2/engine/dataman/DataManWriter.cpp
@@ -41,7 +41,8 @@ void DataManWriter::EndStep()
         m_BP3Serializer.CloseStream(m_IO);
         m_Man.WriteWAN(m_BP3Serializer.m_Data.m_Buffer,
                        m_BP3Serializer.m_Data.m_Position);
-        m_BP3Serializer.ResetBuffer(m_BP3Serializer.m_Data);
+        m_BP3Serializer.ResetBuffer(m_BP3Serializer.m_Data, true);
+        m_BP3Serializer.ResetIndices();
     }
 }
 

diff --git a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
@@ -164,6 +164,12 @@ void BP3Serializer::CloseStream(IO &io)
     ProfilerStop("buffering");
 }
 
+void BP3Serializer::ResetIndices()
+{
+    m_MetadataSet.AttributesIndices.clear();
+    m_MetadataSet.VarsIndices.clear();
+}
+
 std::string BP3Serializer::GetRankProfilingJSON(
     const std::vector<std::string> &transportsTypes,
     const std::vector<profiling::IOChrono *> &transportsProfilers) noexcept

diff --git a/source/adios2/toolkit/format/bp3/BP3Serializer.h b/source/adios2/toolkit/format/bp3/BP3Serializer.h
@@ -95,6 +95,8 @@ class BP3Serializer : public BP3Base
      */
     void CloseStream(IO &io);
 
+    void ResetIndices();
+
     /**
      * Get a string with profiling information for this rank
      * @param name stream name