From 107164d6fba5db462e739247380d6154448ca295 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Fri, 13 Oct 2023 23:13:05 -0400
Subject: [PATCH] Tweak Remote class and test multi-threaded file remote access
 (#3834)

* Tweak Remote class and test multi-threaded file remote access

* rework

* Bad clang-format

* New formulation
---
 source/adios2/toolkit/remote/Remote.cpp | 22 +++++++++++--------
 source/adios2/toolkit/remote/Remote.h   | 28 ++++++++++---------------
 testing/adios2/engine/bp/CMakeLists.txt | 19 +++++++++++------
 3 files changed, 37 insertions(+), 32 deletions(-)

diff --git a/source/adios2/toolkit/remote/Remote.cpp b/source/adios2/toolkit/remote/Remote.cpp
index 0fe3199420..3a397b05a9 100644
--- a/source/adios2/toolkit/remote/Remote.cpp
+++ b/source/adios2/toolkit/remote/Remote.cpp
@@ -50,16 +50,20 @@ void ReadResponseHandler(CManager cm, CMConnection conn, void *vevent, void *cli
     return;
 };
 
+CManagerSingleton &CManagerSingleton::Instance(RemoteCommon::Remote_evpath_state &ev_state)
+{
+    std::mutex mtx;
+    const std::lock_guard<std::mutex> lock(mtx);
+    static CManagerSingleton instance;
+    ev_state = instance.internalEvState;
+    return instance;
+}
+
 void Remote::InitCMData()
 {
-    std::lock_guard<std::mutex> lockGuard(m_CMInitMutex);
-    bool first = true;
-    auto CM = CManagerSingleton::Instance(first);
-    ev_state.cm = CM->m_cm;
-    RegisterFormats(ev_state);
-    if (first)
-    {
-        CMfork_comm_thread(ev_state.cm);
+    (void)CManagerSingleton::Instance(ev_state);
+    static std::once_flag flag;
+    std::call_once(flag, [&]() {
         CMregister_handler(ev_state.OpenResponseFormat, (CMHandlerFunc)OpenResponseHandler,
                            &ev_state);
         CMregister_handler(ev_state.ReadResponseFormat, (CMHandlerFunc)ReadResponseHandler,
@@ -68,7 +72,7 @@ void Remote::InitCMData()
                            (CMHandlerFunc)OpenSimpleResponseHandler, &ev_state);
         CMregister_handler(ev_state.ReadResponseFormat, (CMHandlerFunc)ReadResponseHandler,
                            &ev_state);
-    }
+    });
 }
 
 void Remote::Open(const std::string hostname, const int32_t port, const std::string filename,
diff --git a/source/adios2/toolkit/remote/Remote.h b/source/adios2/toolkit/remote/Remote.h
index 75824cdc31..608d35f637 100644
--- a/source/adios2/toolkit/remote/Remote.h
+++ b/source/adios2/toolkit/remote/Remote.h
@@ -63,32 +63,26 @@ class Remote
     bool m_Active = false;
 };
 
+#ifdef ADIOS2_HAVE_SST
 class CManagerSingleton
 {
 public:
-#ifdef ADIOS2_HAVE_SST
+    static CManagerSingleton &Instance(RemoteCommon::Remote_evpath_state &ev_state);
+
+private:
     CManager m_cm = NULL;
-#endif
-    static CManagerSingleton *Instance(bool &first)
+    RemoteCommon::Remote_evpath_state internalEvState;
+    CManagerSingleton()
     {
-        static CManagerSingleton *ptr = new CManagerSingleton();
-        static bool internal_first = true;
-        first = internal_first;
-        internal_first = false;
-        return ptr;
+        m_cm = CManager_create();
+        internalEvState.cm = m_cm;
+        RegisterFormats(internalEvState);
+        CMfork_comm_thread(internalEvState.cm);
     }
 
-protected:
-#ifdef ADIOS2_HAVE_SST
-    CManagerSingleton() { m_cm = CManager_create(); }
-
     ~CManagerSingleton() { CManager_close(m_cm); }
-#else
-    CManagerSingleton() {}
-
-    ~CManagerSingleton() {}
-#endif
 };
+#endif
 
 } // end namespace adios2
 
diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt
index 10f724cc1b..03479c7edc 100644
--- a/testing/adios2/engine/bp/CMakeLists.txt
+++ b/testing/adios2/engine/bp/CMakeLists.txt
@@ -107,22 +107,29 @@ bp_gtest_add_tests_helper(LargeMetadata MPI_ALLOW)
 set(BP5LargeMeta "Engine.BP.BPLargeMetadata.BPWrite1D_LargeMetadata.BP5.Serial")
 
 if ((NOT WIN32) AND ADIOS2_HAVE_SST)
-# prototype for remote server testing
-# (we don't really use SST here, just EVPath, but ADIOS2_HAVE_SST is the most relevant conditional)
-   macro(add_remote_tests_helper testname)
+   # prototype for remote server testing
+   # (we don't really use SST here, just EVPath, but ADIOS2_HAVE_SST is the most relevant conditional)
+
+   macro(add_get_remote_tests_helper testname)
       add_test(NAME "Remote.BP${testname}.GetRemote" COMMAND Test.Engine.BP.${testname}.Serial bp5)
       set_tests_properties(Remote.BP${testname}.GetRemote PROPERTIES FIXTURES_REQUIRED Server ENVIRONMENT "DoRemote=1")
    endmacro()
 
+   macro(add_file_remote_tests_helper testname)
+      add_test(NAME "Remote.BP${testname}.FileRemote" COMMAND Test.Engine.BP.${testname}.Serial bp5)
+      set_tests_properties(Remote.BP${testname}.FileRemote PROPERTIES FIXTURES_REQUIRED Server ENVIRONMENT "DoFileRemote=1")
+   endmacro()
+
    add_test(NAME remoteServerSetup   COMMAND remote_server -background)
    set_tests_properties(remoteServerSetup         PROPERTIES FIXTURES_SETUP    Server)
 
    add_test(NAME remoteServerCleanup COMMAND remote_server -kill_server)
    set_tests_properties(remoteServerCleanup         PROPERTIES FIXTURES_CLEANUP    Server)
 
- #add remote tests below this line
-   add_remote_tests_helper(WriteReadADIOS2stdio)
-   add_remote_tests_helper(WriteMemorySelectionRead)
+   ##### add remote tests below this line
+   add_get_remote_tests_helper(WriteReadADIOS2stdio)
+   add_get_remote_tests_helper(WriteMemorySelectionRead)
+   add_file_remote_tests_helper(WriteMemorySelectionRead)
 endif()
 
 if(ADIOS2_HAVE_MPI)