Fix CampaignReader when a campaign has multiple hosts. Directory indi…

…ces are global, not per-host indices
ornladios · Jul 23, 2024 · 33f059a · 33f059a
1 parent 5b8a834
commit 33f059a
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 14 deletions.
diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp
@@ -61,7 +61,8 @@ static int sqlcb_directory(void *p, int argc, char **argv, char **azColName)
     CampaignData *cdp = reinterpret_cast<CampaignData *>(p);
     size_t hostid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int");
     size_t hostidx = hostid - 1; // SQL rows start from 1, vector idx start from 0
-    cdp->hosts[hostidx].directory.push_back(argv[1]);
+    cdp->directory.push_back(argv[1]);
+    cdp->hosts[hostidx].dirIdx.push_back(cdp->directory.size() - 1);
     return 0;
 };
 
@@ -73,7 +74,7 @@ static int sqlcb_bpdataset(void *p, int argc, char **argv, char **azColName)
     size_t hostid = helper::StringToSizeT(std::string(argv[1]), "SQL callback convert text to int");
     size_t dirid = helper::StringToSizeT(std::string(argv[2]), "SQL callback convert text to int");
     cds.hostIdx = hostid - 1; // SQL rows start from 1, vector idx start from 0
-    cds.dirIdx = dirid - 1;
+    cds.dirIdx = dirid - 1;   // SQL rows start from 1, vector idx start from 0
     cds.name = argv[3];
     cdp->bpdatasets[dsid] = cds;
     return 0;

diff --git a/source/adios2/engine/campaign/CampaignData.h b/source/adios2/engine/campaign/CampaignData.h
@@ -31,7 +31,7 @@ struct CampaignHost
 {
     std::string hostname;
     std::string longhostname;
-    std::vector<std::string> directory;
+    std::vector<size_t> dirIdx; // index in CampaignData.directory global list of dirs
 };
 
 struct CampaignBPFile
@@ -56,6 +56,7 @@ struct CampaignData
 {
     std::string version;
     std::vector<CampaignHost> hosts;
+    std::vector<std::string> directory;
     std::map<size_t, CampaignBPDataset> bpdatasets;
 };
 

diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp
@@ -198,18 +198,17 @@ void CampaignReader::InitTransports()
             CampaignHost &h = m_CampaignData.hosts[hostidx];
             std::cout << "    host =" << h.hostname << "  long name = " << h.longhostname
                       << "  directories: \n";
-            for (size_t diridx = 0; diridx < h.directory.size(); ++diridx)
+            for (size_t diridx = 0; diridx < h.dirIdx.size(); ++diridx)
             {
-                std::cout << "      dir = " << h.directory[diridx] << "\n";
+                std::cout << "      dir = " << m_CampaignData.directory[h.dirIdx[diridx]] << "\n";
             }
         }
         std::cout << "  datasets:\n";
         for (auto &it : m_CampaignData.bpdatasets)
         {
             CampaignBPDataset &ds = it.second;
             std::cout << "    " << m_CampaignData.hosts[ds.hostIdx].hostname << ":"
-                      << m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] << PathSeparator
-                      << ds.name << "\n";
+                      << m_CampaignData.directory[ds.dirIdx] << PathSeparator << ds.name << "\n";
             for (auto &bpf : ds.files)
             {
                 std::cout << "      file: " << bpf.name << "\n";
@@ -237,8 +236,7 @@ void CampaignReader::InitTransports()
                 if (ho.protocol == HostAccessProtocol::S3)
                 {
                     const std::string endpointURL = ho.endpoint;
-                    const std::string objPath =
-                        m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + "/" + ds.name;
+                    const std::string objPath = m_CampaignData.directory[ds.dirIdx] + "/" + ds.name;
                     Params p;
                     p.emplace("Library", "awssdk");
                     p.emplace("endpoint", endpointURL);
@@ -249,8 +247,7 @@ void CampaignReader::InitTransports()
                     p.emplace("recheck_metadata", (ho.recheckMetadata ? "true" : "false"));
                     io.AddTransport("File", p);
                     io.SetEngine("BP5");
-                    localPath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] +
-                                PathSeparator + ds.name;
+                    localPath = m_CampaignData.directory[ds.dirIdx] + PathSeparator + ds.name;
                     if (ho.isAWS_EC2)
                     {
                         adios2sys::SystemTools::PutEnv("AWS_EC2_METADATA_DISABLED=false");
@@ -277,7 +274,7 @@ void CampaignReader::InitTransports()
             if (!done)
             {
                 const std::string remotePath =
-                    m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name;
+                    m_CampaignData.directory[ds.dirIdx] + PathSeparator + ds.name;
                 const std::string remoteURL =
                     m_CampaignData.hosts[ds.hostIdx].hostname + ":" + remotePath;
                 localPath = m_Options.cachepath + PathSeparator +
@@ -302,8 +299,7 @@ void CampaignReader::InitTransports()
         }
         else
         {
-            localPath =
-                m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name;
+            localPath = m_CampaignData.directory[ds.dirIdx] + PathSeparator + ds.name;
             if (m_Options.verbose > 0)
             {
                 std::cout << "Open local file " << localPath << "\n";