Backend implementation

docs/source/dev/design.rst

@@ -23,7 +23,7 @@ Therefore, enabling users to handle hierarchical, self-describing file formats w
 
 .. literalinclude:: IOTask.hpp
    :language: cpp
-   :lines: 48-78
+   :lines: 50-81
 
 Every task is designed to be a fully self-contained description of one such atomic operation. By describing a required minimal step of work (without any side-effect), these operations are the foundation of the unified handling mechanism across suitable file formats.
 The actual low-level exchange of data is implemented in ``IOHandlers``, one per file format (possibly two if handlingi MPI-parallel work is possible and requires different behaviour).

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -206,6 +206,8 @@ class ADIOS2IOHandlerImpl
     void
     deregister(Writable *, Parameter<Operation::DEREGISTER> const &) override;
 
+    void touch(Writable *, Parameter<Operation::TOUCH> const &) override;
+
     /**
      * @brief The ADIOS2 access type to chose for Engines opened
      * within this instance.

include/openPMD/IO/AbstractIOHandlerImpl.hpp

@@ -395,6 +395,11 @@ class AbstractIOHandlerImpl
     virtual void
     deregister(Writable *, Parameter<Operation::DEREGISTER> const &param) = 0;
 
+    /** Treat this writable's file as open/active/dirty.
+     */
+    virtual void
+    touch(Writable *, Parameter<Operation::TOUCH> const &param) = 0;
+
     AbstractIOHandler *m_handler;
     bool m_verboseIOTasks = false;
 

include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp

@@ -80,6 +80,7 @@ class HDF5IOHandlerImpl : public AbstractIOHandlerImpl
     void listAttributes(Writable *, Parameter<Operation::LIST_ATTS> &) override;
     void
     deregister(Writable *, Parameter<Operation::DEREGISTER> const &) override;
+    void touch(Writable *, Parameter<Operation::TOUCH> const &) override;
 
     std::unordered_map<Writable *, std::string> m_fileNames;
     std::unordered_map<std::string, hid_t> m_fileNamesWithID;

include/openPMD/IO/IOTask.hpp

@@ -48,20 +48,36 @@ Writable *getWritable(Attributable *);
 /** Type of IO operation between logical and persistent data.
  */
 OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){
-    CREATE_FILE,      CHECK_FILE,     OPEN_FILE,     CLOSE_FILE,
+    CREATE_FILE,
+    CHECK_FILE,
+    OPEN_FILE,
+    CLOSE_FILE,
     DELETE_FILE,
 
-    CREATE_PATH,      CLOSE_PATH,     OPEN_PATH,     DELETE_PATH,
+    CREATE_PATH,
+    CLOSE_PATH,
+    OPEN_PATH,
+    DELETE_PATH,
     LIST_PATHS,
 
-    CREATE_DATASET,   EXTEND_DATASET, OPEN_DATASET,  DELETE_DATASET,
-    WRITE_DATASET,    READ_DATASET,   LIST_DATASETS, GET_BUFFER_VIEW,
+    CREATE_DATASET,
+    EXTEND_DATASET,
+    OPEN_DATASET,
+    DELETE_DATASET,
+    WRITE_DATASET,
+    READ_DATASET,
+    LIST_DATASETS,
+    GET_BUFFER_VIEW,
 
-    DELETE_ATT,       WRITE_ATT,      READ_ATT,      LIST_ATTS,
+    DELETE_ATT,
+    WRITE_ATT,
+    READ_ATT,
+    LIST_ATTS,
 
     ADVANCE,
     AVAILABLE_CHUNKS, //!< Query chunks that can be loaded in a dataset
-    DEREGISTER //!< Inform the backend that an object has been deleted.
+    DEREGISTER, //!< Inform the backend that an object has been deleted.
+    TOUCH //!< tell the backend that the file is to be considered active
 }; // note: if you change the enum members here, please update
    // docs/source/dev/design.rst
 
@@ -658,6 +674,23 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::DEREGISTER>
     void const *former_parent = nullptr;
 };
 
+template <>
+struct OPENPMDAPI_EXPORT Parameter<Operation::TOUCH> : public AbstractParameter
+{
+    explicit Parameter() = default;
+
+    Parameter(Parameter const &) = default;
+    Parameter(Parameter &&) = default;
+
+    Parameter &operator=(Parameter const &) = default;
+    Parameter &operator=(Parameter &&) = default;
+
+    std::unique_ptr<AbstractParameter> to_heap() && override
+    {
+        return std::make_unique<Parameter<Operation::TOUCH>>(std::move(*this));
+    }
+};
+
 /** @brief Self-contained description of a single IO operation.
  *
  * Contained are

include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

@@ -240,6 +240,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
     void
     deregister(Writable *, Parameter<Operation::DEREGISTER> const &) override;
 
+    void touch(Writable *, Parameter<Operation::TOUCH> const &) override;
+
     std::future<void> flush();
 
 private:

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -895,6 +895,7 @@ void ADIOS2IOHandlerImpl::openFile(
     // lazy opening is deathly in parallel situations
     auto &fileData = getFileData(file, IfFileNotOpen::OpenImplicitly);
     *parameters.out_parsePreference = fileData.parsePreference;
+    m_dirty.emplace(std::move(file));
 }
 
 void ADIOS2IOHandlerImpl::closeFile(
@@ -1482,6 +1483,13 @@ void ADIOS2IOHandlerImpl::deregister(
     m_files.erase(writable);
 }
 
+void ADIOS2IOHandlerImpl::touch(
+    Writable *writable, Parameter<Operation::TOUCH> const &)
+{
+    auto file = refreshFileFromParent(writable, /* preferParentFile = */ false);
+    m_dirty.emplace(std::move(file));
+}
+
 adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode(std::string const &fullPath)
 {
     switch (m_handler->m_backendAccess)

src/IO/AbstractIOHandlerImpl.cpp

@@ -362,6 +362,14 @@ std::future<void> AbstractIOHandlerImpl::flush()
                 deregister(i.writable, parameter);
                 break;
             }
+            case O::TOUCH: {
+                auto &parameter =
+                    deref_dynamic_cast<Parameter<O::TOUCH>>(i.parameter.get());
+                writeToStderr(
+                    "[", i.writable->parent, "->", i.writable, "] DEREGISTER");
+                touch(i.writable, parameter);
+                break;
+            }
             }
         }
         catch (...)

src/IO/HDF5/HDF5IOHandler.cpp

@@ -2901,6 +2901,11 @@ void HDF5IOHandlerImpl::deregister(
     m_fileNames.erase(writable);
 }
 
+void HDF5IOHandlerImpl::touch(Writable *, Parameter<Operation::TOUCH> const &)
+{
+    // no-op
+}
+
 std::optional<HDF5IOHandlerImpl::File>
 HDF5IOHandlerImpl::getFile(Writable *writable)
 {

src/IO/JSON/JSONIOHandlerImpl.cpp

@@ -1039,6 +1039,13 @@ void JSONIOHandlerImpl::deregister(
     m_files.erase(writable);
 }
 
+void JSONIOHandlerImpl::touch(
+    Writable *writable, Parameter<Operation::TOUCH> const &)
+{
+    auto file = refreshFileFromParent(writable);
+    m_dirty.emplace(std::move(file));
+}
+
 auto JSONIOHandlerImpl::getFilehandle(File const &fileName, Access access)
     -> std::tuple<std::unique_ptr<FILEHANDLE>, std::istream *, std::ostream *>
 {

src/Iteration.cpp

@@ -22,6 +22,7 @@
 #include "openPMD/Dataset.hpp"
 #include "openPMD/Datatype.hpp"
 #include "openPMD/IO/AbstractIOHandler.hpp"
+#include "openPMD/IO/IOTask.hpp"
 #include "openPMD/Series.hpp"
 #include "openPMD/auxiliary/DerefDynamicCast.hpp"
 #include "openPMD/auxiliary/Filesystem.hpp"
@@ -306,6 +307,8 @@ void Iteration::flushVariableBased(
 
 void Iteration::flush(internal::FlushParams const &flushParams)
 {
+    Parameter<Operation::TOUCH> touch;
+    IOHandler()->enqueue(IOTask(&writable(), touch));
     if (access::readOnly(IOHandler()->m_frontendAccess))
     {
         for (auto &m : meshes)