Add compressed SpaceTrajectory (#242) and Cereal support

* Initial support for Cereal serialization

* SpaceTrajectory using Cereal

* Document SpaceTrajectory

* Update according to rc83 code review

* Add versioning to Group serialization; doxygen tags

* Doxygen comments

CMakeLists.txt

@@ -12,6 +12,7 @@ set(CMAKE_CXX_EXTENSIONS NO)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(PythonInterp 3 REQUIRED)
 find_package(Threads REQUIRED)
+find_package(zlib REQUIRED)
 enable_testing()
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -115,6 +116,26 @@ if(NOT modernjson_POPULATED)
 endif()
 add_compile_definitions("NLOHMANN_JSON_HPP") # older versions used this macro. Now it's suffixed with "_"
 
+# CEREAL
+FetchContent_Declare(
+    cereal
+    URL "https://github.com/USCiLab/cereal/archive/v1.3.0.tar.gz"
+    URL_HASH MD5=4342e811f245403646c4175258f413f1)
+FetchContent_GetProperties(cereal)
+if(NOT cereal_POPULATED)
+    FetchContent_Populate(cereal)
+endif()
+
+# ZSTR
+FetchContent_Declare(
+    zstr
+    URL "https://github.com/mateidavid/zstr/archive/v1.0.1.tar.gz"
+    URL_HASH MD5=42de51b1c6adac0ec957a24088ef7523)
+FetchContent_GetProperties(zstr)
+if(NOT zstr_POPULATED)
+    FetchContent_Populate(zstr)
+endif()
+
 # RANGE-V3
 FetchContent_Declare(
     rangev3
@@ -277,7 +298,7 @@ endif ()
 # to disable potential compiler warnings
 include_directories(SYSTEM ${eigen_SOURCE_DIR} ${doctest_SOURCE_DIR} ${modernjson_SOURCE_DIR}/include ${rangev3_SOURCE_DIR}/include
     ${Pybind11IncludeDir} ${DocoptIncludeDir} ${CppsidIncludeDir} ${XdrfileIncludeDir} ${SpdlogIncludeDir}
-    ${ProgressTrackerIncludeDir} ${exprtk_SOURCE_DIR})
+    ${ProgressTrackerIncludeDir} ${exprtk_SOURCE_DIR} ${cereal_SOURCE_DIR}/include ${zstr_SOURCE_DIR}/src)
 
 #  Compiler specific flags
 ## GCC

docs/_docs/analysis.md

@@ -493,7 +493,6 @@ generated. For use with rod-like particles on surfaces, the `absz`
 keyword may be used to ensure orientations on only one
 half-sphere.
 
-**Important:**
 Exactly _one inactive_ `molecule` must be added to the simulation using the `inactive`
 keyword when inserting the initial molecules in the [topology](topology).
 
@@ -527,6 +526,25 @@ with the following information:
 - state of random number generator (if `saverandom=true`)
 
 
+### Space Trajectory (experimental)
+
+Save all particle and group information to a compressed, binary trajectory format.
+The following properties are saved:
+
+ - all particle properties (id, position, charge, dipole etc.)
+ - all group properties (id, size, capacity etc.)
+ - todo: geometry, energy
+
+The file suffix must be either `.traj` (uncomressed) or `.ztraj` (compressed).
+For the latter, the file size is reduced by roughly a factor of two using zlib
+compression.
+
+`spacetraj`  | Description
+------------ | ---------------------------------------
+`file`       | Filename of output .traj/.ztraj file
+`nstep`      | Interval between samples.
+
+
 ### XTC trajectory
 
 Generates a Gromacs XTC trajectory file with particle positions and box

docs/schema.yml

@@ -736,6 +736,18 @@ properties:
                     required: [atoms, file, nstep]
                     additionalProperties: false
 
+                spacetraj:
+                    properties:
+                        file:
+                            type: string
+                            pattern: "(.*?)\\.(traj|ztraj)$"
+                            description: "Output filename (.traj/.ztraj)"
+                        nstep: {type: integer}
+                        nskip: {type: integer, default: 0, description: Initial steps to skip}
+                    required: [file, nstep]
+                    additionalProperties: false
+                    type: object
+
                 systemenergy:
                     properties:
                         file: {type: string}

src/CMakeLists.txt

@@ -36,6 +36,7 @@ set_source_files_properties(${objs} PROPERTIES LANGUAGE CXX)
 
 # faunus header files
 set(hdrs
+    ${CMAKE_SOURCE_DIR}/src/aux/eigen_cerealisation.hpp
     ${CMAKE_SOURCE_DIR}/src/aux/eigensupport.h
     ${CMAKE_SOURCE_DIR}/src/aux/iteratorsupport.h
     ${CMAKE_SOURCE_DIR}/src/aux/multimatrix.h
@@ -95,7 +96,7 @@ set_target_properties(functionparser PROPERTIES
 # targer: libfaunus - main functionality of faunus
 add_library(libfaunus STATIC ${objs} ${hdrs})
 target_compile_definitions(libfaunus PRIVATE SPDLOG_COMPILED_LIB)
-target_link_libraries(libfaunus PRIVATE ${CMAKE_THREAD_LIBS_INIT} xdrfile spdlog functionparser)
+target_link_libraries(libfaunus PRIVATE ${CMAKE_THREAD_LIBS_INIT} xdrfile spdlog functionparser ZLIB::ZLIB)
 set_target_properties(libfaunus PROPERTIES
     POSITION_INDEPENDENT_CODE ON)
     #LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

src/analysis.cpp

@@ -5,7 +5,9 @@
 #include "multipole.h"
 #include "aux/iteratorsupport.h"
 #include "aux/eigensupport.h"
-#include "spdlog/spdlog.h"
+#include <spdlog/spdlog.h>
+#include <zstr.hpp>
+#include <cereal/archives/binary.hpp>
 
 #include <iomanip>
 #include <iostream>
@@ -417,6 +419,8 @@ CombinedAnalysis::CombinedAnalysis(const json &j, Space &spc, Energy::Hamiltonia
                             emplace_back<WidomInsertion>(it.value(), spc, pot);
                         else if (it.key() == "xtcfile")
                             emplace_back<XTCtraj>(it.value(), spc);
+                        else if (it.key() == "spacetraj")
+                            emplace_back<SpaceTrajectory>(it.value(), spc.groups);
                         // additional analysis go here...
 
                         if (this->vec.size() == oldsize)
@@ -1435,5 +1439,45 @@ void VirtualTranslate::_to_disk() {
         output_file.flush(); // empty buffer
 }
 
+SpaceTrajectory::SpaceTrajectory(const json &j, Space::Tgvec &groups) : groups(groups) {
+    from_json(j);
+    name = "space trajectory";
+    filename = j.at("file");
+    if (useCompression())
+        stream = std::make_unique<zstr::ofstream>(MPI::prefix + filename, std::ios::binary);
+    else
+        stream = std::make_unique<std::ofstream>(MPI::prefix + filename, std::ios::binary);
+
+    if (stream != nullptr && *stream)
+        archive = std::make_unique<cereal::BinaryOutputArchive>(*stream);
+
+    if (not archive)
+        throw std::runtime_error("error creating "s + filename);
+}
+
+bool SpaceTrajectory::useCompression() const {
+    assert(!filename.empty());
+    std::string suffix = filename.substr(filename.find_last_of(".") + 1);
+    if (suffix == "ztraj")
+        return true;
+    else if (suffix == "traj")
+        return false;
+    else
+        throw std::runtime_error("Trajectory file suffix must be `.traj` or `.ztraj`");
+}
+
+void SpaceTrajectory::_sample() {
+    assert(archive);
+    for (auto &group : groups) {
+        (*archive)(group);
+    }
+}
+
+void SpaceTrajectory::_to_json(json &j) const { j = {{"file", filename}}; }
+
+void SpaceTrajectory::_to_disk() {
+    assert(*stream);
+    stream->flush();
+}
 } // namespace Analysis
 } // namespace Faunus

src/analysis.h

@@ -7,6 +7,10 @@
 #include "auxiliary.h"
 #include <set>
 
+namespace cereal {
+class BinaryOutputArchive;
+}
+
 namespace Faunus {
 
 namespace Energy {
@@ -544,6 +548,34 @@ class QRtraj : public Analysisbase {
     QRtraj(const json &j, Space &spc);
 };
 
+/**
+ * @brief Trajectory with full Space information
+ *
+ * The following are saved in (compressed) binary form:
+ *
+ * - all particle properties (id, position, charge, dipole etc.)
+ * - all group properties (id, size, capacity etc.)
+ *
+ * If zlib compression is enabled the file size
+ * is reduced by roughly a factor of two.
+ *
+ * @todo Geometry information
+ */
+class SpaceTrajectory : public Analysisbase {
+  private:
+    Space::Tgvec &groups; // reference to all groups
+    std::string filename;
+    std::unique_ptr<std::ostream> stream;
+    std::unique_ptr<cereal::BinaryOutputArchive> archive;
+    void _sample() override;
+    void _to_json(json &j) const override;
+    void _to_disk() override;
+    bool useCompression() const; //!< decide from filename if zlib should be used
+
+  public:
+    SpaceTrajectory(const json &, Space::Tgvec &);
+};
+
 struct CombinedAnalysis : public BasePointerVector<Analysisbase> {
     CombinedAnalysis(const json &j, Space &spc, Energy::Hamiltonian &pot);
     void sample();

src/aux/eigen_cerealisation.hpp

@@ -0,0 +1,85 @@
+/*
+ * eos - A 3D Morphable Model fitting library written in modern C++11/14.
+ *
+ * File: include/eos/morphablemodel/io/eigen_cerealisation.hpp
+ *
+ * Copyright 2017 Patrik Huber
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#ifndef EIGENCEREALISATION_HPP_
+#define EIGENCEREALISATION_HPP_
+
+#include "cereal/cereal.hpp"
+
+#include "Eigen/Core"
+
+#include <cstdint>
+
+/**
+ * @brief Serialisation of Eigen matrices for the serialisation
+ * library cereal (http://uscilab.github.io/cereal/index.html).
+ *
+ * Contains serialisation for Eigen matrices to binary archives, i.e. matrices like
+ * \c Eigen::MatrixXf, \c Eigen::Matrix4d, or \c Eigen::Vector3f.
+ *
+ * Todo: Add serialisation to and from JSON. Need to find out how to combine the two
+ * variants of SFINAE that are used.
+ */
+namespace cereal {
+
+/**
+ * @brief Serialise an Eigen::Matrix using cereal.
+ *
+ * Note: Writes the binary data from Matrix::data(), so not sure what happens if a matrix ever has
+ * non-contiguous data (if that can ever happen with Eigen).
+ *
+ * @param[in] ar The archive to serialise to.
+ * @param[in] matrix The matrix to serialise.
+ */
+template <class Archive, class _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols>
+inline typename std::enable_if<traits::is_output_serializable<BinaryData<_Scalar>, Archive>::value, void>::type
+save(Archive &ar, const Eigen::Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> &matrix) {
+    const std::int32_t rows = static_cast<std::int32_t>(matrix.rows());
+    const std::int32_t cols = static_cast<std::int32_t>(matrix.cols());
+    ar(rows);
+    ar(cols);
+    ar(binary_data(matrix.data(), rows * cols * sizeof(_Scalar)));
+}
+
+/**
+ * @brief De-serialise an Eigen::Matrix using cereal.
+ *
+ * Reads the block of binary data back from a cereal archive into the Eigen::Matrix.
+ *
+ * @param[in] ar The archive to deserialise from.
+ * @param[in] matrix The matrix to deserialise into.
+ */
+template <class Archive, class _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols>
+inline typename std::enable_if<traits::is_input_serializable<BinaryData<_Scalar>, Archive>::value, void>::type
+load(Archive &ar, Eigen::Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> &matrix) {
+    std::int32_t rows;
+    std::int32_t cols;
+    ar(rows);
+    ar(cols);
+
+    matrix.resize(rows, cols);
+
+    ar(binary_data(matrix.data(), static_cast<std::size_t>(rows * cols * sizeof(_Scalar))));
+}
+
+} /* namespace cereal */
+
+#endif /* EIGENCEREALISATION_HPP_ */

src/aux/eigensupport.h

@@ -3,6 +3,8 @@
 #include <Eigen/Core>
 #include <nlohmann/json.hpp>
 
+#include "aux/eigen_cerealisation.hpp"
+
 // Eigen<->JSON (de)serialization
 namespace Eigen {
 

src/average.h

@@ -15,6 +15,8 @@ namespace Faunus
       unsigned long long int cnt=0; //!< Number of values
       T sum=0;                      //!< Sum
 
+      template <class Archive> void serialize(Archive &archive) { archive(sum, sqsum, cnt); }
+
       double avg() const {
           return sum / static_cast<double>(cnt);
       } //!< Average