Merge pull request #389 from favreau/master

Added MorphIO as an optional dependency to load morphologies from files

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "platform/deps/LASTools"]
 	path = platform/deps/LASTools
 	url = https://github.com/LAStools/LAStools.git
+[submodule "platform/deps/MorphIO"]
+	path = platform/deps/MorphIO
+	url = https://github.com/BlueBrain/MorphIO.git

CMakeLists.txt

@@ -43,14 +43,20 @@ find_package(assimp)
 find_package(LibJpegTurbo)
 find_package(Rockets)
 find_package(CGAL)
-find_package(LASlib)
+find_package(LASlib OPTIONAL_COMPONENTS)
+find_package(MorphIO OPTIONAL_COMPONENTS)
+find_package(HighFive OPTIONAL_COMPONENTS)
 
 set(BIOEXPLORER_SOURCE_DIRS ${PROJECT_SOURCE_DIR} ${CMAKE_BINARY_DIR}/generated)
 
 # ==============================================================================
 # Options
 # ==============================================================================
 
+if(MorphIO_FOUND)
+    message(STATUS "[Core] MorphIO module enabled")
+endif()
+
 # Find the Intel C++ compiler
 find_program(CMAKE_CXX_COMPILER NAMES icpc)
 if(CMAKE_CXX_COMPILER)

bioexplorer/backend/CMakeLists.txt

@@ -24,7 +24,6 @@ option(${NAME}_USE_CGAL "Use CGAL meshing features" ON)
 # Packages
 find_package(OpenMP)
 find_package(PQXX REQUIRED)
-find_package(LASlib)
 
 # Compiler flags
 add_compile_options("-fopenmp")
@@ -62,6 +61,7 @@ set(${NAME}_SOURCES
     science/io/OOCManager.cpp
     science/io/db/DBConnector.cpp
     science/io/cache/MemoryCache.cpp
+    science/io/filesystem/MorphologyLoader.cpp
     science/molecularsystems/EnzymeReaction.cpp
     science/molecularsystems/Molecule.cpp
     science/molecularsystems/Membrane.cpp
@@ -97,6 +97,7 @@ set(${NAME}_PUBLIC_HEADERS
     science/io/OOCManager.h
     science/io/CacheLoader.h
     science/io/cache/MemoryCache.h
+    science/io/filesystem/MorphologyLoader.h
     science/io/db/DBConnector.h
     science/common/Assembly.h
     science/common/Node.h
@@ -150,6 +151,9 @@ set(${NAME}_PUBLIC_HEADERS
     science/BioExplorerPlugin.h
 )
 
+set(${NAME}_PUBLIC_MODULE_LIBRARIES)
+set(${NAME}_PRIVATE_MODULE_LIBRARIES)
+
 if(${CGAL_FOUND} AND NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")
     add_compile_options(-frounding-math)
 endif()
@@ -160,8 +164,11 @@ if(ASSIMP_FOUND)
     list(APPEND ${NAME}_PRIVATE_MODULE_LIBRARIES ${ASSIMP_LIBRARIES})
 endif()
 
-set(${NAME}_PUBLIC_MODULE_LIBRARIES)
-set(${NAME}_PRIVATE_MODULE_LIBRARIES)
+if(MorphIO_FOUND)
+    list(APPEND ${NAME}_PRIVATE_MODULE_LIBRARIES morphio HighFive)
+    include_directories(${HDF5_INCLUDE_DIRS})
+endif()
+
 # ==============================================================================
 # OSPRay module
 # ==============================================================================

bioexplorer/backend/science/BioExplorerPlugin.cpp

@@ -849,6 +849,7 @@ Response BioExplorerPlugin::_setGeneralSettings(const GeneralSettingsDetails &pa
         instance->setLoggingLevel(payload.loggingLevel);
         instance->setDBLoggingLevel(payload.databaseLoggingLevel);
         instance->setV1Compatibility(payload.v1Compatibility);
+        instance->setLoadMorphologiesFromFileSystem(payload.loadMorphologiesFromFileSystem);
 
         MemoryCache::getInstance()->setEnabled(payload.cacheEnabled);
         PLUGIN_INFO(3, "Setting general options for the plugin");

bioexplorer/backend/science/api/Params.cpp

@@ -88,6 +88,7 @@ bool from_json(GeneralSettingsDetails &param, const std::string &payload)
         FROM_JSON(param, js, databaseLoggingLevel);
         FROM_JSON(param, js, v1Compatibility);
         FROM_JSON(param, js, cacheEnabled);
+        FROM_JSON(param, js, loadMorphologiesFromFileSystem);
     }
     catch (...)
     {

bioexplorer/backend/science/common/GeneralSettings.h

@@ -125,6 +125,21 @@ class GeneralSettings
      */
     void setV1Compatibility(const bool value) { _v1Compatibility = value; }
 
+    /**
+     * @brief Get value of the loading morphologies from file system option
+     *
+     * @return true Loading morphologies from file system is enabled
+     * @return false Loading morphologies from file system is disabled
+     */
+    bool getLoadMorphologiesFromFileSystem() const { return _loadMorphologiesFromFilesystem; }
+
+    /**
+     * @brief Set the loading morphologies from file system option
+     *
+     * @param value Enabled is true, disabled otherwise
+     */
+    void setLoadMorphologiesFromFileSystem(const bool value) { _loadMorphologiesFromFilesystem = value; }
+
     static std::mutex _mutex;
     static GeneralSettings* _instance;
 
@@ -136,6 +151,7 @@ class GeneralSettings
     size_t _loggingLevel{1};
     size_t _dbLoggingLevel{0};
     bool _v1Compatibility{false};
+    bool _loadMorphologiesFromFilesystem{false};
 };
 } // namespace common
 } // namespace bioexplorer

bioexplorer/backend/science/common/Types.h

@@ -328,6 +328,9 @@ using VasculaturePtr = std::shared_ptr<Vasculature>;
 
 namespace morphology
 {
+const std::string NEURON_CONFIG_MORPHOLOGY_FOLDER = "alternate_morphologies";
+const std::string NEURON_CONFIG_MORPHOLOGY_FILE_EXTENSION = "morphology_file_extension";
+
 class Morphologies;
 using MorphologiesPtr = std::shared_ptr<Morphologies>;
 
@@ -494,6 +497,12 @@ namespace io
 class OOCManager;
 using OOCManagerPtr = std::shared_ptr<OOCManager>;
 
+namespace filesystem
+{
+class MorphologyLoader;
+using MorphologyLoaderPtr = std::shared_ptr<MorphologyLoader>;
+} // namespace filesystem
+
 namespace db
 {
 class DBConnector;
@@ -549,6 +558,7 @@ typedef struct
     uint32_t databaseLoggingLevel;
     bool v1Compatibility{false};
     bool cacheEnabled{false};
+    bool loadMorphologiesFromFileSystem{false};
 } GeneralSettingsDetails;
 
 typedef struct
@@ -1537,6 +1547,8 @@ enum class NeuronSectionType
     basal_dendrite = 3,
     apical_dendrite = 4
 };
+using NeuronSectionTypes = std::vector<NeuronSectionType>;
+const int64_t SOMA_AS_PARENT = -1;
 
 typedef struct
 {

bioexplorer/backend/science/io/cache/MemoryCache.cpp

@@ -16,21 +16,25 @@
 
 #include "MemoryCache.h"
 
+#include <science/common/GeneralSettings.h>
 #include <science/common/Logs.h>
 #include <science/io/db/DBConnector.h>
+#include <science/io/filesystem/MorphologyLoader.h>
 
 namespace bioexplorer
 {
 namespace io
 {
+using namespace common;
+using namespace morphology;
 using namespace details;
 using namespace db;
+using namespace filesystem;
 
 MemoryCache* MemoryCache::_instance = nullptr;
 std::mutex MemoryCache::_mutex;
 
-const morphology::SectionMap& MemoryCache::getNeuronSections(const DBConnector& connector, const uint64_t neuronId,
-                                                             const NeuronsDetails& details)
+const morphology::SectionMap& MemoryCache::getNeuronSections(const uint64_t neuronId, const NeuronsDetails& details)
 {
     if (_enabled)
     {
@@ -39,7 +43,25 @@ const morphology::SectionMap& MemoryCache::getNeuronSections(const DBConnector&
             return (*it).second;
     }
 
-    _sections[neuronId] = connector.getNeuronSections(details.populationName, neuronId, details.sqlSectionFilter);
+    if (GeneralSettings::getInstance()->getLoadMorphologiesFromFileSystem())
+    {
+        NeuronSectionTypes sectionTypes;
+        if (details.loadAxon)
+            sectionTypes.push_back(NeuronSectionType::axon);
+        if (details.loadApicalDendrites)
+            sectionTypes.push_back(NeuronSectionType::apical_dendrite);
+        if (details.loadBasalDendrites)
+            sectionTypes.push_back(NeuronSectionType::basal_dendrite);
+
+        const auto morphologyLoader = _getMorphologyLoader(details.populationName);
+        const auto path = DBConnector::getInstance().getNeuronMorphologyRelativePath(details.populationName, neuronId);
+        _sections[neuronId] = morphologyLoader->getNeuronSections(path, sectionTypes);
+    }
+    else
+    {
+        _sections[neuronId] =
+            DBConnector::getInstance().getNeuronSections(details.populationName, neuronId, details.sqlSectionFilter);
+    }
     return _sections[neuronId];
 }
 
@@ -50,5 +72,27 @@ void MemoryCache::setEnabled(const bool enabled)
     if (!enabled)
         _sections.clear();
 }
+
+MorphologyLoaderPtr MemoryCache::_getMorphologyLoader(const std::string& populationName)
+{
+    const auto itm = _morphologyLoaders.find(populationName);
+    if (itm != _morphologyLoaders.end())
+        return (*itm).second;
+
+    const auto& connector = DBConnector::getInstance();
+    const auto configurationValues = connector.getNeuronConfiguration(populationName);
+
+    const auto itc = configurationValues.find(NEURON_CONFIG_MORPHOLOGY_FOLDER);
+    if (itc == configurationValues.end())
+        PLUGIN_THROW(NEURON_CONFIG_MORPHOLOGY_FOLDER + " is not defined in the configuration table");
+
+    const auto itf = configurationValues.find(NEURON_CONFIG_MORPHOLOGY_FILE_EXTENSION);
+    if (itf == configurationValues.end())
+        PLUGIN_THROW(NEURON_CONFIG_MORPHOLOGY_FILE_EXTENSION + " is not defined in the configuration table");
+
+    _morphologyLoaders[populationName] =
+        std::shared_ptr<MorphologyLoader>(new MorphologyLoader((*itc).second, (*itf).second));
+    return _morphologyLoaders[populationName];
+}
 } // namespace io
 } // namespace bioexplorer

bioexplorer/backend/science/io/cache/MemoryCache.h

@@ -56,17 +56,18 @@ class MemoryCache
      *
      * @return Sections
      */
-    const morphology::SectionMap& getNeuronSections(const db::DBConnector& connector, const uint64_t neuronId,
-                                                    const details::NeuronsDetails& details);
+    const morphology::SectionMap& getNeuronSections(const uint64_t neuronId, const details::NeuronsDetails& details);
 
     static std::mutex _mutex;
     static MemoryCache* _instance;
 
 private:
     ~MemoryCache() {}
+    io::filesystem::MorphologyLoaderPtr _getMorphologyLoader(const std::string& populationName);
 
     bool _enabled{false};
     std::map<uint64_t, morphology::SectionMap> _sections;
+    std::map<std::string, io::filesystem::MorphologyLoaderPtr> _morphologyLoaders;
 };
 } // namespace io
 } // namespace bioexplorer

bioexplorer/backend/science/io/db/DBConnector.cpp

@@ -640,6 +640,60 @@ TriangleMesh DBConnector::getAstrocyteMicroDomain(const std::string& populationN
     return mesh;
 }
 
+StringMap DBConnector::getNeuronConfiguration(const std::string& populationName) const
+{
+    CHECK_DB_INITIALIZATION
+
+    StringMap values;
+    pqxx::nontransaction transaction(*_connections[omp_get_thread_num() % _dbNbConnections]);
+    try
+    {
+        Timer chrono;
+        std::string sql = "SELECT guid, value FROM " + populationName + ".configuration";
+
+        PLUGIN_DB_INFO(1, sql);
+        auto res = transaction.exec(sql);
+        for (auto c = res.begin(); c != res.end(); ++c)
+            values[c[0].as<std::string>()] = c[1].as<std::string>();
+        PLUGIN_DB_TIMER(chrono.elapsed(), "getNeuronConfiguration(populationName=" << populationName << ")");
+    }
+    catch (const pqxx::sql_error& e)
+    {
+        PLUGIN_THROW(e.what());
+    }
+
+    return values;
+}
+
+std::string DBConnector::getNeuronMorphologyRelativePath(const std::string& populationName,
+                                                         const uint64_t neuronId) const
+{
+    CHECK_DB_INITIALIZATION
+
+    std::string relativePath;
+    pqxx::nontransaction transaction(*_connections[omp_get_thread_num() % _dbNbConnections]);
+    try
+    {
+        Timer chrono;
+        const std::string sql = "SELECT code FROM " + populationName +
+                                ".morphology WHERE guid=(SELECT morphology_guid FROM " + populationName +
+                                ".node WHERE guid=" + std::to_string(neuronId) + ")";
+
+        PLUGIN_DB_INFO(1, sql);
+        auto res = transaction.exec(sql);
+        if (res.empty())
+            PLUGIN_THROW("No relative path defined for the neuron " + std::to_string(neuronId));
+        for (auto c = res.begin(); c != res.end(); ++c)
+            relativePath = c[0].as<std::string>();
+        PLUGIN_DB_TIMER(chrono.elapsed(), "getNeuronConfiguration(populationName=" << populationName << ")");
+    }
+    catch (const pqxx::sql_error& e)
+    {
+        PLUGIN_THROW(e.what());
+    }
+    return relativePath;
+}
+
 uint64_t DBConnector::getNumberOfNeurons(const std::string& populationName, const std::string& sqlCondition) const
 {
     CHECK_DB_INITIALIZATION
@@ -704,8 +758,8 @@ NeuronSomaMap DBConnector::getNeurons(const std::string& populationName, const s
             soma.morphologyId = c[10].as<uint64_t>();
             somas[c[0].as<uint64_t>()] = soma;
         }
-        PLUGIN_DB_TIMER(chrono.elapsed(), "getNeuronSections(populationName=" << populationName << ", sqlCondition="
-                                                                              << sqlCondition << ")");
+        PLUGIN_DB_TIMER(chrono.elapsed(),
+                        "getNeurons(populationName=" << populationName << ", sqlCondition=" << sqlCondition << ")");
     }
     catch (const pqxx::sql_error& e)
     {

bioexplorer/backend/science/io/db/DBConnector.h

@@ -230,6 +230,14 @@ class DBConnector
      */
     core::TriangleMesh getAstrocyteMicroDomain(const std::string& populationName, const uint64_t astrocyteId) const;
 
+    /**
+     * @brief Get the Neuron Circuit Configuration details
+     *
+     * @param populationName Name of the population
+     * @return StringMap Map of key value strings
+     */
+    StringMap getNeuronConfiguration(const std::string& populationName) const;
+
     /**
      * @brief Get the number of neurons for a given population and a specific filter
      *
@@ -250,6 +258,15 @@ class DBConnector
      */
     morphology::NeuronSomaMap getNeurons(const std::string& populationName, const std::string& sqlCondition = "") const;
 
+    /**
+     * @brief Get the relative path to the morphology file for a given neuron Id
+     *
+     * @param populationName Name of the population
+     * @param neuronId Identifier of the neuron
+     * @return std::string Relative path to the morphology file
+     */
+    std::string getNeuronMorphologyRelativePath(const std::string& populationName, const uint64_t neuronId) const;
+
     /**
      * @brief Get the sections of a given neuron
      *