Support directly mmap-ing datafiles

CHANGELOG.md

@@ -1,5 +1,8 @@
 # UNRELEASED
   - Changes from 5.19.0:
+    - Features:
+      - ADDED: direct mmapping of datafiles is now supported via the `-mmap` switch. [#5242](https://github.com/Project-OSRM/osrm-backend/pull/5242)
+      - REMOVED: the previous `--memory_file` switch is now deprecated and will fallback to `--mmap` [#5242](https://github.com/Project-OSRM/osrm-backend/pull/5242)
     - Windows:
       - FIXED: Windows builds again. [#5249](https://github.com/Project-OSRM/osrm-backend/pull/5249)
 

docs/nodejs/api.md

@@ -25,7 +25,9 @@ var osrm = new OSRM('network.osrm');
                Make sure you prepared the dataset with the correct toolchain.
     -   `options.shared_memory` **[Boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Boolean)?** Connects to the persistent shared memory datastore.
                This requires you to run `osrm-datastore` prior to creating an `OSRM` object.
-    -   `options.memory_file` **[String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String)?** Path to a file on disk to store the memory using mmap.
+    -   `options.memory_file` **[String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String)?** *DEPRECATED*
+               Old behaviour: Path to a file on disk to store the memory using mmap.  Current behaviour: setting this value is the same as setting `mmap_memory: true`.
+    -   `options.mmap_memory` **[Boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Boolean)?** Map on-disk files to virtual memory addresses (mmap), rather than loading into RAM.
     -   `options.path` **[String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String)?** The path to the `.osrm` files. This is mutually exclusive with setting {options.shared_memory} to true.
     -   `options.max_locations_trip` **[Number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number)?** Max. locations supported in trip query (default: unlimited).
     -   `options.max_locations_viaroute` **[Number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number)?** Max. locations supported in viaroute query (default: unlimited).

features/lib/osrm_loader.js

@@ -84,7 +84,47 @@ class OSRMDirectLoader extends OSRMBaseLoader {
                 throw new Error(util.format('osrm-routed %s: %s', errorReason(err), err.cmd));
             }
         });
-        callback();
+
+        this.child.readyFunc = (data) => {
+            if (/running and waiting for requests/.test(data)) {
+              this.child.stdout.removeListener('data', this.child.readyFunc);
+              callback();
+            }
+        };
+        this.child.stdout.on('data',this.child.readyFunc);
+    }
+};
+
+class OSRMmmapLoader extends OSRMBaseLoader {
+    constructor (scope) {
+        super(scope);
+    }
+
+    load (inputFile, callback) {
+        this.inputFile = inputFile;
+        this.shutdown(() => {
+            this.launch(callback);
+        });
+    }
+
+    osrmUp (callback) {
+        if (this.osrmIsRunning()) return callback(new Error("osrm-routed already running!"));
+
+        const command_arguments = util.format('%s -p %d -i %s -a %s --mmap', this.inputFile, this.scope.OSRM_PORT, this.scope.OSRM_IP, this.scope.ROUTING_ALGORITHM);
+        this.child = this.scope.runBin('osrm-routed', command_arguments, this.scope.environment, (err) => {
+            if (err && err.signal !== 'SIGINT') {
+                this.child = null;
+                throw new Error(util.format('osrm-routed %s: %s', errorReason(err), err.cmd));
+            }
+        });
+
+        this.child.readyFunc = (data) => {
+          if (/running and waiting for requests/.test(data)) {
+            this.child.stdout.removeListener('data', this.child.readyFunc);
+            callback();
+          }
+        };
+        this.child.stdout.on('data',this.child.readyFunc);
     }
 };
 
@@ -135,22 +175,32 @@ class OSRMLoader {
         this.scope = scope;
         this.sharedLoader = new OSRMDatastoreLoader(this.scope);
         this.directLoader = new OSRMDirectLoader(this.scope);
+        this.mmapLoader = new OSRMmmapLoader(this.scope);
         this.method = scope.DEFAULT_LOAD_METHOD;
     }
 
     load (inputFile, callback) {
+        if (!this.loader) {
+            this.loader = {shutdown: (cb) => cb() };
+        }
         if (this.method === 'datastore') {
-            this.directLoader.shutdown((err) => {
+            this.loader.shutdown((err) => {
               if (err) return callback(err);
               this.loader = this.sharedLoader;
               this.sharedLoader.load(inputFile, callback);
             });
         } else if (this.method === 'directly') {
-            this.sharedLoader.shutdown((err) => {
+            this.loader.shutdown((err) => {
               if (err) return callback(err);
               this.loader = this.directLoader;
               this.directLoader.load(inputFile, callback);
             });
+        } else if (this.method === 'mmap') {
+            this.loader.shutdown((err) => {
+              if (err) return callback(err);
+              this.loader = this.mmapLoader;
+              this.mmapLoader.load(inputFile, callback);
+            });
         } else {
             callback(new Error('*** Unknown load method ' + method));
         }

features/support/env.js

@@ -32,7 +32,7 @@ module.exports = function () {
         this.DEFAULT_ENVIRONMENT = Object.assign({STXXLCFG: stxxl_config}, process.env);
         this.DEFAULT_PROFILE = 'bicycle';
         this.DEFAULT_INPUT_FORMAT = 'osm';
-        this.DEFAULT_LOAD_METHOD = 'datastore';
+        this.DEFAULT_LOAD_METHOD = process.argv[process.argv.indexOf('-m') +1].match('mmap') ? 'mmap' : 'datastore';
         this.DEFAULT_ORIGIN = [1,1];
         this.OSM_USER = 'osrm';
         this.OSM_UID = 1;

include/engine/datafacade/mmap_memory_allocator.hpp

@@ -10,6 +10,7 @@
 #include <boost/iostreams/device/mapped_file.hpp>
 
 #include <memory>
+#include <string>
 
 namespace osrm
 {
@@ -24,17 +25,16 @@ namespace datafacade
 class MMapMemoryAllocator : public ContiguousBlockAllocator
 {
   public:
-    explicit MMapMemoryAllocator(const storage::StorageConfig &config,
-                                 const boost::filesystem::path &memory_file);
+    explicit MMapMemoryAllocator(const storage::StorageConfig &config);
     ~MMapMemoryAllocator() override final;
 
     // interface to give access to the datafacades
     const storage::SharedDataIndex &GetIndex() override final;
 
   private:
     storage::SharedDataIndex index;
-    util::vector_view<char> mapped_memory;
-    boost::iostreams::mapped_file mapped_memory_file;
+    std::vector<boost::iostreams::mapped_file> mapped_memory_files;
+    std::string rtree_filename;
 };
 
 } // namespace datafacade

include/engine/datafacade_provider.hpp

@@ -32,9 +32,8 @@ class ExternalProvider final : public DataFacadeProvider<AlgorithmT, FacadeT>
   public:
     using Facade = typename DataFacadeProvider<AlgorithmT, FacadeT>::Facade;
 
-    ExternalProvider(const storage::StorageConfig &config,
-                     const boost::filesystem::path &memory_file)
-        : facade_factory(std::make_shared<datafacade::MMapMemoryAllocator>(config, memory_file))
+    ExternalProvider(const storage::StorageConfig &config)
+        : facade_factory(std::make_shared<datafacade::MMapMemoryAllocator>(config))
     {
     }
 

include/engine/engine.hpp

@@ -63,12 +63,16 @@ template <typename Algorithm> class Engine final : public EngineInterface
                                 << "\" with algorithm " << routing_algorithms::name<Algorithm>();
             facade_provider = std::make_unique<WatchingProvider<Algorithm>>(config.dataset_name);
         }
-        else if (!config.memory_file.empty())
+        else if (!config.memory_file.empty() || config.use_mmap)
         {
-            util::Log(logDEBUG) << "Using memory mapped filed at " << config.memory_file
-                                << " with algorithm " << routing_algorithms::name<Algorithm>();
-            facade_provider = std::make_unique<ExternalProvider<Algorithm>>(config.storage_config,
-                                                                            config.memory_file);
+            if (!config.memory_file.empty())
+            {
+                util::Log(logWARNING)
+                    << "The 'memory_file' option is DEPRECATED - using direct mmaping instead";
+            }
+            util::Log(logDEBUG) << "Using direct memory mapping with algorithm "
+                                << routing_algorithms::name<Algorithm>();
+            facade_provider = std::make_unique<ExternalProvider<Algorithm>>(config.storage_config);
         }
         else
         {

include/engine/engine_config.hpp

@@ -89,6 +89,7 @@ struct EngineConfig final
     int max_alternatives = 3; // set an arbitrary upper bound; can be adjusted by user
     bool use_shared_memory = true;
     boost::filesystem::path memory_file;
+    bool use_mmap = true;
     Algorithm algorithm = Algorithm::CH;
     std::string verbosity;
     std::string dataset_name;

include/nodejs/node_osrm_support.hpp

@@ -142,6 +142,10 @@ inline engine_config_ptr argumentsToEngineConfig(const Nan::FunctionCallbackInfo
     if (shared_memory.IsEmpty())
         return engine_config_ptr();
 
+    auto mmap_memory = params->Get(Nan::New("mmap_memory").ToLocalChecked());
+    if (mmap_memory.IsEmpty())
+        return engine_config_ptr();
+
     if (!memory_file->IsUndefined())
     {
         if (path->IsUndefined())
@@ -190,6 +194,18 @@ inline engine_config_ptr argumentsToEngineConfig(const Nan::FunctionCallbackInfo
             return engine_config_ptr();
         }
     }
+    if (!mmap_memory->IsUndefined())
+    {
+        if (mmap_memory->IsBoolean())
+        {
+            engine_config->use_mmap = Nan::To<bool>(mmap_memory).FromJust();
+        }
+        else
+        {
+            Nan::ThrowError("mmap_memory option must be a boolean");
+            return engine_config_ptr();
+        }
+    }
 
     if (path->IsUndefined() && !engine_config->use_shared_memory)
     {

include/storage/block.hpp

@@ -16,10 +16,15 @@ struct Block
 {
     std::uint64_t num_entries;
     std::uint64_t byte_size;
+    std::uint64_t offset;
 
-    Block() : num_entries(0), byte_size(0) {}
+    Block() : num_entries(0), byte_size(0), offset(0) {}
+    Block(std::uint64_t num_entries, std::uint64_t byte_size, std::uint64_t offset)
+        : num_entries(num_entries), byte_size(byte_size), offset(offset)
+    {
+    }
     Block(std::uint64_t num_entries, std::uint64_t byte_size)
-        : num_entries(num_entries), byte_size(byte_size)
+        : num_entries(num_entries), byte_size(byte_size), offset(0)
     {
     }
 };
@@ -29,7 +34,7 @@ using NamedBlock = std::tuple<std::string, Block>;
 template <typename T> Block make_block(uint64_t num_entries)
 {
     static_assert(sizeof(T) % alignof(T) == 0, "aligned T* can't be used as an array pointer");
-    return Block{num_entries, sizeof(T) * num_entries};
+    return Block{num_entries, sizeof(T) * num_entries, 0};
 }
 }
 }

include/storage/serialization.hpp

@@ -9,6 +9,7 @@
 #include "storage/shared_datatype.hpp"
 #include "storage/tar.hpp"
 
+#include <boost/assert.hpp>
 #include <boost/function_output_iterator.hpp>
 #include <boost/iterator/function_input_iterator.hpp>
 
@@ -30,22 +31,37 @@ namespace serialization
 namespace detail
 {
 template <typename T, typename BlockT = unsigned char>
-inline BlockT packBits(const T &data, std::size_t index, std::size_t count)
+inline BlockT packBits(const T &data, std::size_t base_index, const std::size_t count)
 {
     static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
+    static_assert(std::is_unsigned<BlockT>::value, "BlockT must be unsigned type");
+    static_assert(std::is_integral<BlockT>::value, "BlockT must be an integral type");
+    static_assert(CHAR_BIT == 8, "Non-8-bit bytes not supported, sorry!");
+    BOOST_ASSERT(sizeof(BlockT) * CHAR_BIT >= count);
+
+    // Note: if this packing is changed, be sure to update vector_view<bool>
+    //       as well, so that on-disk and in-memory layouts match.
     BlockT value = 0;
-    for (std::size_t bit = 0; bit < count; ++bit, ++index)
-        value = (value << 1) | data[index];
+    for (std::size_t bit = 0; bit < count; ++bit)
+    {
+        value |= (data[base_index + bit] ? BlockT{1} : BlockT{0}) << bit;
+    }
     return value;
 }
 
 template <typename T, typename BlockT = unsigned char>
-inline void unpackBits(T &data, std::size_t index, std::size_t count, BlockT value)
+inline void
+unpackBits(T &data, const std::size_t base_index, const std::size_t count, const BlockT value)
 {
     static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
-    const BlockT mask = BlockT{1} << (count - 1);
-    for (std::size_t bit = 0; bit < count; value <<= 1, ++bit, ++index)
-        data[index] = value & mask;
+    static_assert(std::is_unsigned<BlockT>::value, "BlockT must be unsigned type");
+    static_assert(std::is_integral<BlockT>::value, "BlockT must be an integral type");
+    static_assert(CHAR_BIT == 8, "Non-8-bit bytes not supported, sorry!");
+    BOOST_ASSERT(sizeof(BlockT) * CHAR_BIT >= count);
+    for (std::size_t bit = 0; bit < count; ++bit)
+    {
+        data[base_index + bit] = value & (BlockT{1} << bit);
+    }
 }
 
 template <typename VectorT>
@@ -55,15 +71,16 @@ void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &d
     data.resize(count);
     std::uint64_t index = 0;
 
-    constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
+    using BlockType = std::uint64_t;
+    constexpr std::uint64_t BLOCK_BITS = CHAR_BIT * sizeof(BlockType);
 
-    const auto decode = [&](const std::uint64_t block) {
-        auto read_size = std::min<std::size_t>(count - index, WORD_BITS);
-        unpackBits<VectorT, std::uint64_t>(data, index, read_size, block);
-        index += WORD_BITS;
+    const auto decode = [&](const BlockType block) {
+        auto read_size = std::min<std::size_t>(count - index, BLOCK_BITS);
+        unpackBits<VectorT, BlockType>(data, index, read_size, block);
+        index += BLOCK_BITS;
     };
 
-    reader.ReadStreaming<std::uint64_t>(name, boost::make_function_output_iterator(decode));
+    reader.ReadStreaming<BlockType>(name, boost::make_function_output_iterator(decode));
 }
 
 template <typename VectorT>
@@ -73,19 +90,20 @@ void writeBoolVector(tar::FileWriter &writer, const std::string &name, const Vec
     writer.WriteElementCount64(name, count);
     std::uint64_t index = 0;
 
-    constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
+    using BlockType = std::uint64_t;
+    constexpr std::uint64_t BLOCK_BITS = CHAR_BIT * sizeof(BlockType);
 
     // FIXME on old boost version the function_input_iterator does not work with lambdas
     // so we need to wrap it in a function here.
-    const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
-        auto write_size = std::min<std::size_t>(count - index, WORD_BITS);
-        auto packed = packBits<VectorT, std::uint64_t>(data, index, write_size);
-        index += WORD_BITS;
+    const std::function<BlockType()> encode_function = [&]() -> BlockType {
+        auto write_size = std::min<std::size_t>(count - index, BLOCK_BITS);
+        auto packed = packBits<VectorT, BlockType>(data, index, write_size);
+        index += BLOCK_BITS;
         return packed;
     };
 
-    std::uint64_t number_of_blocks = (count + WORD_BITS - 1) / WORD_BITS;
-    writer.WriteStreaming<std::uint64_t>(
+    std::uint64_t number_of_blocks = (count + BLOCK_BITS - 1) / BLOCK_BITS;
+    writer.WriteStreaming<BlockType>(
         name,
         boost::make_function_input_iterator(encode_function, boost::infinite()),
         number_of_blocks);
@@ -266,9 +284,9 @@ template <typename K, typename V> void write(io::BufferWriter &writer, const std
     }
 }
 
-inline void read(io::BufferReader &reader, DataLayout &layout) { read(reader, layout.blocks); }
+inline void read(io::BufferReader &reader, BaseDataLayout &layout) { read(reader, layout.blocks); }
 
-inline void write(io::BufferWriter &writer, const DataLayout &layout)
+inline void write(io::BufferWriter &writer, const BaseDataLayout &layout)
 {
     write(writer, layout.blocks);
 }