Merge branch 'release/2024.4.1' into attribute-validities

CMakeLists.txt

@@ -4,7 +4,7 @@ cmake_policy(SET CMP0117 NEW)
 
 project(mapget CXX)
 
-set(MAPGET_VERSION 2024.4.0)
+set(MAPGET_VERSION 2024.4.1)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)

deps.cmake

@@ -34,7 +34,7 @@ else()
 
   FetchContent_Declare(fmt
     GIT_REPOSITORY "https://github.com/fmtlib/fmt.git"
-    GIT_TAG        "10.0.0"
+    GIT_TAG        "11.0.2"
     GIT_SHALLOW    ON)
   FetchContent_MakeAvailable(fmt)
 

libs/http-service/src/cli.cpp

@@ -340,9 +340,9 @@ struct FetchCommand
         request->onSourceDataLayer(fn);
         cli.request(request)->wait();
 
-        if (request->getStatus() == NoDataSource)
+        if (request->getStatus() == RequestStatus::NoDataSource)
             raise("Failed to fetch sources: no matching data source.");
-        if (request->getStatus() == Aborted)
+        if (request->getStatus() == RequestStatus::Aborted)
             raise("Failed to fetch sources: request aborted.");
     }
 };

libs/http-service/src/http-service.cpp

@@ -162,6 +162,7 @@ struct HttpService::Impl
         std::mutex mutex_;
         std::condition_variable resultEvent_;
 
+        uint64_t requestId_;
         std::stringstream buffer_;
         std::string responseType_;
         std::unique_ptr<TileLayerStream::Writer> writer_;
@@ -170,9 +171,11 @@ struct HttpService::Impl
 
         HttpTilesRequestState()
         {
+            static std::atomic_uint64_t nextRequestId;
             writer_ = std::make_unique<TileLayerStream::Writer>(
                 [&, this](auto&& msg, auto&& msgType) { buffer_ << msg; },
                 stringOffsets_);
+            requestId_ = nextRequestId++;
         }
 
         void parseRequestFromJson(nlohmann::json const& requestJson)
@@ -217,6 +220,32 @@ struct HttpService::Impl
         }
     };
 
+    mutable std::mutex clientRequestMapMutex_;
+    mutable std::map<std::string, std::shared_ptr<HttpTilesRequestState>> requestStatePerClientId_;
+
+    void abortRequestsForClientId(std::string clientId, std::shared_ptr<HttpTilesRequestState> newState = nullptr) const
+    {
+        std::unique_lock clientRequestMapAccess(clientRequestMapMutex_);
+        auto clientRequestIt = requestStatePerClientId_.find(clientId);
+        if (clientRequestIt != requestStatePerClientId_.end()) {
+            // Ensure that any previous requests from the same clientId
+            // are finished post-haste!
+            bool anySoftAbort = false;
+            for (auto const& req : clientRequestIt->second->requests_) {
+                if (!req->isDone()) {
+                    self_.abort(req);
+                    anySoftAbort = true;
+                }
+            }
+            if (anySoftAbort)
+                log().warn("Soft-aborting tiles request {}", clientRequestIt->second->requestId_);
+            requestStatePerClientId_.erase(clientRequestIt);
+        }
+        if (newState) {
+            requestStatePerClientId_.emplace(clientId, newState);
+        }
+    }
+
     /**
      * Wraps around the generic mapget service's request() function
      * to include httplib request decoding and response encoding.
@@ -231,6 +260,7 @@ struct HttpService::Impl
         // Within one HTTP request, all requested tiles from the same map+layer
         // combination should be in a single LayerTilesRequest.
         auto state = std::make_shared<HttpTilesRequestState>();
+        log().info("Processing tiles request {}", state->requestId_);
         for (auto& requestJson : requestsJson) {
             state->parseRequestFromJson(requestJson);
         }
@@ -260,16 +290,22 @@ struct HttpService::Impl
             // Send a status report detailing for each request
             // whether its data source is unavailable or it was aborted.
             res.status = 400;
-            std::vector<int> requestStatuses{};
+            std::vector<std::underlying_type_t<RequestStatus>> requestStatuses{};
             for (const auto& r : state->requests_) {
-                requestStatuses.push_back(r->getStatus());
+                requestStatuses.push_back(static_cast<std::underlying_type_t<RequestStatus>>(r->getStatus()));
             }
             res.set_content(
                 nlohmann::json::object({{"requestStatuses", requestStatuses}}).dump(),
                 "application/json");
             return;
         }
 
+        // Parse/Process clientId.
+        if (j.contains("clientId")) {
+            auto clientId = j["clientId"].get<std::string>();
+            abortRequestsForClientId(clientId, state);
+        }
+
         // For efficiency, set up httplib to stream tile layer responses to client:
         // (1) Lambda continuously supplies response data to httplib's DataSink,
         //     picking up data from state->buffer_ until all tile requests are done.
@@ -319,16 +355,35 @@ struct HttpService::Impl
             // cleanup callback to abort the requests.
             [state, this](bool success)
             {
-                log().debug("Request finished, success: {}", success);
                 if (!success) {
+                    log().warn("Aborting tiles request {}", state->requestId_);
                     for (auto& request : state->requests_) {
                         self_.abort(request);
                     }
                 }
+                else {
+                    log().info("Tiles request {} was successful.", state->requestId_);
+                }
             });
     }
 
-    void handleSourcesRequest(const httplib::Request&, httplib::Response& res)
+    void handleAbortRequest(const httplib::Request& req, httplib::Response& res) const
+    {
+        // Parse the JSON request.
+        nlohmann::json j = nlohmann::json::parse(req.body);
+        auto requestsJson = j["requests"];
+
+        if (j.contains("clientId")) {
+            auto const clientId = j["clientId"].get<std::string>();
+            abortRequestsForClientId(clientId);
+        }
+        else {
+            res.status = 400;
+            res.set_content("Missing clientId", "text/plain");
+        }
+    }
+
+    void handleSourcesRequest(const httplib::Request&, httplib::Response& res) const
     {
         auto sourcesInfo = nlohmann::json::array();
         for (auto& source : self_.info()) {
@@ -337,7 +392,7 @@ struct HttpService::Impl
         res.set_content(sourcesInfo.dump(), "application/json");
     }
 
-    void handleStatusRequest(const httplib::Request&, httplib::Response& res)
+    void handleStatusRequest(const httplib::Request&, httplib::Response& res) const
     {
         auto serviceStats = self_.getStatistics();
         auto cacheStats = self_.cache()->getStatistics();
@@ -358,7 +413,7 @@ struct HttpService::Impl
         res.set_content(oss.str(), "text/html");
     }
 
-    void handleLocateRequest(const httplib::Request& req, httplib::Response& res)
+    void handleLocateRequest(const httplib::Request& req, httplib::Response& res) const
     {
         // Parse the JSON request.
         nlohmann::json j = nlohmann::json::parse(req.body);
@@ -578,6 +633,11 @@ void HttpService::setup(httplib::Server& server)
         [&](const httplib::Request& req, httplib::Response& res)
         { impl_->handleTilesRequest(req, res); });
 
+    server.Post(
+        "/abort",
+        [&](const httplib::Request& req, httplib::Response& res)
+        { impl_->handleAbortRequest(req, res); });
+
     server.Get(
         "/sources",
         [this](const httplib::Request& req, httplib::Response& res)

libs/model/src/featurelayer.cpp

@@ -861,9 +861,16 @@ std::vector<IdPart> const& TileFeatureLayer::getPrimaryIdComposition(const std::
 
 void TileFeatureLayer::setStrings(std::shared_ptr<simfil::StringPool> const& newDict)
 {
+    auto oldDict = strings();
+    // Reset simfil environment and clear expression cache
+    impl_->expressionCache_.reset(makeEnvironment(newDict));
+    ModelPool::setStrings(newDict);
+    if (!oldDict || *newDict == *oldDict)
+        return;
+
     // Re-map old string IDs to new string IDs
     for (auto& attr : impl_->attributes_) {
-        if (auto resolvedName = strings()->resolve(attr.name_)) {
+        if (auto resolvedName = oldDict->resolve(attr.name_)) {
             attr.name_ = newDict->emplace(*resolvedName);
         }
     }
@@ -873,20 +880,15 @@ void TileFeatureLayer::setStrings(std::shared_ptr<simfil::StringPool> const& new
         }
     }
     for (auto& fid : impl_->featureIds_) {
-        if (auto resolvedName = strings()->resolve(fid.typeId_)) {
+        if (auto resolvedName = oldDict->resolve(fid.typeId_)) {
             fid.typeId_ = newDict->emplace(*resolvedName);
         }
     }
     for (auto& rel : impl_->relations_) {
-        if (auto resolvedName = strings()->resolve(rel.name_)) {
+        if (auto resolvedName = oldDict->resolve(rel.name_)) {
             rel.name_ = newDict->emplace(*resolvedName);
         }
     }
-
-    // Reset simfil environment and clear expression cache
-    impl_->expressionCache_.reset(makeEnvironment(newDict));
-
-    ModelPool::setStrings(newDict);
 }
 
 simfil::ModelNode::Ptr TileFeatureLayer::clone(

libs/service/include/mapget/service/service.h

@@ -13,7 +13,7 @@
 namespace mapget
 {
 
-enum RequestStatus {
+enum class RequestStatus {
     Open = 0x0,
     Success = 0x1, /** The request has been fully satisfied. */
     NoDataSource = 0x2, /** No data source could provide the requested map + layer. */
@@ -97,7 +97,7 @@ class LayerTilesRequest
     // Mutex/condition variable for reading/setting request status.
     std::mutex statusMutex_;
     std::condition_variable statusConditionVariable_;
-    RequestStatus status_ = Open;
+    RequestStatus status_ = RequestStatus::Open;
 };
 
 /**

libs/service/src/datasource.cpp

@@ -1,6 +1,7 @@
 #include "datasource.h"
 #include <memory>
 #include <stdexcept>
+#include <chrono>
 #include "mapget/model/sourcedatalayer.h"
 #include "mapget/model/info.h"
 
@@ -14,30 +15,41 @@ TileLayer::Ptr DataSource::get(const MapTileKey& k, Cache::Ptr& cache, DataSourc
         throw std::runtime_error("Layer info is null");
 
     auto result = TileLayer::Ptr{};
+
+    auto start = std::chrono::steady_clock::now();
     switch (layerInfo->type_) {
     case mapget::LayerType::Features: {
-        auto result = std::make_shared<TileFeatureLayer>(
+        auto tileFeatureLayer = std::make_shared<TileFeatureLayer>(
             k.tileId_,
             info.nodeId_,
             info.mapId_,
             info.getLayer(k.layerId_),
             cache->getStringPool(info.nodeId_));
-        fill(result);
-        return result;
+        fill(tileFeatureLayer);
+        result = tileFeatureLayer;
+        break;
     }
     case mapget::LayerType::SourceData: {
-        auto result = std::make_shared<TileSourceDataLayer>(
+        auto tileSourceDataLayer = std::make_shared<TileSourceDataLayer>(
             k.tileId_,
             info.nodeId_,
             info.mapId_,
             info.getLayer(k.layerId_),
             cache->getStringPool(info.nodeId_));
-        fill(result);
-        return result;
+        fill(tileSourceDataLayer);
+        result = tileSourceDataLayer;
+        break;
     }
     default:
-        return nullptr;
+        break;
+    }
+
+    // Notify the tile how long it took to fill.
+    if (result) {
+        auto duration = std::chrono::steady_clock::now() - start;
+        result->setInfo("fill-time-ms", std::chrono::duration_cast<std::chrono::milliseconds>(duration).count());
     }
+    return result;
 }
 
 simfil::StringId DataSource::cachedStringPoolOffset(const std::string& nodeId, Cache::Ptr const& cache)

libs/service/src/service.cpp

@@ -420,7 +420,11 @@ struct Service::Impl : public Service::Controller
     {
         std::unique_lock lock(jobsMutex_);
         // Remove the request from the list of requests.
-        requests_.remove_if([r](auto&& request) { return r == request; });
+        auto numRemoved = requests_.remove_if([r](auto&& request) { return r == request; });
+        // Clear its jobs to mark it as done.
+        if (numRemoved) {
+            r->setStatus(RequestStatus::Aborted);
+        }
     }
 
     std::vector<DataSourceInfo> getDataSourceInfos()