From b6b1992c6f9475ab28994e40061da63170e7a200 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Sat, 18 May 2024 12:03:53 +0100 Subject: [PATCH] Allow a limit to be set on expiry of line segments Accidentally or deliberately moving a node a long way creates a long line segment which can expire large numbers of tiles causing disruption to the rendering process as osm2pgsql either runs out of memory or generates a huge list of tiles to expire. Limiting expiry of long line segments prevents that and also helps reduce disruption of the resulting rendered maps with long lines that are almost certainly incorrect. --- man/osm2pgsql.1 | 3 +++ man/osm2pgsql.md | 3 +++ src/command-line-parser.cpp | 6 ++++++ src/expire-config.hpp | 5 +++++ src/expire-tiles.cpp | 5 +++++ src/flex-lua-table.cpp | 9 +++++++++ src/options.hpp | 3 +++ src/output-flex.cpp | 1 + src/output-pgsql.cpp | 1 + tests/test-expire-from-geometry.cpp | 31 +++++++++++++++++++++++++++++ 10 files changed, 67 insertions(+) diff --git a/man/osm2pgsql.1 b/man/osm2pgsql.1 index 824626e8a..49eaa0fd6 100644 --- a/man/osm2pgsql.1 +++ b/man/osm2pgsql.1 @@ -313,6 +313,9 @@ Create a tile expiry list. -o, --expire-output=FILENAME Output file name for expired tiles list. .TP +--expire-segment-length=SIZE +Max length for a line segment to be expired. +.TP --expire-bbox-size=SIZE Max size for a polygon to expire the whole polygon, not just the boundary. diff --git a/man/osm2pgsql.md b/man/osm2pgsql.md index 872f715ba..8634b4753 100644 --- a/man/osm2pgsql.md +++ b/man/osm2pgsql.md @@ -278,6 +278,9 @@ mandatory for short options too. -o, \--expire-output=FILENAME : Output file name for expired tiles list. +\--expire-segment-length=SIZE +: Max length for a line segment to be expired. + \--expire-bbox-size=SIZE : Max size for a polygon to expire the whole polygon, not just the boundary. diff --git a/src/command-line-parser.cpp b/src/command-line-parser.cpp index ee252a358..3b0beaba8 100644 --- a/src/command-line-parser.cpp +++ b/src/command-line-parser.cpp @@ -460,6 +460,12 @@ options_t parse_command_line(int argc, char *argv[]) // Expire options // ---------------------------------------------------------------------- + // --expire-segment-length + app.add_option("--expire-segment-length", options.expire_tiles_max_segment) + ->description("Max length for a line segment to be expired (default: no limit).") + ->type_name("SIZE") + ->group("Expire options"); + // --expire-bbox-size app.add_option("--expire-bbox-size", options.expire_tiles_max_bbox) ->description("Max size for a polygon to expire the whole polygon, not " diff --git a/src/expire-config.hpp b/src/expire-config.hpp index f2113fc2b..14c3dee06 100644 --- a/src/expire-config.hpp +++ b/src/expire-config.hpp @@ -33,6 +33,11 @@ struct expire_config_t /// Buffer around expired feature as fraction of the tile size. double buffer = 0.1; + /** + * Maximum length of a line segment that will be expired. + */ + double line_segment_limit = 0.0; + /** * Maximum width/heigth of bbox of a (multi)polygon before hybrid mode * expiry switches from full-area to boundary-only expire. diff --git a/src/expire-tiles.cpp b/src/expire-tiles.cpp index 604524b18..5d06eddd2 100644 --- a/src/expire-tiles.cpp +++ b/src/expire-tiles.cpp @@ -156,6 +156,11 @@ void expire_tiles::from_line_segment(geom::point_t const &a, geom::point_t const &b, expire_config_t const &expire_config) { + if (expire_config.line_segment_limit > 0.0 && + distance(a, b) > expire_config.line_segment_limit) { + return; + } + auto tilec_a = coords_to_tile(a); auto tilec_b = coords_to_tile(b); diff --git a/src/flex-lua-table.cpp b/src/flex-lua-table.cpp index 242463d09..6da6ce60b 100644 --- a/src/flex-lua-table.cpp +++ b/src/flex-lua-table.cpp @@ -271,6 +271,15 @@ static void parse_and_set_expire_options(lua_State *lua_state, throw fmt_error("Unknown expire mode '{}'.", mode); } + lua_getfield(lua_state, -1, "line_segment_limit"); + if (lua_isnumber(lua_state, -1)) { + config.line_segment_limit = lua_tonumber(lua_state, -1); + } else if (!lua_isnil(lua_state, -1)) { + throw std::runtime_error{"Optional expire field 'line_segment_limit' " + "must contain a number."}; + } + lua_pop(lua_state, 1); // ""line_segment_limit" + lua_getfield(lua_state, -1, "full_area_limit"); if (lua_isnumber(lua_state, -1)) { if (config.mode != expire_mode::hybrid) { diff --git a/src/options.hpp b/src/options.hpp index c0acd0890..fc36b5950 100644 --- a/src/options.hpp +++ b/src/options.hpp @@ -95,6 +95,9 @@ struct options_t std::shared_ptr projection; ///< SRS of projection + /// Max length of line segment that will be expired + double expire_tiles_max_segment = 0.0; + /// Max bbox size in either dimension to expire full bbox for a polygon double expire_tiles_max_bbox = 20000.0; diff --git a/src/output-flex.cpp b/src/output-flex.cpp index 0c8ccb8cc..32a7590fa 100644 --- a/src/output-flex.cpp +++ b/src/output-flex.cpp @@ -1189,6 +1189,7 @@ output_flex_t::output_flex_t(std::shared_ptr const &mid, if (table.has_geom_column() && table.geom_column().srid() == 3857) { expire_config_t config{}; config.expire_output = m_expire_outputs->size() - 1; + config.line_segment_limit = options.expire_tiles_max_segment; if (options.expire_tiles_max_bbox > 0.0) { config.mode = expire_mode::hybrid; config.full_area_limit = options.expire_tiles_max_bbox; diff --git a/src/output-pgsql.cpp b/src/output-pgsql.cpp index 9f2fc621d..5fc6bbcb3 100644 --- a/src/output-pgsql.cpp +++ b/src/output-pgsql.cpp @@ -438,6 +438,7 @@ output_pgsql_t::output_pgsql_t(std::shared_ptr const &mid, m_buffer(32768, osmium::memory::Buffer::auto_grow::yes), m_rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes) { + m_expire_config.line_segment_limit = get_options()->expire_tiles_max_segment; m_expire_config.full_area_limit = get_options()->expire_tiles_max_bbox; if (get_options()->expire_tiles_max_bbox > 0.0) { m_expire_config.mode = expire_mode::hybrid; diff --git a/tests/test-expire-from-geometry.cpp b/tests/test-expire-from-geometry.cpp index 695d5ee21..6d66aeaea 100644 --- a/tests/test-expire-from-geometry.cpp +++ b/tests/test-expire-from-geometry.cpp @@ -165,6 +165,37 @@ TEST_CASE("expire linestring crossing tile boundary", "[NoDB]") CHECK(tile_t::from_quadkey(tiles[1], zoom) == tile_t{zoom, 2048, 2047}); } +TEST_CASE("expire linestring with long segment", "[NoDB]") +{ + expire_config_t expire_config; + expire_config.line_segment_limit = 10000; + expire_tiles et{zoom, defproj}; + + SECTION("line") + { + geom::linestring_t const line{{5000.0, 5000.0}, {15000.0, 15000.0}}; + et.from_geometry(line, expire_config); + } + + SECTION("geom") + { + geom::linestring_t line{{5000.0, 5000.0}, {15000.0, 15000.0}}; + geom::geometry_t const geom{std::move(line)}; + et.from_geometry(geom, expire_config); + } + + SECTION("geom with check") + { + geom::linestring_t line{{5000.0, 5000.0}, {15000.0, 15000.0}}; + geom::geometry_t geom{std::move(line)}; + geom.set_srid(3857); + et.from_geometry_if_3857(geom, expire_config); + } + + auto const tiles = et.get_tiles(); + REQUIRE(tiles.size() == 0); +} + TEST_CASE("expire small polygon", "[NoDB]") { expire_config_t const expire_config;