Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use boost asio to run threadpool #197

Merged
merged 10 commits into from
Feb 27, 2021
Merged

Use boost asio to run threadpool #197

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
96b75d5
Individual shape and osm store and small optimization
kleunen Feb 21, 2021
37ac730
Use boost asio thread pool to schedule tile generation
kleunen Feb 21, 2021
f7d9853
CI use boost 1.67
kleunen Feb 21, 2021
6304130
Fix bufferstream reading
kleunen Feb 21, 2021
225031d
Fix tile counter
kleunen Feb 22, 2021
919f680
Fix mmap size message to give correct value
kleunen Feb 22, 2021
8a2e3e4
Fix initialization of unordered_map
kleunen Feb 23, 2021
664cd22
Allow initialization of store to prevent rehashing and small optimiza…
kleunen Feb 23, 2021
b00d660
Use intrusive pointer for reference counting and small memory optimiz…
kleunen Feb 25, 2021
43dc1a2
Attribute store sets of attributes
kleunen Feb 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,35 @@ jobs:

steps:
- uses: actions/checkout@v2

- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends build-essential liblua5.1-0 liblua5.1-0-dev libprotobuf-dev libsqlite3-dev protobuf-compiler shapelib libshp-dev libboost-program-options-dev libboost-filesystem-dev libboost-system-dev
sudo apt-get install -y --no-install-recommends build-essential liblua5.1-0 liblua5.1-0-dev libprotobuf-dev libsqlite3-dev protobuf-compiler shapelib libshp-dev

- name: Build and install
env:
S_CFLAGS: -flto -ffunction-sections -fdata-sections
S_CXXFLAGS: -flto -ffunction-sections -fdata-sections
S_LDFLAGS: -Wl,-gc-sections
run: |
make -j 2
mkdir build
cd build
export CFLAGS=${S_CFLAGS} && export CXXFLAGS=${S_CXXFLAGS} && export LDFLAGS=${S_LDFLAGS}
BOOST_ROOT=$BOOST_ROOT_1_72_0 cmake -DTILEMAKER_USE_STATIC_BOOST=true ..
make -j
sudo make install

- name: Build openmaptiles-compatible mbtiles files of Liechtenstein
run: |
curl http://download.geofabrik.de/europe/${AREA}-latest.osm.pbf -o ${AREA}.osm.pbf
tilemaker ${AREA}.osm.pbf --config=resources/config-openmaptiles.json --process=resources/process-openmaptiles.lua --output=${AREA}.mbtiles --verbose

- name: 'Upload compiled executable'
uses: actions/upload-artifact@v2
with:
name: tilemaker
path: build/tilemaker

Github-Action:
name: Generate mbtiles with Github Action
runs-on: ubuntu-latest
Expand Down
22 changes: 18 additions & 4 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
cmake_minimum_required(VERSION 3.9)
cmake_minimum_required(VERSION 2.8)

if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Release")
endif()

project(tilemaker)

OPTION(TILEMAKER_USE_STATIC_BOOST "Statically link with boost libraries" OFF)

list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")

if("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}")
Expand All @@ -15,7 +17,19 @@ endif()
include_directories(include)
include_directories(${CMAKE_BINARY_DIR}) # for generated files

find_package(Boost 1.56 REQUIRED COMPONENTS system filesystem program_options)
IF (TILEMAKER_USE_STATIC_BOOST)
MESSAGE (STATUS "Staticly linking with Boost")
SET (Boost_USE_STATIC_LIBS TRUE)
ELSE ()
MESSAGE (STATUS "Dynamically linking with Boost")
SET (Boost_USE_STATIC_LIBS FALSE)
ENDIF ()

IF ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
LINK_DIRECTORIES(${Boost_LIBRARY_DIRS})
ENDIF ()

find_package(Boost 1.66 REQUIRED COMPONENTS system filesystem program_options)
include_directories(SYSTEM ${Boost_INCLUDE_DIRS})

find_package(Protobuf REQUIRED)
Expand All @@ -34,7 +48,6 @@ find_package(ZLIB REQUIRED)
include_directories(${ZLIB_INCLUDE_DIR})

set(CMAKE_CXX_STANDARD 11)
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)

if(MSVC)
add_definitions(-D_USE_MATH_DEFINES)
Expand All @@ -56,6 +69,7 @@ file(GLOB tilemaker_src_files
"clipper/*.cpp"
)
add_executable(tilemaker vector_tile.pb.cc osmformat.pb.cc ${tilemaker_src_files})
target_link_libraries(tilemaker ${Boost_LIBRARIES} ${PROTOBUF_LIBRARY} ${LIBSHP_LIBRARIES} ${SQLITE3_LIBRARIES} ${LUA_LIBRARIES} ${ZLIB_LIBRARY} ${THREAD_LIB})
target_link_libraries(tilemaker ${PROTOBUF_LIBRARY} ${LIBSHP_LIBRARIES} ${SQLITE3_LIBRARIES} ${LUA_LIBRARIES} ${ZLIB_LIBRARY} ${THREAD_LIB} ${CMAKE_DL_LIBS}
Boost::system Boost::filesystem Boost::program_options)

install(TARGETS tilemaker RUNTIME DESTINATION bin)
1 change: 1 addition & 0 deletions include/geomtypes.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ namespace boost { namespace geometry { namespace traits {
return p.second;
}
};

}}}

BOOST_GEOMETRY_REGISTER_LINESTRING(mmap::linestring_t)
Expand Down
78 changes: 47 additions & 31 deletions include/osm_store.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,16 +245,22 @@ class OSMStore

using point_allocator_t = boost::interprocess::node_allocator<Point, mmap_file_t::segment_manager>;
using point_store_t = std::deque<Point, point_allocator_t>;
point_store_t *generated_points_store;

using linestring_t = mmap::linestring_t;
using linestring_allocator_t = boost::interprocess::node_allocator<linestring_t, mmap_file_t::segment_manager>;
using linestring_store_t = std::deque<linestring_t, scoped_alloc_t<linestring_allocator_t>>;
linestring_store_t *generated_linestring_store;

using multi_polygon_store_t = std::deque<mmap::multi_polygon_t,
scoped_alloc_t<boost::interprocess::node_allocator<mmap::multi_polygon_t, mmap_file_t::segment_manager>>>;
multi_polygon_store_t *generated_multi_polygon_store;

struct generated {
point_store_t *points_store;
linestring_store_t *linestring_store;
multi_polygon_store_t *multi_polygon_store;
};

generated osm_generated;
generated shp_generated;

std::string osm_store_filename;
constexpr static std::size_t init_map_size = 64000000;
Expand All @@ -280,12 +286,19 @@ class OSMStore
ways.reopen(mmap_file);
relations.reopen(mmap_file);

generated_points_store = mmap_file.find_or_construct<point_store_t>
("point_store")(mmap_file.get_segment_manager());
generated_linestring_store = mmap_file.find_or_construct<linestring_store_t>
("linestring_store")(mmap_file.get_segment_manager());
generated_multi_polygon_store = mmap_file.find_or_construct<multi_polygon_store_t>
("multi_polygon_store")(mmap_file.get_segment_manager());
osm_generated.points_store = mmap_file.find_or_construct<point_store_t>
("osm_point_store")(mmap_file.get_segment_manager());
osm_generated.linestring_store = mmap_file.find_or_construct<linestring_store_t>
("osm_linestring_store")(mmap_file.get_segment_manager());
osm_generated.multi_polygon_store = mmap_file.find_or_construct<multi_polygon_store_t>
("osm_multi_polygon_store")(mmap_file.get_segment_manager());

shp_generated.points_store = mmap_file.find_or_construct<point_store_t>
("shp_point_store")(mmap_file.get_segment_manager());
shp_generated.linestring_store = mmap_file.find_or_construct<linestring_store_t>
("shp_linestring_store")(mmap_file.get_segment_manager());
shp_generated.multi_polygon_store = mmap_file.find_or_construct<multi_polygon_store_t>
("shp_multi_polygon_store")(mmap_file.get_segment_manager());
}

template<typename Func>
Expand All @@ -296,7 +309,7 @@ class OSMStore
return;
} catch(boost::interprocess::bad_alloc &e) {
map_size = map_size * 2;
std::cout << "Resizing osm store to size: " << map_size << std::endl;
std::cout << "Resizing osm store to size: " << (map_size / 1000000) << "M " << std::endl;
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The actual filesize of the osm_store.dat file appears to be twice the reported value - maybe change 1000000 to 500000?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, i noticed this too. I am not sure why this is, I do give this value to boost interprocess grow method. If interprocess allocates 2 * size I specify, i would rather ask for size / 2 there. Because otherwise it would be confusing.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://github.com/systemed/tilemaker/blob/master/include/osm_store.h#L297-L307

Copy link
Contributor Author

@kleunen kleunen Feb 22, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, wait. It grows with extra bytes, not to the specified size.

So I should first call grow and then multiply size * 2

   //!Tries to resize mapped file so that we have room for
   //!more objects.
   //!
   //!This function is not synchronized so no other thread or process should
   //!be reading or writing the file
   static bool grow(const char *filename, size_type extra_bytes)
   {
      return base_t::template grow
         <basic_managed_mapped_file>(filename, extra_bytes);
   }


mmap_file = mmap_file_t();

Expand Down Expand Up @@ -352,37 +365,44 @@ class OSMStore
relations.insert_front(i, outerWayVec.begin(), outerWayVec.end(), innerWayVec.begin(), innerWayVec.end());
}

std::size_t getMemorySize() const { return map_size; }

using handle_t = mmap_file_t::handle_t;

generated &osm() { return osm_generated; }
generated &shp() { return shp_generated; }

template<class T>
T const &retrieve(handle_t handle) const {
return *(static_cast<T const *>(mmap_file.get_address_from_handle(handle)));
}

template<typename T>
std::size_t store_point(T const &input) {
handle_t store_point(generated &store, T const &input) {
perform_mmap_operation([&]() {
generated_points_store->emplace_back(input.x(), input.y());
store.points_store->emplace_back(input.x(), input.y());
});
return generated_points_store->size() - 1;
}

Point const &retrieve_point(std::size_t index) const {
return generated_points_store->at(index);
return mmap_file.get_handle_from_address(&store.points_store->back());
}


template<typename Input>
std::size_t store_linestring(Input const &src)
handle_t store_linestring(generated &store, Input const &src)
{
perform_mmap_operation([&]() {
generated_linestring_store->resize(generated_linestring_store->size() + 1);
boost::geometry::assign(generated_linestring_store->back(), src);
store.linestring_store->resize(store.linestring_store->size() + 1);
boost::geometry::assign(store.linestring_store->back(), src);
});
return generated_linestring_store->size() - 1;
}

mmap::linestring_t retrieve_linestring(std::size_t index) const {
return generated_linestring_store->at(index);
return mmap_file.get_handle_from_address(&store.linestring_store->back());
}

template<typename Input>
std::size_t store_multi_polygon(Input const &src)
handle_t store_multi_polygon(generated &store, Input const &src)
{
perform_mmap_operation([&]() {
mmap::multi_polygon_t result(generated_multi_polygon_store->get_allocator());
mmap::multi_polygon_t result(store.multi_polygon_store->get_allocator());
result.reserve(src.size());

for(auto const &polygon: src) {
Expand All @@ -402,14 +422,10 @@ class OSMStore
result.emplace_back(outer, inners);
}

generated_multi_polygon_store->push_back(result);
store.multi_polygon_store->push_back(result);
});

return generated_multi_polygon_store->size() - 1;
}

mmap::multi_polygon_t const &retrieve_multi_polygon(std::size_t index) const {
return generated_multi_polygon_store->at(index);
return mmap_file.get_handle_from_address(&store.multi_polygon_store->back());
}

// @brief Make the store empty
Expand Down
47 changes: 20 additions & 27 deletions include/output_object.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,10 @@ class OutputObject {
virtual Geometry buildWayGeometry(OSMStore &osmStore, const TileBbox &bbox) const = 0;

///\brief Add a node geometry
virtual LatpLon buildNodeGeometry(OSMStore &osmStore, const TileBbox &bbox) const = 0;
virtual LatpLon buildNodeGeometry(OSMStore &osmStore, const TileBbox &bbox) const
{
throw std::runtime_error("Geometry type is not point");
}

///\brief Check if the object intersects with the given point
virtual bool intersects(OSMStore &osmStore, Point const &p) const = 0;
Expand All @@ -92,15 +95,15 @@ class OutputObject {
class OutputObjectOsmStorePoint : public OutputObject
{
public:
OutputObjectOsmStorePoint(OutputGeometryType type, uint_least8_t l, NodeID id, std::size_t index)
: OutputObject(type, false, l, id), index(index)
OutputObjectOsmStorePoint(OutputGeometryType type, uint_least8_t l, NodeID id, OSMStore::handle_t handle)
: OutputObject(type, false, l, id), handle(handle)
{
assert(type == POINT || type == CENTROID || type == CACHED_POINT);
}

Geometry buildWayGeometry(OSMStore &osmStore, const TileBbox &bbox) const
{
auto &p = osmStore.retrieve_point(index);
auto &p = osmStore.retrieve<mmap::point_t>(handle);
if (geom::within(p, bbox.clippingBox)) {
return p;
}
Expand All @@ -109,7 +112,7 @@ class OutputObjectOsmStorePoint : public OutputObject

LatpLon buildNodeGeometry(OSMStore &osmStore, const TileBbox &bbox) const
{
auto const &pt = osmStore.retrieve_point(index);
auto const &pt = osmStore.retrieve<mmap::point_t>(handle);
LatpLon out;
out.latp = pt.y();
out.lon = pt.x();
Expand All @@ -118,55 +121,50 @@ class OutputObjectOsmStorePoint : public OutputObject

bool intersects(OSMStore &osmStore, Point const &p) const
{
return boost::geometry::intersects(osmStore.retrieve_point(index), p);
return boost::geometry::intersects(osmStore.retrieve<mmap::point_t>(handle), p);
}

private:
std::size_t index;
OSMStore::handle_t handle;
};

class OutputObjectOsmStoreLinestring : public OutputObject
{
public:
OutputObjectOsmStoreLinestring(OutputGeometryType type, uint_least8_t l, NodeID id, std::size_t index)
: OutputObject(type, false, l, id), index(index)
OutputObjectOsmStoreLinestring(OutputGeometryType type, uint_least8_t l, NodeID id, OSMStore::handle_t handle)
: OutputObject(type, false, l, id), handle(handle)
{
assert(type == LINESTRING || type == CACHED_LINESTRING);
}

Geometry buildWayGeometry(OSMStore &osmStore, const TileBbox &bbox) const
{
MultiLinestring out;
geom::intersection(osmStore.retrieve_linestring(index), bbox.clippingBox, out);
geom::intersection(osmStore.retrieve<mmap::linestring_t>(handle), bbox.clippingBox, out);
return out;
}

LatpLon buildNodeGeometry(OSMStore &osmStore, const TileBbox &bbox) const
{
throw std::runtime_error("Geometry type is not point");
}

bool intersects(OSMStore &osmStore, Point const &p) const
{
return boost::geometry::intersects(osmStore.retrieve_linestring(index), p);
return boost::geometry::intersects(osmStore.retrieve<mmap::linestring_t>(handle), p);
}

private:
std::size_t index;
OSMStore::handle_t handle;
};

class OutputObjectOsmStoreMultiPolygon : public OutputObject
{
public:
OutputObjectOsmStoreMultiPolygon(OutputGeometryType type, uint_least8_t l, NodeID id, std::size_t index)
: OutputObject(type, false, l, id), index(index)
OutputObjectOsmStoreMultiPolygon(OutputGeometryType type, uint_least8_t l, NodeID id, OSMStore::handle_t handle)
: OutputObject(type, false, l, id), handle(handle)
{
assert(type == POLYGON || type == CACHED_POLYGON);
}

Geometry buildWayGeometry(OSMStore &osmStore, const TileBbox &bbox) const
{
auto mp = osmStore.retrieve_multi_polygon(index);
auto mp = osmStore.retrieve<mmap::multi_polygon_t>(handle);

Polygon clippingPolygon;

Expand All @@ -188,18 +186,13 @@ class OutputObjectOsmStoreMultiPolygon : public OutputObject
}
}

LatpLon buildNodeGeometry(OSMStore &osmStore, const TileBbox &bbox) const
{
throw std::runtime_error("Geometry type is not point");
}

bool intersects(OSMStore &osmStore, Point const &p) const
{
return boost::geometry::intersects(osmStore.retrieve_multi_polygon(index), p);
return boost::geometry::intersects(osmStore.retrieve<mmap::multi_polygon_t>(handle), p);
}

private:
std::size_t index;
OSMStore::handle_t handle;
};

typedef std::shared_ptr<OutputObject> OutputObjectRef;
Expand Down
6 changes: 3 additions & 3 deletions include/pbf_blocks.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@
------------------- */

// Read and parse a protobuf message
void readMessage(google::protobuf::Message *message, std::fstream *input, unsigned int size);
void readMessage(google::protobuf::Message *message, std::istream &input, unsigned int size);

// Read an osm.pbf sequence of header length -> BlobHeader -> Blob
// and parse the unzipped contents into a message
void readBlock(google::protobuf::Message *messagePtr, std::fstream *inputPtr);
void readBlock(google::protobuf::Message *messagePtr, std::istream &input);

void writeBlock(google::protobuf::Message *messagePtr, std::fstream *outputPtr, std::string headerType);
void writeBlock(google::protobuf::Message *messagePtr, std::ostream &output, std::string headerType);
/* -------------------
Tag handling
------------------- */
Expand Down
2 changes: 1 addition & 1 deletion include/read_pbf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ class PbfReader
PbfReader();
virtual ~PbfReader();

int ReadPbfFile(const std::string &inputFile, std::unordered_set<std::string> &nodeKeys);
int ReadPbfFile(std::istream &inputFile, std::unordered_set<std::string> &nodeKeys);

///Pointer to output object. Loaded objects are sent here.
PbfReaderOutput * output;
Expand Down
Loading