From 37276371453f208369067bc1f4bb9e4045bc3bbe Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Mon, 11 Jan 2016 12:03:01 +0100 Subject: [PATCH 1/6] Build osmium_convert in order to be able to eventually replace osmosis This provides a script to build the libosmium examples such as osmium_convert (to convert from xml to pbf). I initially tried a CMake ExternalProject setup, but this was more complicated than I initially thought; this is the more elegant solution. The goal is to eventually rip out osmosis, so that we no longer depend on java for the cucumber tests. References: - https://github.com/Project-OSRM/osrm-backend/issues/1788 --- .gitignore | 1 + scripts/build_osmium_tools.sh | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100755 scripts/build_osmium_tools.sh diff --git a/.gitignore b/.gitignore index c94aeb42819..e6a125f9e62 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ Thumbs.db ####################### /build/ /cmake/postinst +/third_party/libosmium/tmp/ # Eclipse related files # ######################### diff --git a/scripts/build_osmium_tools.sh b/scripts/build_osmium_tools.sh new file mode 100755 index 00000000000..81777ec397f --- /dev/null +++ b/scripts/build_osmium_tools.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Builds command line tools shipped with libosmium for example osmium_convert +# CMake build directory is build/osmium; binaries are located under build/osmium/examples + + +# e: exit on first error, x: print commands +set -ex + +BUILD_DIR=build/osmium + +cmake -E remove_directory $BUILD_DIR +cmake -E make_directory $BUILD_DIR +cmake -E chdir $BUILD_DIR cmake ../../third_party/libosmium -DCMAKE_BUILD_TYPE=Release -DBUILD_EXAMPLES=1 -DBUILD_TESTING=0 +cmake -E chdir $BUILD_DIR cmake --build . From 43dcdb21ec191b4c60795daec1101e94ec9ca1d2 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Wed, 13 Jan 2016 14:34:36 +0100 Subject: [PATCH 2/6] Revert "Build osmium_convert in order to be able to eventually replace osmosis" This reverts commit 344588114dd5076385b35e0fbaef966de681f95f. --- .gitignore | 1 - scripts/build_osmium_tools.sh | 15 --------------- 2 files changed, 16 deletions(-) delete mode 100755 scripts/build_osmium_tools.sh diff --git a/.gitignore b/.gitignore index e6a125f9e62..c94aeb42819 100644 --- a/.gitignore +++ b/.gitignore @@ -41,7 +41,6 @@ Thumbs.db ####################### /build/ /cmake/postinst -/third_party/libosmium/tmp/ # Eclipse related files # ######################### diff --git a/scripts/build_osmium_tools.sh b/scripts/build_osmium_tools.sh deleted file mode 100755 index 81777ec397f..00000000000 --- a/scripts/build_osmium_tools.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -# Builds command line tools shipped with libosmium for example osmium_convert -# CMake build directory is build/osmium; binaries are located under build/osmium/examples - - -# e: exit on first error, x: print commands -set -ex - -BUILD_DIR=build/osmium - -cmake -E remove_directory $BUILD_DIR -cmake -E make_directory $BUILD_DIR -cmake -E chdir $BUILD_DIR cmake ../../third_party/libosmium -DCMAKE_BUILD_TYPE=Release -DBUILD_EXAMPLES=1 -DBUILD_TESTING=0 -cmake -E chdir $BUILD_DIR cmake --build . From 9959fd26bb6e7c4ef05ffcc308c897c8ad4b8682 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Wed, 13 Jan 2016 14:41:55 +0100 Subject: [PATCH 3/6] Do not convert osm xml files to pbf for the cucumber tests --- features/support/data.rb | 18 +----------------- features/support/exceptions.rb | 6 ------ 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/features/support/data.rb b/features/support/data.rb index 62ed7d370ca..eabd934bd4d 100644 --- a/features/support/data.rb +++ b/features/support/data.rb @@ -229,17 +229,6 @@ def write_osm end end -def convert_osm_to_pbf - unless File.exist?("#{osm_file}.osm.pbf") - log_preprocess_info - log "== Converting #{osm_file}.osm to protobuffer format...", :preprocess - unless system "osmosis --read-xml #{osm_file}.osm --write-pbf #{osm_file}.osm.pbf omitmetadata=true >>#{PREPROCESS_LOG_FILE} 2>&1" - raise OsmosisError.new $?, "osmosis exited with code #{$?.exitstatus}" - end - log '', :preprocess - end -end - def extracted? Dir.chdir TEST_FOLDER do File.exist?("#{extracted_file}.osrm") && @@ -258,15 +247,10 @@ def write_timestamp File.open( "#{prepared_file}.osrm.timestamp", 'w') {|f| f.write(OSM_TIMESTAMP) } end -def pbf? - input_format=='pbf' -end - def write_input_data Dir.chdir TEST_FOLDER do write_osm write_timestamp - convert_osm_to_pbf if pbf? end end @@ -274,7 +258,7 @@ def extract_data Dir.chdir TEST_FOLDER do log_preprocess_info log "== Extracting #{osm_file}.osm...", :preprocess - unless system "#{BIN_PATH}/osrm-extract #{osm_file}.osm#{'.pbf' if pbf?} #{@extract_args} --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1" + unless system "#{BIN_PATH}/osrm-extract #{osm_file}.osm #{@extract_args} --profile #{PROFILES_PATH}/#{@profile}.lua >>#{PREPROCESS_LOG_FILE} 2>&1" log "*** Exited with code #{$?.exitstatus}.", :preprocess raise ExtractError.new $?.exitstatus, "osrm-extract exited with code #{$?.exitstatus}." end diff --git a/features/support/exceptions.rb b/features/support/exceptions.rb index 02f59d388a2..d04dc385c18 100644 --- a/features/support/exceptions.rb +++ b/features/support/exceptions.rb @@ -37,12 +37,6 @@ def initialize code, msg end end -class OsmosisError < OSRMError - def initialize code, msg - super 'osmosis', code, msg, PREPROCESS_LOG_FILE, 40 - end -end - class ExtractError < OSRMError def initialize code, msg super 'osrm-extract', code, msg, PREPROCESS_LOG_FILE, 3 From 03faf2665bcedc5514422f587041965d9a9cca2d Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Wed, 13 Jan 2016 14:43:14 +0100 Subject: [PATCH 4/6] No longer install osmosis on travis --- .travis.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5d8e5fa8a82..d5a186008d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -117,12 +117,8 @@ install: mkdir cmake && travis_retry wget --quiet -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C cmake export PATH=${DEPS_DIR}/cmake/bin:${PATH} - OSMOSIS_URL="http://bretth.dev.openstreetmap.org/osmosis-build/osmosis-latest.tgz" - mkdir osmosis && travis_retry wget --quiet -O - ${OSMOSIS_URL} | tar -xz -C osmosis - export PATH=${DEPS_DIR}/osmosis/bin:${PATH} - elif [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then - brew install cmake boost libzip libstxxl libxml2 lua51 luabind tbb GDAL osmosis + brew install cmake boost libzip libstxxl libxml2 lua51 luabind tbb GDAL fi before_script: From 7c8626d1448ef28bdd62568304a33a202e1c7eae Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Wed, 13 Jan 2016 14:44:29 +0100 Subject: [PATCH 5/6] No longer install osmosis in docker env --- docker/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4535949a1bb..aa37682f573 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,8 +12,6 @@ RUN pip install awscli # luabind RUN curl https://gist.githubusercontent.com/DennisOSRM/f2eb7b948e6fe1ae319e/raw/install-luabind.sh | sudo bash -# osmosis -RUN curl -s https://gist.githubusercontent.com/DennisOSRM/803a64a9178ec375069f/raw/ | sudo bash RUN useradd -ms /bin/bash mapbox USER mapbox From 1157cb9b9514f1cba47ccf974df8a5ab4051b669 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 13 Jan 2016 17:57:03 +0100 Subject: [PATCH 6/6] Remove obsolete protobuf test --- features/testbot/protobuffer.feature | 156 --------------------------- 1 file changed, 156 deletions(-) delete mode 100644 features/testbot/protobuffer.feature diff --git a/features/testbot/protobuffer.feature b/features/testbot/protobuffer.feature deleted file mode 100644 index e32de26c25b..00000000000 --- a/features/testbot/protobuffer.feature +++ /dev/null @@ -1,156 +0,0 @@ -@routing @pbf @testbot -Feature: Importing protobuffer (.pbf) format -# Test normally read .osm, which is faster than .pbf files, -# since we don't need to use osmosis to first convert to .pbf -# The scenarios in this file test the ability to import .pbf files, -# including nodes, way, restictions, and a various special situations. - - Background: - Given the profile "testbot" - And the import format "pbf" - - Scenario: Testbot - Protobuffer import, nodes and ways - Given the node map - | | | | d | - | a | b | c | | - | | | | e | - - And the ways - | nodes | highway | oneway | - | abc | primary | | - | cd | primary | yes | - | ce | river | | - | de | primary | | - - When I route I should get - | from | to | route | - | d | c | de,ce | - | e | d | de | - - - Scenario: Testbot - Protobuffer import, turn restiction relations - Given the node map - | | n | | - | w | j | e | - | | s | | - - And the ways - | nodes | oneway | - | sj | yes | - | nj | -1 | - | wj | -1 | - | ej | -1 | - - And the relations - | type | way:from | way:to | node:via | restriction | - | restriction | sj | wj | j | no_left_turn | - - When I route I should get - | from | to | route | - | s | w | | - | s | n | sj,nj | - | s | e | sj,ej | - - - Scenario: Testbot - Protobuffer import, distances at longitude 45 - Given the node locations - | node | lat | lon | - | a | 80 | 45 | - | b | 0 | 45 | - - And the ways - | nodes | - | ab | - - When I route I should get - | from | to | route | distance | - | a | b | ab | 8905559m ~0.1% | - - Scenario: Testbot - Protobuffer import, distances at longitude 80 - Given the node locations - | node | lat | lon | - | a | 80 | 80 | - | b | 0 | 80 | - - And the ways - | nodes | - | ab | - - When I route I should get - | from | to | route | distance | - | a | b | ab | 8905559m ~0.1% | - - Scenario: Testbot - Protobuffer import, empty dataset - Given the node map - | | - - Given the ways - | nodes | - - When the data has been prepared - Then "osrm-extract" should return code 1 - - - Scenario: Testbot - Protobuffer import, streetnames with UTF characters - Given the node map - | a | b | c | d | - - And the ways - | nodes | name | - | ab | Scandinavian København | - | bc | Japanese 東京 | - | cd | Cyrillic Москва | - - When I route I should get - | from | to | route | - | a | b | Scandinavian København | - | b | c | Japanese 東京 | - | c | d | Cyrillic Москва | - - Scenario: Testbot - Protobuffer import, bearing af 45 degree intervals - Given the node map - | b | a | h | - | c | x | g | - | d | e | f | - - And the ways - | nodes | - | xa | - | xb | - | xc | - | xd | - | xe | - | xf | - | xg | - | xh | - - When I route I should get - | from | to | route | compass | bearing | - | x | a | xa | N | 0 | - | x | b | xb | NW | 315 | - | x | c | xc | W | 270 | - | x | d | xd | SW | 225 | - | x | e | xe | S | 180 | - | x | f | xf | SE | 135 | - | x | g | xg | E | 90 | - | x | h | xh | NE | 45 | - - - Scenario: Testbot - Protobuffer import, rraffic signals should incur a delay - Given the node map - | a | b | c | - | d | e | f | - - And the nodes - | node | highway | - | e | traffic_signals | - - And the ways - | nodes | - | abc | - | def | - - When I route I should get - | from | to | route | time | distance | - | a | c | abc | 20s +-1 | 200m +-1 | - | d | f | def | 27s +-1 | 200m +-1 |