Closed kgrimsby closed 3 years ago
aoles
commented
3 years ago Thanks for reporting this! We have observed similar failures with the recent releases and we are currently investigating the issue. However, so far we had no luck in triggering the problem on OSM datasets smaller than planet, so we really appreciate your feedback. Would you mind providing the contents of your app.config file, this would help us reproducing and debugging the problem. Cheers!
kgrimsby
commented
3 years ago @aoles I've also experienced another error, seems a bit random which error I get. The other I can't recall completely, but something about "-1 index is out of bounds of length xx".
I've attached my app.config.
{
"ors": {
"info": {
"base_url": "https://openrouteservice.org/",
"support_mail": "support@openrouteservice.org",
"author_tag": "openrouteservice",
"content_licence": "LGPL 3.0"
},
"services": {
"matrix": {
"enabled": true,
"maximum_routes": 20000,
"maximum_routes_flexible": 25,
"maximum_search_radius": 5000,
"maximum_visited_nodes": 10000000,
"allow_resolve_locations": true,
"attribution": "openrouteservice.org, OpenStreetMap contributors"
},
"isochrones": {
"enabled": true,
"maximum_range_distance": [
{
"profiles": "any",
"value": 50000
},
{
"profiles": "driving-car, driving-hgv",
"value": 10000000
}
],
"maximum_range_time": [
{
"profiles": "any",
"value": 18000
},
{
"profiles": "driving-car, driving-hgv",
"value": 3600
}
],
"fastisochrones": {
"maximum_range_distance": [
{
"profiles": "any",
"value": 50000
},
{
"profiles": "driving-car, driving-hgv",
"value": 500000
}
],
"maximum_range_time": [
{
"profiles": "any",
"value": 18000
},
{
"profiles": "driving-car, driving-hgv",
"value": 10800
}
],
"profiles": {
"default_params": {
"enabled": false,
"threads": 12,
"weightings": "recommended",
"maxcellnodes": 5000
},
"profile-vehicles-hgv": {
"enabled": true,
"threads": 12,
"weightings": "recommended, shortest",
"maxcellnodes": 5000
}
}
},
"maximum_intervals": 10,
"maximum_locations": 2,
"allow_compute_area": true
},
"routing": {
"enabled": true,
"mode": "normal",
"routing_description": "This is a routing file from openrouteservice",
"routing_name": "openrouteservice routing",
"sources": [
"data/data.osm.pbf"
],
"init_threads": 8,
"attribution": "openrouteservice.org, OpenStreetMap contributors",
"elevation_preprocessed": false,
"profiles": {
"active": [
"car"
],
"default_params": {
"encoder_flags_size": 8,
"graphs_root_path": "data/graphs",
"elevation_provider": "multi",
"elevation_cache_path": "data/elevation_cache",
"elevation_cache_clear": true,
"instructions": true,
"maximum_distance": 10000000,
"maximum_distance_dynamic_weights": 10000000,
"maximum_distance_avoid_areas": 10000000,
"maximum_waypoints": 200,
"maximum_snapping_radius": 400,
"maximum_avoid_polygon_area": 200000000,
"maximum_avoid_polygon_extent": 20000,
"maximum_distance_alternative_routes": 10000000,
"maximum_alternative_routes": 3,
"maximum_distance_round_trip_routes": 10000000,
"maximum_speed_lower_bound": 80,
"preparation": {
"min_network_size": 200,
"min_one_way_network_size": 200,
"methods": {
"lm": {
"enabled": true,
"threads": 8,
"weightings": "recommended,shortest",
"landmarks": 16
}
}
},
"execution": {
"methods": {
"lm": {
"disabling_allowed": true,
"active_landmarks": 8
}
}
}
},
"profile-car": {
"profiles": "driving-car",
"parameters": {
"encoder_flags_size": 8,
"encoder_options": "turn_costs=true|block_fords=false|use_acceleration=false",
"maximum_distance": 10000000,
"elevation": true,
"maximum_snapping_radius": 350,
"preparation": {
"min_network_size": 200,
"min_one_way_network_size": 200,
"methods": {
"ch": {
"enabled": true,
"threads": 4,
"weightings": "fastest"
},
"lm": {
"enabled": false,
"threads": 4,
"weightings": "fastest,shortest",
"landmarks": 16
},
"core": {
"enabled": true,
"threads": 4,
"weightings": "fastest,shortest",
"landmarks": 64,
"lmsets": "highways;allow_all"
}
}
},
"execution": {
"methods": {
"ch": {
"disabling_allowed": true
},
"lm": {
"disabling_allowed": true,
"active_landmarks": 6
},
"core": {
"disabling_allowed": true,
"active_landmarks": 6
}
}
},
"ext_storages": {
"WayCategory": {},
"HeavyVehicle": {},
"WaySurfaceType": {},
"RoadAccessRestrictions": {
"use_for_warnings": true
}
}
}
},
"profile-hgv": {
"profiles": "driving-hgv",
"parameters": {
"encoder_flags_size": 8,
"encoder_options": "turn_costs=true|block_fords=false|use_acceleration=false",
"maximum_distance": 10000000,
"elevation": true,
"preparation": {
"min_network_size": 200,
"min_one_way_network_size": 200,
"methods": {
"ch": {
"enabled": true,
"threads": 1,
"weightings": "recommended"
},
"lm": {
"enabled": true,
"threads": 1,
"weightings": "recommended,shortest",
"landmarks": 16
},
"core": {
"enabled": true,
"threads": 1,
"weightings": "recommended,shortest",
"landmarks": 64,
"lmsets": "highways;allow_all"
}
}
},
"execution": {
"methods": {
"ch": {
"disabling_allowed": true
},
"lm": {
"disabling_allowed": true,
"active_landmarks": 6
},
"core": {
"disabling_allowed": true,
"active_landmarks": 6
}
}
},
"ext_storages": {
"WayCategory": {},
"HeavyVehicle": {
"restrictions": true
},
"WaySurfaceType": {}
}
}
},
"profile-bike-regular": {
"profiles": "cycling-regular",
"parameters": {
"encoder_options": "consider_elevation=true|turn_costs=true|block_fords=false",
"elevation": true,
"ext_storages": {
"WayCategory": {},
"WaySurfaceType": {},
"HillIndex": {},
"TrailDifficulty": {}
}
}
},
"profile-bike-mountain": {
"profiles": "cycling-mountain",
"parameters": {
"encoder_options": "consider_elevation=true|turn_costs=true|block_fords=false",
"elevation": true,
"ext_storages": {
"WayCategory": {},
"WaySurfaceType": {},
"HillIndex": {},
"TrailDifficulty": {}
}
}
},
"profile-bike-road": {
"profiles": "cycling-road",
"parameters": {
"encoder_options": "consider_elevation=true|turn_costs=true|block_fords=false",
"elevation": true,
"ext_storages": {
"WayCategory": {},
"WaySurfaceType": {},
"HillIndex": {},
"TrailDifficulty": {}
}
}
},
"profile-bike-electric": {
"profiles": "cycling-electric",
"parameters": {
"encoder_options": "consider_elevation=true|turn_costs=true|block_fords=false",
"elevation": true,
"ext_storages": {
"WayCategory": {},
"WaySurfaceType": {},
"HillIndex": {},
"TrailDifficulty": {}
}
}
},
"profile-walking": {
"profiles": "foot-walking",
"parameters": {
"encoder_options": "block_fords=false",
"elevation": true,
"ext_storages": {
"WayCategory": {},
"WaySurfaceType": {},
"HillIndex": {},
"TrailDifficulty": {}
}
}
},
"profile-hiking": {
"profiles": "foot-hiking",
"parameters": {
"encoder_options": "block_fords=false",
"elevation": true,
"ext_storages": {
"WayCategory": {},
"WaySurfaceType": {},
"HillIndex": {},
"TrailDifficulty": {}
}
}
},
"profile-wheelchair": {
"profiles": "wheelchair",
"parameters": {
"encoder_options": "block_fords=true",
"elevation": true,
"maximum_snapping_radius": 50,
"ext_storages": {
"WayCategory": {},
"WaySurfaceType": {},
"Wheelchair": {
"KerbsOnCrossings": "true"
},
"OsmId": {}
}
}
}
}
}
},
"logging": {
"enabled": true,
"level_file": "DEBUG_LOGGING.json",
"location": "/var/log/ors",
"stdout": true
},
"system_message": [
{
"active": false,
"text": "This message would be sent with every routing bike fastest request",
"condition": {
"request_service": "routing",
"request_profile": "cycling-regular,cycling-mountain,cycling-road,cycling-electric",
"request_preference": "fastest"
}
},
{
"active": false,
"text": "This message would be sent with every request for geojson response",
"condition": {
"api_format": "geojson"
}
},
{
"active": false,
"text": "This message would be sent with every request on API v1 from January 2020 until June 2050",
"condition": {
"api_version": 1,
"time_after": "2020-01-01T00:00:00Z",
"time_before": "2050-06-01T00:00:00Z"
}
},
{
"active": false,
"text": "This message would be sent with every request"
}
]
}
}
Thank's a lot for getting back to me so quickly! Is Philippines the smallest OSM dataset which gives you the error, or were you able to further narrow it down?
@aoles I've also experienced another error, seems a bit random which error I get. The other I can't recall completely, but something about "-1 index is out of bounds of length xx".
Makes total sense to me, this is what we observe too.
aoles
commented
3 years ago A quick solution to circumvent the issue might be setting "threads": 1 in core preparations for the given profile (car in your case @kgrimsby). We are working on a proper fix.
kgrimsby
commented
3 years ago Thank's a lot for getting back to me so quickly! Is Philippines the smallest OSM dataset which gives you the error, or were you able to further narrow it down?
I haven't tried with smaller datasets.
A quick solution to circumvent the issue might be setting
"threads": 1in core preparations for the given profile (car in your case @kgrimsby). We are working on a proper fix.
I will try and get back to you.
What is latest version without this bug?
Thanks for the quick replies!
aoles
commented
3 years ago Thanks for your feedback! I was able to reproduce the problem on the OSM Berlin dataset, haven't tried smaller. This looks like some sort of a threading issue.
What is latest version without this bug?
We haven't identified nor fixed the bug yet.
kgrimsby
commented
3 years ago I can confirm that setting "threads": 1 in core preparations on my active profile seemed to fix the issue. At least now I get the familiar output from graphhopper!
15 Apr 21:37:46 INFO [core.PrepareCore] - 0, updates:0, nodes: 1 629 611, shortcuts:0, dijkstras:5 407 393, t(dijk):6.97, t(period):0.0, t(lazy):0.0, t(neighbor):0.0, meanDegree:1, algo:24MB, totalMB:8187, usedMB:827
15 Apr 21:39:03 INFO [core.PrepareCore] - 325 922, updates:2, nodes: 1 303 689, shortcuts:449, dijkstras:14 838 579, t(dijk):71.09, t(period):67.71, t(lazy):0.0, t(neighbor):7.71, meanDegree:1, algo:24MB, totalMB:8187, usedMB:855
15 Apr 21:39:59 INFO [core.PrepareCore] - 651 844, updates:4, nodes: 977 767, shortcuts:481, dijkstras:21 712 914, t(dijk):117.96, t(period):118.29, t(lazy):0.0, t(neighbor):12.57, meanDegree:1, algo:24MB, totalMB:8187, usedMB:728
15 Apr 21:40:39 INFO [core.PrepareCore] - 977 766, updates:6, nodes: 651 845, shortcuts:496, dijkstras:26 321 265, t(dijk):151.41, t(period):153.41, t(lazy):0.0, t(neighbor):16.85, meanDegree:1, algo:24MB, totalMB:8187, usedMB:844
15 Apr 21:41:21 INFO [core.PrepareCore] - 1 303 688, updates:8, nodes: 325 923, shortcuts:260 587, dijkstras:30 764 482, t(dijk):185.51, t(period):173.06, t(lazy):0.0, t(neighbor):33.27, meanDegree:2, algo:24MB, totalMB:8187, usedMB:997
15 Apr 21:41:48 INFO [core.PrepareCore] - 1 629 610, updates:10, nodes: 67 800, shortcuts:492 439, dijkstras:34 043 457, t(dijk):206.08, t(period):178.38, t(lazy):3.08, t(neighbor):47.15, meanDegree:2, algo:24MB, totalMB:8187, usedMB:842
15 Apr 21:41:54 INFO [core.PrepareCore] - took:262, new shortcuts: 541 916, prepare|fastest, dijkstras:34804475, t(dijk):210.38, t(period):178.38, t(lazy):4.57, t(neighbor):50.51, meanDegree:2, initSize:1629611, periodic:10, lazy:10, neighbor:90, totalMB:8187, usedMB:862The bugfix is now available in https://github.com/GIScience/openrouteservice/commit/2678e5feb5110da44b582cf9aab5a1b9ae3afeb0. It will be merged into master and released as soon as some additional testing has been completed.
For the record: The underlying problem was a concurrency issue caused by the fact that the same instance of TurnRestrictionsCoreEdgeFilter has been used across different weightings. This caused problems in a multi-threaded scenario because then the edge iterators inside the edge filter became shared across threads. The issue became evident with more than one core preparation weighting enabled and preparation threads >1.
Problem deploying osm.pbf-files to ors in docker/kubernetes
Here's what I did
Deployed docker image on kubernetes using both latest and v6.4.1 tag. Provided norway-latest.osm.pbf and phillippines-latest.osm.pbf from Geofabrik.
Here's what I got
java.util.concurrent.ExecutionException: java.lang.RuntimeException: java.util.concurrent.ExecutionException: java.lang.IllegalArgumentException: from and to edge cannot be NO_EDGE at java.util.concurrent.FutureTask.report(FutureTask.java:122) ~[?:?] at java.util.concurrent.FutureTask.get(FutureTask.java:191) ~[?:?] at org.heigit.ors.routing.RoutingProfileManager.initialize(RoutingProfileManager.java:167) ~[classes/:6.4.1] at org.heigit.ors.routing.RoutingProfileManager.getInstance(RoutingProfileManager.java:62) ~[classes/:6.4.1] at org.heigit.ors.servlet.listeners.ORSInitContextListener.lambda$contextInitialized$0(ORSInitContextListener.java:40) ~[classes/:6.4.1] at java.lang.Thread.run(Thread.java:834) [?:?] Caused by: java.lang.RuntimeException: java.util.concurrent.ExecutionException: java.lang.IllegalArgumentException: from and to edge cannot be NO_EDGE at org.heigit.ors.routing.graphhopper.extensions.core.CoreAlgoFactoryDecorator.prepare(CoreAlgoFactoryDecorator.java:306) ~[classes/:6.4.1] at org.heigit.ors.routing.graphhopper.extensions.ORSGraphHopper.prepareCore(ORSGraphHopper.java:737) ~[classes/:6.4.1] at org.heigit.ors.routing.graphhopper.extensions.ORSGraphHopper.postProcessing(ORSGraphHopper.java:657) ~[classes/:6.4.1] at com.graphhopper.GraphHopper.process(GraphHopper.java:703) ~[graphhopper-core-v0.13.10.jar:?] at com.graphhopper.GraphHopper.importOrLoad(GraphHopper.java:680) ~[graphhopper-core-v0.13.10.jar:?] at org.heigit.ors.routing.graphhopper.extensions.ORSGraphHopper.importOrLoad(ORSGraphHopper.java:155) ~[classes/:6.4.1] at org.heigit.ors.routing.RoutingProfile.initGraphHopper(RoutingProfile.java:194) ~[classes/:6.4.1] at org.heigit.ors.routing.RoutingProfile.(RoutingProfile.java:138) ~[classes/:6.4.1]
at org.heigit.ors.routing.RoutingProfileLoader.call(RoutingProfileLoader.java:35) ~[classes/:6.4.1]
at org.heigit.ors.routing.RoutingProfileLoader.call(RoutingProfileLoader.java:21) ~[classes/:6.4.1]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) ~[?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
... 1 more
Dockerfile
FROM openrouteservice/openrouteservice:v6.4.1
ENV BUILD_GRAPHS=False ENV JAVA_OPTS="-Djava.awt.headless=true -server -XX:TargetSurvivorRatio=75 -XX:SurvivorRatio=64 -XX:MaxTenuringThreshold=3 -XX:+UseConcMarkSweepGC -XX:ParallelGCThreads=4 -Xms8g -Xmx8g" ENV CATALINA_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9001 -Dcom.sun.management.jmxremote.rmi.port=9001 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost"
COPY app.config /ors-core/openrouteservice/src/main/resources/ COPY custom-entrypoint.sh /
RUN apt-get -y install awscli
RUN aws s3 cp s3:/xxx.xxx.xxx/data3.osm.pbf /ors-core/data/data.osm.pbf
ENTRYPOINT [] CMD ["/bin/bash", "/custom-entrypoint.sh"]
custom-entrypoint.sh
rm -rf /ors-core/data/graphs/car rm -rf /ors-core/data/elevation_cache/*
/docker-entrypoint.sh
Have anyone experienced this problem?