Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Build enhancements graphbuilder #1

Closed
wants to merge 4 commits into from
Closed

Build enhancements graphbuilder #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b0cd4ea
Use a queue to split graph builder work (instead of splitting the ite…
lseelenbinder Mar 28, 2024
54f7128
Replace language_poly vector with rtree in graph building for 6x spee…
lseelenbinder Mar 28, 2024
eacbb60
Clang-tidy.
lseelenbinder Mar 31, 2024
f9e360a
Significantly improve performance of graph building.
lseelenbinder Mar 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 63 additions & 60 deletions src/mjolnir/admin.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

#include <spatialite.h>

namespace valhalla {
namespace mjolnir {

namespace valhalla::mjolnir {

// Get the dbhandle of a sqlite db. Used for timezones and admins DBs.
sqlite3* GetDBHandle(const std::string& database) {
Expand Down Expand Up @@ -37,7 +37,7 @@ uint32_t GetMultiPolyId(const std::multimap<uint32_t, multi_polygon_type>& polys
uint32_t index = 0;
point_type p(ll.lng(), ll.lat());
for (const auto& poly : polys) {
if (boost::geometry::covered_by(p, poly.second)) {
if (bg::covered_by(p, poly.second, bg::strategy::within::crossings_multiply<point_type>())) {
const auto& admin = graphtile.admins_builder(poly.first);
if (!admin.state_offset())
index = poly.first;
Expand All @@ -55,8 +55,9 @@ uint32_t GetMultiPolyId(const std::multimap<uint32_t, multi_polygon_type>& polys
point_type p(ll.lng(), ll.lat());
for (const auto& poly : polys) {
// TODO: we recently discovered that boost::geometry doesn't do bbox checks to speed things up
if (boost::geometry::covered_by(p, poly.second))
if (bg::covered_by(p, poly.second, bg::strategy::within::crossings_multiply<point_type>())) {
return poly.first;
}
}
return index;
}
Expand All @@ -70,52 +71,30 @@ uint32_t GetMultiPolyId(const std::multimap<uint32_t, multi_polygon_type>& polys
// languages that will be considered for any name* or destination* keys. Basically, we only support
// the languages that are on the signs in that area. Note: The first pair always contains an empty
// language which makes the name key with no language the most important key.
std::vector<std::pair<std::string, bool>>
GetMultiPolyIndexes(const std::vector<std::tuple<std::string, multi_polygon_type, bool>>& polys,
const PointLL& ll) {

auto process_languages = [](const std::vector<std::string>& langs, bool is_default,
std::vector<std::pair<std::string, bool>>& languages) {
for (const auto& l : langs) {
if (stringLanguage(l) != Language::kNone) {
std::vector<std::pair<std::string, bool>>::iterator it =
std::find_if(languages.begin(), languages.end(),
[&l](const std::pair<std::string, bool>& p) { return p.first == l; });
if (it == languages.end()) {
languages.emplace_back(l, false);
} else if (is_default) { // fr - nl or fr;en in default lang column
it->second = false;
}
}
}
};

std::vector<std::pair<std::string, bool>> GetMultiPolyIndexes(const language_poly_index& polys,
const PointLL& ll) {
std::vector<std::pair<std::string, bool>> languages;
std::vector<std::pair<std::string, bool>>::iterator it;
std::string lang;

// first entry is blank for the default name
languages.emplace_back("", false);

point_type p(ll.lng(), ll.lat());

for (const auto& poly : polys) {

if (boost::geometry::covered_by(p, std::get<1>(poly))) {
lang = std::get<0>(poly);
it = std::find_if(languages.begin(), languages.end(),
[&lang](const std::pair<std::string, bool>& p) { return p.first == lang; });

if (it == languages.end()) {
std::vector<std::string> langs = GetTagTokens(std::get<0>(poly), " - ");
if (langs.size() >= 2) {
process_languages(langs, std::get<2>(poly), languages);
} else {
langs = GetTagTokens(std::get<0>(poly));
if (langs.size() >= 2) {
process_languages(langs, std::get<2>(poly), languages);
} else {
languages.emplace_back(std::get<0>(poly), std::get<2>(poly));
for (auto it = polys.qbegin(bg::index::covers(p)); it != polys.qend(); ++it) {
// Make sure the point is in the admin poly (not just its envelope)
if (bg::covered_by(p, std::get<1>(*it), bg::strategy::within::crossings_multiply<point_type>())) {
auto& langs = std::get<2>(*it);
bool is_default = std::get<3>(*it);

for (const auto& l : langs) {
if (stringLanguage(l) != Language::kNone) {
auto needle =
std::find_if(languages.begin(), languages.end(),
[&l](const std::pair<std::string, bool>& p) { return p.first == l; });
if (needle == languages.end()) {
languages.emplace_back(l, false);
} else if (is_default) { // fr - nl or fr;en in default lang column
needle->second = false;
}
}
}
Expand Down Expand Up @@ -181,14 +160,26 @@ std::multimap<uint32_t, multi_polygon_type> GetTimeZones(sqlite3* db_handle,
return polys;
}

/***
* Parses a language tag into a vector of individual tokens.
*/
std::vector<std::string> ParseLanguageTokens(const std::string& lang_tag) {
auto langs = GetTagTokens(lang_tag, " - ");
if (langs.size() == 1) {
langs = GetTagTokens(langs.at(0));
}

return langs;
}

void GetData(sqlite3* db_handle,
sqlite3_stmt* stmt,
const std::string& sql,
GraphTileBuilder& tilebuilder,
std::multimap<uint32_t, multi_polygon_type>& polys,
std::unordered_map<uint32_t, bool>& drive_on_right,
std::unordered_map<uint32_t, bool>& allow_intersection_names,
std::vector<std::tuple<std::string, multi_polygon_type, bool>>& language_ploys,
language_poly_index& language_ploys,
bool languages_only = false) {
uint32_t result = 0;
bool dor = true;
Expand Down Expand Up @@ -253,15 +244,21 @@ void GetData(sqlite3* db_handle,

uint32_t index = tilebuilder.AddAdmin(country_name, state_name, country_iso, state_iso);
multi_polygon_type multi_poly;
boost::geometry::read_wkt(geom, multi_poly);
bg::read_wkt(geom, multi_poly);
polys.emplace(index, multi_poly);
drive_on_right.emplace(index, dor);
allow_intersection_names.emplace(index, intersection_name);

if (!default_language.empty())
language_ploys.push_back(std::make_tuple(default_language, multi_poly, true));
if (!supported_languages.empty())
language_ploys.push_back(std::make_tuple(supported_languages, multi_poly, false));
bg::model::box<point_type> box{};
bg::envelope(multi_poly, box);
if (!default_language.empty()) {
auto langs = ParseLanguageTokens(default_language);
language_ploys.insert(std::make_tuple(box, multi_poly, langs, true));
}
if (!supported_languages.empty()) {
auto langs = ParseLanguageTokens(supported_languages);
language_ploys.insert(std::make_tuple(box, multi_poly, langs, false));
}

} else {

Expand All @@ -281,12 +278,18 @@ void GetData(sqlite3* db_handle,
}

multi_polygon_type multi_poly;
boost::geometry::read_wkt(geom, multi_poly);
bg::read_wkt(geom, multi_poly);
bg::model::box<point_type> box{};
bg::envelope(multi_poly, box);

if (!default_language.empty())
language_ploys.push_back(std::make_tuple(default_language, multi_poly, true));
if (!supported_languages.empty())
language_ploys.push_back(std::make_tuple(supported_languages, multi_poly, false));
if (!default_language.empty()) {
auto langs = ParseLanguageTokens(default_language);
language_ploys.insert(std::make_tuple(box, multi_poly, langs, true));
}
if (!supported_languages.empty()) {
auto langs = ParseLanguageTokens(supported_languages);
language_ploys.insert(std::make_tuple(box, multi_poly, langs, false));
}
}

result = sqlite3_step(stmt);
Expand All @@ -303,7 +306,7 @@ std::multimap<uint32_t, multi_polygon_type>
GetAdminInfo(sqlite3* db_handle,
std::unordered_map<uint32_t, bool>& drive_on_right,
std::unordered_map<uint32_t, bool>& allow_intersection_names,
std::vector<std::tuple<std::string, multi_polygon_type, bool>>& language_ploys,
language_poly_index& language_polys,
const AABB2<PointLL>& aabb,
GraphTileBuilder& tilebuilder) {
std::multimap<uint32_t, multi_polygon_type> polys;
Expand All @@ -326,7 +329,7 @@ GetAdminInfo(sqlite3* db_handle,
sql += std::to_string(aabb.miny()) + ", " + std::to_string(aabb.maxx()) + ",";
sql += std::to_string(aabb.maxy()) + ")) order by admin_level desc, name;";
GetData(db_handle, stmt, sql, tilebuilder, polys, drive_on_right, allow_intersection_names,
language_ploys, true);
language_polys, true);

// state query
sql = "SELECT country.name, state.name, country.iso_code, ";
Expand All @@ -342,7 +345,7 @@ GetAdminInfo(sqlite3* db_handle,
sql += std::to_string(aabb.miny()) + ", " + std::to_string(aabb.maxx()) + ",";
sql += std::to_string(aabb.maxy()) + ")) order by state.name, country.name;";
GetData(db_handle, stmt, sql, tilebuilder, polys, drive_on_right, allow_intersection_names,
language_ploys);
language_polys);

// country query
sql = "SELECT name, \"\", iso_code, \"\", drive_on_right, allow_intersection_names, admin_level, ";
Expand All @@ -356,7 +359,7 @@ GetAdminInfo(sqlite3* db_handle,
sql += std::to_string(aabb.miny()) + ", " + std::to_string(aabb.maxx()) + ",";
sql += std::to_string(aabb.maxy()) + ")) order by name;";
GetData(db_handle, stmt, sql, tilebuilder, polys, drive_on_right, allow_intersection_names,
language_ploys);
language_polys);

if (stmt) { // just in case something bad happened.
sqlite3_finalize(stmt);
Expand Down Expand Up @@ -460,5 +463,5 @@ std::unordered_map<std::string, std::vector<int>> GetCountryAccess(sqlite3* db_h
return country_access;
}

} // namespace mjolnir
} // namespace valhalla
} // namespace valhalla::mjolnir

Loading
Loading