Fabio/brouter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

pseudo-tags from DB, here: preload and use in-memory matching

Browse source
This commit is contained in:
Arndt Brenschede 2023-05-28 19:25:42 +02:00
parent 624edc63ee
commit bfe1f4a6a4
2 changed files with 129 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

121
brouter-map-creator/src/main/java/btools/mapcreator/DatabasePseudoTagProvider.java Normal file
View file

@ -0,0 +1,121 @@
/**
* DatabasePseudoTagProvider reads Pseudo Tags from a database and adds them
* to the osm-data
*/
package btools.mapcreator;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
import btools.util.CompactLongMap;
import btools.util.FrozenLongMap;
public class DatabasePseudoTagProvider {
long cntOsmWays = 0L;
long cntWayModified = 0L;
Map<String, Long> pseudoTagsFound;
FrozenLongMap<Map<String, String>> dbData;
public DatabasePseudoTagProvider(String jdbcurl) {
try (Connection conn = DriverManager.getConnection(jdbcurl)) {
conn.setAutoCommit(false);
System.out.println("DatabasePseudoTagProvider start connection to the database........" + jdbcurl);
Map<String, String> databaseField2Tag = new HashMap<>();
databaseField2Tag.put("noise_class", "estimated_noise_class");
databaseField2Tag.put("river_class", "estimated_river_class");
databaseField2Tag.put("forest_class", "estimated_forest_class");
databaseField2Tag.put("town_class", "estimated_town_class");
databaseField2Tag.put("traffic_class", "estimated_traffic_class");
pseudoTagsFound = new HashMap<>();
for (String pseudoTag : databaseField2Tag.values()) {
pseudoTagsFound.put(pseudoTag, 0L);
}
Map<Map<String, String>, Map<String, String>> mapUnifier = new HashMap<>();
CompactLongMap<Map<String, String>> data = new CompactLongMap<>();
System.out.println("DatabasePseudoTagProvider connect to the database ok........");
String sql_all_tags = "SELECT * from all_tags";
try(PreparedStatement psAllTags = conn.prepareStatement(sql_all_tags)) {
psAllTags.setFetchSize(100);
// process the results
ResultSet rsBrouter = psAllTags.executeQuery();
long dbRows = 0L;
while (rsBrouter.next()) {
long osm_id = rsBrouter.getLong("losmid");
Map<String, String> row = new HashMap<>(5);
for (String key : databaseField2Tag.keySet()) {
String value = rsBrouter.getString(key);
if (value != null && !value.isEmpty()) {
row.put(databaseField2Tag.get(key), value);
}
}
// apply the instance-unifier for the row-map
Map<String, String> knownRow = mapUnifier.get(row);
if (knownRow != null) {
row = knownRow;
} else {
mapUnifier.put(row, row);
}
data.put(osm_id, row);
dbRows++;
if (dbRows % 1000000L == 0L) {
System.out.println(".. from database: rows =" + data.size() + " unique rows=" + mapUnifier.size());
}
}
}
System.out.println("freezing result map..");
dbData = new FrozenLongMap<>(data);
System.out.println("read from database: rows =" + dbData.size() + " unique rows=" + mapUnifier.size());
} catch (SQLException g) {
System.err.format("DatabasePseudoTagProvider execute sql .. SQL State: %s\n%s\n", g.getSQLState(), g.getMessage());
System.exit(1);
} catch (Exception f) {
f.printStackTrace();
System.exit(1);
}
}
public void addTags(long osm_id, Map<String, String> map) {
cntOsmWays++;
if ((cntOsmWays % 1000000L) == 0) {
String out = "Osm Ways processed=" + cntOsmWays + " way modifs=" + cntWayModified;
for (String key : pseudoTagsFound.keySet()) {
out += " " + key + "=" + pseudoTagsFound.get(key);
}
System.out.println(out);
}
Map<String, String> dbTags = dbData.get(osm_id);
if (dbTags == null) {
return;
}
cntWayModified = cntWayModified + 1;
for (String key : dbTags.keySet()) {
map.put(key, dbTags.get(key));
pseudoTagsFound.put(key, pseudoTagsFound.get(key) + 1L);
}
}
}

103
brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java
View file

@ -12,12 +12,6 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
import btools.expressions.BExpressionContextNode;
@ -39,18 +33,7 @@ public class OsmCutter extends MapCreatorBase {
public RestrictionCutter restrictionCutter;
public NodeFilter nodeFilter;
Connection conn = null;
PreparedStatement psAllTags = null;
ResultSet rsBrouter = null;
int cntHighways = 0;
int cntWayModified = 0;
String jdbcurl;
Map<String, String> databaseField2Tag;
Map<String, Integer> databaseFieldsFound;
private DatabasePseudoTagProvider dbPseudoTagProvider;
public static void main(String[] args) throws Exception {
System.out.println("*** OsmCutter: cut an osm map in node-tiles + a way file");
@ -130,7 +113,7 @@ public class OsmCutter extends MapCreatorBase {
}
public void setJdbcUrl(String url) {
this.jdbcurl = url;
dbPseudoTagProvider = new DatabasePseudoTagProvider(url);
}
@Override
@ -179,78 +162,6 @@ public class OsmCutter extends MapCreatorBase {
}
}
private void generateTagsFromDatabase(long osm_id, Map<String, String> map) {
if (jdbcurl == null) return;
try {
// is the database allready connected?
if (conn == null) {
String sql_all_tags = "SELECT * from all_tags where losmid = ?";
System.out.println("OsmCutter start connection to the database........" + jdbcurl);
conn = DriverManager.getConnection(jdbcurl);
psAllTags = conn.prepareStatement(sql_all_tags);
databaseField2Tag = new HashMap<>();
databaseField2Tag.put("noise_class", "estimated_noise_class");
databaseField2Tag.put("river_class", "estimated_river_class");
databaseField2Tag.put("forest_class", "estimated_forest_class");
databaseField2Tag.put("town_class", "estimated_town_class");
databaseField2Tag.put("traffic_class", "estimated_traffic_class");
databaseFieldsFound = new HashMap<>();
for (String key : databaseField2Tag.keySet()) {
databaseFieldsFound.put(key, 0);
}
System.out.println("OsmCutter connect to the database ok........");
}
for (Map.Entry<String, String> e : map.entrySet()) {
if (e.getKey().equals("highway")) {
cntHighways = cntHighways + 1;
psAllTags.setLong(1, osm_id);
// process the results
rsBrouter = psAllTags.executeQuery();
if (rsBrouter.next()) {
cntWayModified = cntWayModified + 1;
for (String key : databaseField2Tag.keySet()) {
if (rsBrouter.getString(key) != null) {
map.put(databaseField2Tag.get(key), rsBrouter.getString(key));
databaseFieldsFound.put(key, databaseFieldsFound.get(key) + 1);
}
}
}
if ((cntHighways % 100000) == 0) {
String out = "HW processed=" + cntHighways + " HW modifs=" + cntWayModified;
for (String key : databaseFieldsFound.keySet()) {
out += " " + key + "=" + databaseFieldsFound.get(key);
}
System.out.println(out);
}
return;
}
}
} catch (SQLException g) {
System.err.format(" OsmCutter execute sql .. SQL State: %s\n%s\n", g.getSQLState(), g.getMessage());
System.exit(1);
} catch (Exception f) {
f.printStackTrace();
System.exit(1);
}
}
@Override
public void nextWay(WayData w) throws Exception {
@ -260,10 +171,6 @@ public class OsmCutter extends MapCreatorBase {
// encode tags
if (w.getTagsOrNull() == null) return;
generateTagsFromDatabase(w.wid, w.getTagsOrNull());
generatePseudoTags(w.getTagsOrNull());
int[] lookupData = _expctxWay.createNewLookupData();
for (String key : w.getTagsOrNull().keySet()) {
String value = w.getTag(key);
@ -280,6 +187,12 @@ public class OsmCutter extends MapCreatorBase {
ok |= _expctxWay.getCostfactor() < 10000.;
if (!ok) return;
if (dbPseudoTagProvider != null) {
dbPseudoTagProvider.addTags(w.wid, w.getTagsOrNull());
}
generatePseudoTags(w.getTagsOrNull());
if (wayDos != null) {
w.writeTo(wayDos);
}