Fabio/brouter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

changed db-tag-processing to csv-file

Browse source
This commit is contained in:
Arndt Brenschede 2023-07-02 09:46:38 +02:00
parent 1a2bb197d1
commit 21b0431a1a
1 changed files with 85 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

107
brouter-map-creator/src/main/java/btools/mapcreator/DatabasePseudoTagProvider.java
View file

@ -4,6 +4,12 @@
*/
package btools.mapcreator;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
@ -11,6 +17,9 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import btools.util.CompactLongMap;
import btools.util.FrozenLongMap;
@ -24,29 +33,20 @@ public class DatabasePseudoTagProvider {
FrozenLongMap<Map<String, String>> dbData;
public DatabasePseudoTagProvider(String jdbcurl) {
public static void main(String[] args) {
String jdbcurl = args[0];
String filename = args[1];
try (Connection conn = DriverManager.getConnection(jdbcurl);
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
filename.endsWith(".gz") ? new GZIPOutputStream(new FileOutputStream(filename)) : new FileOutputStream(filename)))) {
try (Connection conn = DriverManager.getConnection(jdbcurl)) {
conn.setAutoCommit(false);
System.out.println("DatabasePseudoTagProvider start connection to the database........" + jdbcurl);
System.out.println("DatabasePseudoTagProvider dumping data from " + jdbcurl + " to file " + filename);
Map<String, String> databaseField2Tag = new HashMap<>();
databaseField2Tag.put("noise_class", "estimated_noise_class");
databaseField2Tag.put("river_class", "estimated_river_class");
databaseField2Tag.put("forest_class", "estimated_forest_class");
databaseField2Tag.put("town_class", "estimated_town_class");
databaseField2Tag.put("traffic_class", "estimated_traffic_class");
pseudoTagsFound = new HashMap<>();
for (String pseudoTag : databaseField2Tag.values()) {
pseudoTagsFound.put(pseudoTag, 0L);
}
Map<Map<String, String>, Map<String, String>> mapUnifier = new HashMap<>();
CompactLongMap<Map<String, String>> data = new CompactLongMap<>();
System.out.println("DatabasePseudoTagProvider connect to the database ok........");
bw.write("losmid;noise_class;river_class;forest_class;town_class;traffic_class\n");
String sql_all_tags = "SELECT * from all_tags";
try(PreparedStatement psAllTags = conn.prepareStatement(sql_all_tags)) {
@ -58,14 +58,56 @@ public class DatabasePseudoTagProvider {
long dbRows = 0L;
while (rsBrouter.next()) {
long osm_id = rsBrouter.getLong("losmid");
StringBuilder line = new StringBuilder();
line.append(rsBrouter.getLong("losmid"))
.append(';').append(rsBrouter.getString("noise_class"))
.append(';').append(rsBrouter.getString("river_class"))
.append(';').append(rsBrouter.getString("forest_class"))
.append(';').append(rsBrouter.getString("town_class"))
.append(';').append(rsBrouter.getString("traffic_class"))
.append('\n');
bw.write(line.toString());
dbRows++;
if (dbRows % 1000000L == 0L) {
System.out.println(".. from database: rows =" + dbRows);
}
}
}
} catch (SQLException g) {
System.err.format("DatabasePseudoTagProvider execute sql .. SQL State: %s\n%s\n", g.getSQLState(), g.getMessage());
System.exit(1);
} catch (Exception f) {
f.printStackTrace();
System.exit(1);
}
}
public DatabasePseudoTagProvider(String filename) {
try (BufferedReader br = new BufferedReader(new InputStreamReader(
filename.endsWith(".gz") ? new GZIPInputStream(new FileInputStream(filename)) : new FileInputStream(filename)))) {
System.out.println("DatabasePseudoTagProvider reading from file: " + filename);
br.readLine(); // skip header line
Map<Map<String, String>, Map<String, String>> mapUnifier = new HashMap<>();
CompactLongMap<Map<String, String>> data = new CompactLongMap<>();
long dbRows = 0L;
for (;;) {
String line = br.readLine();
if (line == null) {
break;
}
StringTokenizer tk = new StringTokenizer(line, ";");
long osm_id = Long.parseLong(tk.nextToken());
Map<String, String> row = new HashMap<>(5);
for (String key : databaseField2Tag.keySet()) {
String value = rsBrouter.getString(key);
if (value != null && !value.isEmpty()) {
row.put(databaseField2Tag.get(key), value);
}
}
row.put("estimated_noise_class", tk.nextToken());
row.put("estimated_river_class", tk.nextToken());
row.put("estimated_forest_class", tk.nextToken());
row.put("estimated_town_class", tk.nextToken());
row.put("estimated_traffic_class", tk.nextToken());
// apply the instance-unifier for the row-map
Map<String, String> knownRow = mapUnifier.get(row);
@ -81,14 +123,9 @@ public class DatabasePseudoTagProvider {
System.out.println(".. from database: rows =" + data.size() + " unique rows=" + mapUnifier.size());
}
}
}
System.out.println("freezing result map..");
dbData = new FrozenLongMap<>(data);
System.out.println("read from database: rows =" + dbData.size() + " unique rows=" + mapUnifier.size());
} catch (SQLException g) {
System.err.format("DatabasePseudoTagProvider execute sql .. SQL State: %s\n%s\n", g.getSQLState(), g.getMessage());
System.exit(1);
System.out.println("read from file: rows =" + dbData.size() + " unique rows=" + mapUnifier.size());
} catch (Exception f) {
f.printStackTrace();
System.exit(1);
@ -119,7 +156,11 @@ public class DatabasePseudoTagProvider {
cntWayModified = cntWayModified + 1;
for (String key : dbTags.keySet()) {
map.put(key, dbTags.get(key));
pseudoTagsFound.put(key, pseudoTagsFound.get(key) + 1L);
Long cnt = pseudoTagsFound.get(key);
if (cnt == null) {
cnt = 0L;
}
pseudoTagsFound.put(key, cnt + 1L);
}
}
}