From 21b0431a1abac376f58940a73dc84a9b52331cd3 Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Sun, 2 Jul 2023 09:46:38 +0200 Subject: [PATCH] changed db-tag-processing to csv-file --- .../mapcreator/DatabasePseudoTagProvider.java | 129 ++++++++++++------ 1 file changed, 85 insertions(+), 44 deletions(-) diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/DatabasePseudoTagProvider.java b/brouter-map-creator/src/main/java/btools/mapcreator/DatabasePseudoTagProvider.java index d0a2b9d..aeabbf6 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/DatabasePseudoTagProvider.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/DatabasePseudoTagProvider.java @@ -4,6 +4,12 @@ */ package btools.mapcreator; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -11,6 +17,9 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; +import java.util.StringTokenizer; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import btools.util.CompactLongMap; import btools.util.FrozenLongMap; @@ -24,29 +33,20 @@ public class DatabasePseudoTagProvider { FrozenLongMap> dbData; - public DatabasePseudoTagProvider(String jdbcurl) { + public static void main(String[] args) { + + String jdbcurl = args[0]; + String filename = args[1]; + + try (Connection conn = DriverManager.getConnection(jdbcurl); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter( + filename.endsWith(".gz") ? new GZIPOutputStream(new FileOutputStream(filename)) : new FileOutputStream(filename)))) { - try (Connection conn = DriverManager.getConnection(jdbcurl)) { conn.setAutoCommit(false); - System.out.println("DatabasePseudoTagProvider start connection to the database........" + jdbcurl); + System.out.println("DatabasePseudoTagProvider dumping data from " + jdbcurl + " to file " + filename); - Map databaseField2Tag = new HashMap<>(); - databaseField2Tag.put("noise_class", "estimated_noise_class"); - databaseField2Tag.put("river_class", "estimated_river_class"); - databaseField2Tag.put("forest_class", "estimated_forest_class"); - databaseField2Tag.put("town_class", "estimated_town_class"); - databaseField2Tag.put("traffic_class", "estimated_traffic_class"); - - pseudoTagsFound = new HashMap<>(); - for (String pseudoTag : databaseField2Tag.values()) { - pseudoTagsFound.put(pseudoTag, 0L); - } - - Map, Map> mapUnifier = new HashMap<>(); - CompactLongMap> data = new CompactLongMap<>(); - - System.out.println("DatabasePseudoTagProvider connect to the database ok........"); + bw.write("losmid;noise_class;river_class;forest_class;town_class;traffic_class\n"); String sql_all_tags = "SELECT * from all_tags"; try(PreparedStatement psAllTags = conn.prepareStatement(sql_all_tags)) { @@ -58,37 +58,74 @@ public class DatabasePseudoTagProvider { long dbRows = 0L; while (rsBrouter.next()) { - long osm_id = rsBrouter.getLong("losmid"); - Map row = new HashMap<>(5); - for (String key : databaseField2Tag.keySet()) { - String value = rsBrouter.getString(key); - if (value != null && !value.isEmpty()) { - row.put(databaseField2Tag.get(key), value); - } - } - - // apply the instance-unifier for the row-map - Map knownRow = mapUnifier.get(row); - if (knownRow != null) { - row = knownRow; - } else { - mapUnifier.put(row, row); - } - data.put(osm_id, row); + StringBuilder line = new StringBuilder(); + line.append(rsBrouter.getLong("losmid")) + .append(';').append(rsBrouter.getString("noise_class")) + .append(';').append(rsBrouter.getString("river_class")) + .append(';').append(rsBrouter.getString("forest_class")) + .append(';').append(rsBrouter.getString("town_class")) + .append(';').append(rsBrouter.getString("traffic_class")) + .append('\n'); + bw.write(line.toString()); dbRows++; - if (dbRows % 1000000L == 0L) { - System.out.println(".. from database: rows =" + data.size() + " unique rows=" + mapUnifier.size()); + System.out.println(".. from database: rows =" + dbRows); } } } + } catch (SQLException g) { + System.err.format("DatabasePseudoTagProvider execute sql .. SQL State: %s\n%s\n", g.getSQLState(), g.getMessage()); + System.exit(1); + } catch (Exception f) { + f.printStackTrace(); + System.exit(1); + } + } + + public DatabasePseudoTagProvider(String filename) { + + try (BufferedReader br = new BufferedReader(new InputStreamReader( + filename.endsWith(".gz") ? new GZIPInputStream(new FileInputStream(filename)) : new FileInputStream(filename)))) { + + System.out.println("DatabasePseudoTagProvider reading from file: " + filename); + + br.readLine(); // skip header line + + Map, Map> mapUnifier = new HashMap<>(); + CompactLongMap> data = new CompactLongMap<>(); + + long dbRows = 0L; + for (;;) { + String line = br.readLine(); + if (line == null) { + break; + } + StringTokenizer tk = new StringTokenizer(line, ";"); + long osm_id = Long.parseLong(tk.nextToken()); + Map row = new HashMap<>(5); + row.put("estimated_noise_class", tk.nextToken()); + row.put("estimated_river_class", tk.nextToken()); + row.put("estimated_forest_class", tk.nextToken()); + row.put("estimated_town_class", tk.nextToken()); + row.put("estimated_traffic_class", tk.nextToken()); + + // apply the instance-unifier for the row-map + Map knownRow = mapUnifier.get(row); + if (knownRow != null) { + row = knownRow; + } else { + mapUnifier.put(row, row); + } + data.put(osm_id, row); + dbRows++; + + if (dbRows % 1000000L == 0L) { + System.out.println(".. from database: rows =" + data.size() + " unique rows=" + mapUnifier.size()); + } + } System.out.println("freezing result map.."); dbData = new FrozenLongMap<>(data); - System.out.println("read from database: rows =" + dbData.size() + " unique rows=" + mapUnifier.size()); - - } catch (SQLException g) { - System.err.format("DatabasePseudoTagProvider execute sql .. SQL State: %s\n%s\n", g.getSQLState(), g.getMessage()); - System.exit(1); + System.out.println("read from file: rows =" + dbData.size() + " unique rows=" + mapUnifier.size()); } catch (Exception f) { f.printStackTrace(); System.exit(1); @@ -119,7 +156,11 @@ public class DatabasePseudoTagProvider { cntWayModified = cntWayModified + 1; for (String key : dbTags.keySet()) { map.put(key, dbTags.get(key)); - pseudoTagsFound.put(key, pseudoTagsFound.get(key) + 1L); + Long cnt = pseudoTagsFound.get(key); + if (cnt == null) { + cnt = 0L; + } + pseudoTagsFound.put(key, cnt + 1L); } } }