changed db-tag-processing to csv-file

brouter-map-creator/src/main/java/btools/mapcreator/DatabasePseudoTagProvider.java

@@ -4,6 +4,12 @@
  */
 package btools.mapcreator;
 
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
@@ -11,6 +17,9 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.StringTokenizer;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
 
 import btools.util.CompactLongMap;
 import btools.util.FrozenLongMap;
@@ -24,29 +33,20 @@ public class DatabasePseudoTagProvider {
 
   FrozenLongMap<Map<String, String>> dbData;
 
-  public DatabasePseudoTagProvider(String jdbcurl) {
+  public static void main(String[] args) {
+
+    String jdbcurl = args[0];
+    String filename = args[1];
+
+    try (Connection conn = DriverManager.getConnection(jdbcurl);
+         BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
+           filename.endsWith(".gz") ? new GZIPOutputStream(new FileOutputStream(filename)) : new FileOutputStream(filename)))) {
 
-    try (Connection conn = DriverManager.getConnection(jdbcurl)) {
       conn.setAutoCommit(false);
 
-      System.out.println("DatabasePseudoTagProvider start connection to the database........" + jdbcurl);
+      System.out.println("DatabasePseudoTagProvider dumping data from " + jdbcurl + " to file " + filename);
 
-      Map<String, String> databaseField2Tag = new HashMap<>();
+      bw.write("losmid;noise_class;river_class;forest_class;town_class;traffic_class\n");
-      databaseField2Tag.put("noise_class", "estimated_noise_class");
-      databaseField2Tag.put("river_class", "estimated_river_class");
-      databaseField2Tag.put("forest_class", "estimated_forest_class");
-      databaseField2Tag.put("town_class", "estimated_town_class");
-      databaseField2Tag.put("traffic_class", "estimated_traffic_class");
-
-      pseudoTagsFound = new HashMap<>();
-      for (String pseudoTag : databaseField2Tag.values()) {
-        pseudoTagsFound.put(pseudoTag, 0L);
-      }
-
-      Map<Map<String, String>, Map<String, String>> mapUnifier = new HashMap<>();
-      CompactLongMap<Map<String, String>> data = new CompactLongMap<>();
-
-      System.out.println("DatabasePseudoTagProvider connect to the database ok........");
 
       String sql_all_tags = "SELECT * from all_tags";
       try(PreparedStatement psAllTags = conn.prepareStatement(sql_all_tags)) {
@@ -58,37 +58,74 @@ public class DatabasePseudoTagProvider {
 
         long dbRows = 0L;
         while (rsBrouter.next()) {
-          long osm_id = rsBrouter.getLong("losmid");
+          StringBuilder line = new StringBuilder();
-          Map<String, String> row = new HashMap<>(5);
+          line.append(rsBrouter.getLong("losmid"))
-          for (String key : databaseField2Tag.keySet()) {
+            .append(';').append(rsBrouter.getString("noise_class"))
-            String value = rsBrouter.getString(key);
+            .append(';').append(rsBrouter.getString("river_class"))
-            if (value != null && !value.isEmpty()) {
+            .append(';').append(rsBrouter.getString("forest_class"))
-              row.put(databaseField2Tag.get(key), value);
+            .append(';').append(rsBrouter.getString("town_class"))
-            }
+            .append(';').append(rsBrouter.getString("traffic_class"))
-          }
+            .append('\n');
-
+          bw.write(line.toString());
-          // apply the instance-unifier for the row-map
-          Map<String, String> knownRow = mapUnifier.get(row);
-          if (knownRow != null) {
-            row = knownRow;
-          } else {
-            mapUnifier.put(row, row);
-          }
-          data.put(osm_id, row);
           dbRows++;
-
           if (dbRows % 1000000L == 0L) {
-            System.out.println(".. from database: rows =" + data.size() + " unique rows=" + mapUnifier.size());
+            System.out.println(".. from database: rows =" + dbRows);
           }
         }
       }
+    } catch (SQLException g) {
+      System.err.format("DatabasePseudoTagProvider execute sql .. SQL State: %s\n%s\n", g.getSQLState(), g.getMessage());
+      System.exit(1);
+    } catch (Exception f) {
+      f.printStackTrace();
+      System.exit(1);
+    }
+  }
+
+  public DatabasePseudoTagProvider(String filename) {
+
+    try (BufferedReader br = new BufferedReader(new InputStreamReader(
+           filename.endsWith(".gz") ? new GZIPInputStream(new FileInputStream(filename)) : new FileInputStream(filename)))) {
+
+      System.out.println("DatabasePseudoTagProvider reading from file: " + filename);
+
+      br.readLine(); // skip header line
+
+      Map<Map<String, String>, Map<String, String>> mapUnifier = new HashMap<>();
+      CompactLongMap<Map<String, String>> data = new CompactLongMap<>();
+
+      long dbRows = 0L;
+      for (;;) {
+        String line = br.readLine();
+        if (line == null) {
+          break;
+        }
+        StringTokenizer tk = new StringTokenizer(line, ";");
+        long osm_id = Long.parseLong(tk.nextToken());
+        Map<String, String> row = new HashMap<>(5);
+        row.put("estimated_noise_class", tk.nextToken());
+        row.put("estimated_river_class", tk.nextToken());
+        row.put("estimated_forest_class", tk.nextToken());
+        row.put("estimated_town_class", tk.nextToken());
+        row.put("estimated_traffic_class", tk.nextToken());
+
+        // apply the instance-unifier for the row-map
+        Map<String, String> knownRow = mapUnifier.get(row);
+        if (knownRow != null) {
+          row = knownRow;
+        } else {
+          mapUnifier.put(row, row);
+        }
+        data.put(osm_id, row);
+        dbRows++;
+
+        if (dbRows % 1000000L == 0L) {
+          System.out.println(".. from database: rows =" + data.size() + " unique rows=" + mapUnifier.size());
+        }
+      }
       System.out.println("freezing result map..");
       dbData = new FrozenLongMap<>(data);
-      System.out.println("read from database: rows =" + dbData.size() + " unique rows=" + mapUnifier.size());
+      System.out.println("read from file: rows =" + dbData.size() + " unique rows=" + mapUnifier.size());
-
-    } catch (SQLException g) {
-      System.err.format("DatabasePseudoTagProvider execute sql .. SQL State: %s\n%s\n", g.getSQLState(), g.getMessage());
-      System.exit(1);
     } catch (Exception f) {
       f.printStackTrace();
       System.exit(1);
@@ -119,7 +156,11 @@ public class DatabasePseudoTagProvider {
     cntWayModified = cntWayModified + 1;
     for (String key : dbTags.keySet()) {
       map.put(key, dbTags.get(key));
-      pseudoTagsFound.put(key, pseudoTagsFound.get(key) + 1L);
+      Long cnt = pseudoTagsFound.get(key);
+      if (cnt == null) {
+        cnt = 0L;
+      }
+      pseudoTagsFound.put(key, cnt + 1L);
     }
   }
 }