Merge branch 'master' into suspectscanner

brouter-server/src/main/java/btools/server/Area.java

@@ -0,0 +1,76 @@
+package btools.server;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class Area
+{
+  private List<Polygon> poslist  = new ArrayList<Polygon>();
+  private List<Polygon> neglist  = new ArrayList<Polygon>();
+
+  public static void main( String[] args ) throws IOException
+  {
+    Area a = new Area( new File( args[0] ) );
+    
+    System.out.println( args[1] + " is in " + args[0] + "=" + a.isInArea( Long.parseLong( args[1] ) ) );
+  }
+  
+  public Area( File f ) throws IOException
+  {
+    BufferedReader br = new BufferedReader( new FileReader( f ) );
+    br.readLine();
+    
+    for(;;)
+    {
+      String head = br.readLine();
+      if ( head == null || "END".equals( head ) )
+      {
+        break;
+      }
+      Polygon pol = new Polygon( br );
+      if ( head.startsWith( "!" ) )
+      {
+        neglist.add( pol );
+      }
+      else
+      {
+        poslist.add( pol );
+      }
+    }
+  }
+  
+  public boolean isInArea( long id )
+  {
+    for( int i=0; i<poslist.size(); i++)
+    {
+      if ( poslist.get(i).isInPolygon( id ) )
+      {
+        for( int j=0; j<neglist.size(); j++)
+        {
+          if ( neglist.get(i).isInPolygon( id ) )
+          {
+            return false;
+          }
+        }
+        return true;
+      }
+    }
+    return false;
+  }
+
+  public boolean isInBoundingBox( long id )
+  {
+    for( int i=0; i<poslist.size(); i++)
+    {
+      if ( poslist.get(i).isInBoundingBox( id ) )
+      {
+        return true;
+      }
+    }
+    return false;
+  }
+}

brouter-server/src/main/java/btools/server/Polygon.java

@@ -0,0 +1,89 @@
+package btools.server;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.StringTokenizer;
+
+public class Polygon
+{
+  private int[] ax;
+  private int[] ay;
+
+  private int minx = Integer.MAX_VALUE;
+  private int miny = Integer.MAX_VALUE;
+  private int maxx = Integer.MIN_VALUE;
+  private int maxy = Integer.MIN_VALUE;
+
+  public Polygon( BufferedReader br ) throws IOException
+  {
+    ArrayList<String> lines = new ArrayList<String>();
+  
+    for(;;)
+    {
+      String line = br.readLine();
+      if ( line == null || "END".equals( line ) )
+      {
+        break;
+      }
+      lines.add( line );
+    }
+    int n = lines.size();
+    ax = new int[n];
+    ay = new int[n];
+    for( int i=0; i<n; i++ )
+    {
+      String line = lines.get(i);
+      StringTokenizer tk = new StringTokenizer( line );
+      double lon = Double.parseDouble( tk.nextToken() );
+      double lat = Double.parseDouble( tk.nextToken() );
+      
+      int x = ax[i] = (int)(lon*1000000. + 180000000);
+      int y = ay[i] = (int)(lat*1000000. +  90000000);
+      
+      if ( x < minx ) minx = x;
+      if ( y < miny ) miny = y;
+      if ( x > maxx ) maxx = x;
+      if ( y > maxy ) maxy = y;
+    }
+  }
+  
+  public boolean isInPolygon( long id )
+  {
+    int x = (int) ( id >> 32 );
+    int y = (int) ( id & 0xffffffff );
+
+    if ( x < minx || x > maxx || y < miny || y > maxy )
+    {
+      return false;
+    }
+
+    int n = ax.length-1; // these are closed polygons
+
+    boolean inside = false;
+    int j = n - 1;
+    for (int i = 0 ;i < n ; j = i++)
+    {
+      if ( (ay[i] > y) != (ay[j] > y) )
+      {
+        long v = ax[j] - ax[i];
+        v *= y - ay[i];
+        v /= ay[j] - ay[i];
+        if ( x <= v +  ax[i])
+        {
+          inside = !inside;
+        }
+      }
+    }
+    return inside;
+  }
+
+  public boolean isInBoundingBox( long id )
+  {
+    int x = (int) ( id >> 32 );
+    int y = (int) ( id & 0xffffffff );
+
+    return x >= minx && x <= maxx && y >= miny && y <= maxy;
+  }
+
+}

brouter-server/src/main/java/btools/server/SuspectManager.java

@@ -5,11 +5,23 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
 import java.util.StringTokenizer;
 import java.util.TreeSet;
 
 public class SuspectManager extends Thread
 {
+  private static SimpleDateFormat dfTimestampZ = new SimpleDateFormat( "yyyy-MM-dd'T'HH:mm:ss" );
+
+  private static String formatZ( Date date )
+  {
+    synchronized( dfTimestampZ )
+    {
+      return dfTimestampZ.format( date );
+    }
+  }
+
   private static String formatAge( File f )
   {
     long age = System.currentTimeMillis() - f.lastModified();
@@ -43,38 +55,74 @@ public class SuspectManager extends Thread
   public static void process( String url, BufferedWriter bw ) throws IOException
   {
     bw.write( "<html><body>\n" );
+    bw.write( "BRouter suspect manager. <a href=\"http://brouter.de/brouter/suspect_manager_help.html\">Help</a><br><br>\n" );
 
     StringTokenizer tk = new StringTokenizer( url, "/" );
     tk.nextToken();
     tk.nextToken();
     long id = 0L;
-    String country = null;
+    String country = "";
+    String filter = null;
 
-    if ( tk.hasMoreTokens() )
+    while ( tk.hasMoreTokens() )
     {
-      String ctry = tk.nextToken();
+      String c = tk.nextToken();
-      if ( new File( "suspects/suspects_" + ctry + ".txt" ).exists() )
+      if ( "all".equals( c ) || "new".equals( c ) )
       {
-        country = ctry;
+        filter = c;
+        break;
       }
+      country += "/" + c;
     }
-    if ( country == null ) // generate country list
+    
+    if ( filter == null ) // generate country list
     {
-      File[] files = new File( "suspects" ).listFiles();
+      bw.write( "<table>\n" );
+      File countryParent = new File( "worldpolys" + country );
+      File[] files = countryParent.listFiles();
       TreeSet<String> names = new TreeSet<String>();
       for ( File f : files )
       {
         String name = f.getName();
-        if ( name.startsWith( "suspects_" ) && name.endsWith( ".txt" ) )
+        if ( name.endsWith( ".poly" ) )
         {
-          names.add( name.substring( 9, name.length() - 4 ) );
+          names.add( name.substring( 0, name.length() - 5 ) );
         }
       }
-      for ( String ctry : names )
+      for ( String c : names )
       {
-        String url2 = "/brouter/suspects/" + ctry;
+        String url2 = "/brouter/suspects" + country + "/" + c;
-        bw.write( "<a href=\"" + url2 + "\">" + ctry + "</a><br>\n" );
+        String linkNew = "<td>&nbsp;<a href=\"" + url2 + "/new\">new</a>&nbsp;</td>";
+        String linkAll = "<td>&nbsp;<a href=\"" + url2 + "/all\">all</a>&nbsp;</td>";
+        
+        String linkSub = "";
+        if ( new File( countryParent, c ).exists() )
+        {
+          linkSub = "<td>&nbsp;<a href=\"" + url2 + "\">sub-regions</a>&nbsp;</td>";
+        }
+        bw.write( "<tr><td>" + c + "</td>" + linkNew + linkAll + linkSub + "\n" );
       }
+      bw.write( "</table>\n" );
+      bw.write( "</body></html>\n" );
+      bw.flush();
+      return;
+    }
+
+    File polyFile = new File( "worldpolys" + country + ".poly" );
+    if ( !polyFile.exists() )
+    {
+      bw.write( "polygon file for country '" + country + "' not found\n" );
+      bw.write( "</body></html>\n" );
+      bw.flush();
+      return;
+    }
+    
+    Area polygon = new Area( polyFile );
+
+    File suspectFile = new File( "worldsuspects.txt" );
+    if ( !suspectFile.exists() )
+    {
+      bw.write( "suspect file worldsuspects.txt not found\n" );
       bw.write( "</body></html>\n" );
       bw.flush();
       return;
@@ -96,46 +144,45 @@ public class SuspectManager extends Thread
 
     if ( showWatchList )
     {
-      File suspects = new File( "suspects/suspects_" + country + ".txt" );
       bw.write( "watchlist for " + country + "\n" );
       bw.write( "<br><a href=\"/brouter/suspects\">back to country list</a><br><br>\n" );
-      if ( suspects.exists() )
+      BufferedReader r = new BufferedReader( new FileReader( suspectFile ) );
+      for ( ;; )
       {
-        BufferedReader r = new BufferedReader( new FileReader( suspects ) );
+        String line = r.readLine();
-        for ( ;; )
+        if ( line == null )
-        {
+          break;
-          String line = r.readLine();
+        StringTokenizer tk2 = new StringTokenizer( line );
-          if ( line == null )
+        id = Long.parseLong( tk2.nextToken() );
-            break;
+        String countryId = country + "/" + filter + "/" + id;
-          StringTokenizer tk2 = new StringTokenizer( line );
-          id = Long.parseLong( tk2.nextToken() );
-          String countryId = country + "/" + id;
 
-          if ( new File( "falsepositives/" + id ).exists() )
+        if ( new File( "falsepositives/" + id ).exists() )
-          {
+        {
-            continue; // known false positive
+          continue; // known false positive
-          }
-          File fixedEntry = new File( "fixedsuspects/" + id );
-          File confirmedEntry = new File( "confirmednegatives/" + id );
-          if ( !( fixedEntry.exists() && confirmedEntry.exists() ) )
-          {
-            continue;
-          }
-          long age = System.currentTimeMillis() - confirmedEntry.lastModified();
-          if ( age / 1000 < 3600 * 24 * 8 )
-          {
-            continue;
-          }
-          String hint = "&nbsp;&nbsp;&nbsp;confirmed " + formatAge( confirmedEntry ) + " ago";
-          int ilon = (int) ( id >> 32 );
-          int ilat = (int) ( id & 0xffffffff );
-          double dlon = ( ilon - 180000000 ) / 1000000.;
-          double dlat = ( ilat - 90000000 ) / 1000000.;
-          String url2 = "/brouter/suspects/" + countryId;
-          bw.write( "<a href=\"" + url2 + "\">" + dlon + "," + dlat + "</a>" + hint + "<br>\n" );
         }
-        r.close();
+        File confirmedEntry = new File( "confirmednegatives/" + id );
+        if ( !( isFixed( id, suspectFile ) && confirmedEntry.exists() ) )
+        {
+          continue;
+        }
+        long age = System.currentTimeMillis() - confirmedEntry.lastModified();
+        if ( age / 1000 < 3600 * 24 * 8 )
+        {
+          continue;
+        }
+        if ( !polygon.isInArea( id ) )
+        {
+          continue; // not in selected polygon
+        }
+        String hint = "&nbsp;&nbsp;&nbsp;confirmed " + formatAge( confirmedEntry ) + " ago";
+        int ilon = (int) ( id >> 32 );
+        int ilat = (int) ( id & 0xffffffff );
+        double dlon = ( ilon - 180000000 ) / 1000000.;
+        double dlat = ( ilat - 90000000 ) / 1000000.;
+        String url2 = "/brouter/suspects" + countryId;
+        bw.write( "<a href=\"" + url2 + "\">" + dlon + "," + dlat + "</a>" + hint + "<br>\n" );
       }
+      r.close();
       bw.write( "</body></html>\n" );
       bw.flush();
       return;
@@ -150,7 +197,7 @@ public class SuspectManager extends Thread
         int wps = NearRecentWps.count( id );
         if ( wps < 8 )
         {
-          message = "marking false-positive requires at least 10 recent nearby waypoints from BRouter-Web, found: " + wps;
+          message = "marking false-positive requires at least 8 recent nearby waypoints from BRouter-Web, found: " + wps;
         }
         else
         {
@@ -172,13 +219,26 @@ public class SuspectManager extends Thread
       }
       if ( "fixed".equals( command ) )
       {
-        new File( "fixedsuspects/" + id ).createNewFile();
+        File fixedMarker = new File( "fixedsuspects/" + id );
+        if ( !fixedMarker.exists() )
+        {
+          fixedMarker.createNewFile();
+        }
         id = 0L;
+
+        int hideDays = 0;
+        if ( tk.hasMoreTokens() )
+        {
+          String param = tk.nextToken();
+          hideDays = Integer.parseInt( param );
+          fixedMarker.setLastModified( System.currentTimeMillis() + hideDays*86400000L );
+        }
+        fixedMarker.setLastModified( System.currentTimeMillis() + hideDays*86400000L );
       }
     }
     if ( id != 0L )
     {
-      String countryId = country + "/" + id;
+      String countryId = country + "/" + filter + "/" + id;
 
       int ilon = (int) ( id >> 32 );
       int ilat = (int) ( id & 0xffffffff );
@@ -195,7 +255,7 @@ public class SuspectManager extends Thread
       }
 
       String url1 = "http://brouter.de/brouter-web/#zoom=18&lat=" + dlat + "&lon=" + dlon
-          + "&layer=OpenStreetMap&lonlats=" + dlon + "," + dlat + "&profile=" + profile;
+          + "&lonlats=" + dlon + "," + dlat + "&profile=" + profile;
 
       // String url1 = "http://localhost:8080/brouter-web/#map=18/" + dlat + "/"
       // + dlon + "/Mapsforge Tile Server&lonlats=" + dlon + "," + dlat;
@@ -204,9 +264,18 @@ public class SuspectManager extends Thread
 
       double slon = 0.00156;
       double slat = 0.001;
-      String url3 = "http://osmose.openstreetmap.fr/de/josm_proxy?load_and_zoom?left=" + ( dlon - slon )
+      String url3 = "http://127.0.0.1:8111/load_and_zoom?left=" + ( dlon - slon )
           + "&bottom=" + ( dlat - slat ) + "&right=" + ( dlon + slon ) + "&top=" + ( dlat + slat );
 
+      Date weekAgo = new Date( System.currentTimeMillis() - 604800000L );
+      String url4a = "https://overpass-turbo.eu/?Q=[date:&quot;" + formatZ( weekAgo ) + "Z&quot;];way[highway]({{bbox}});out meta geom;&C="
+                  + dlat + ";" + dlon + ";18&R";
+
+      String url4b = "https://overpass-turbo.eu/?Q=(node(around%3A1%2C%7B%7Bcenter%7D%7D)-%3E.n%3Bway(bn.n)%3Brel(bn.n%3A%22via%22)%5Btype%3Drestriction%5D%3B)%3Bout%20meta%3B%3E%3Bout%20skel%20qt%3B&C="
+                  + dlat + ";" + dlon + ";18&R";
+
+      String url5 = "https://tyrasd.github.io/latest-changes/#16/" + dlat + "/" + dlon;
+
       if ( message != null )
       {
         bw.write( "<strong>" + message + "</strong><br><br>\n" );
@@ -214,89 +283,113 @@ public class SuspectManager extends Thread
       bw.write( "<a href=\"" + url1 + "\">Open in BRouter-Web</a><br><br>\n" );
       bw.write( "<a href=\"" + url2 + "\">Open in OpenStreetmap</a><br><br>\n" );
       bw.write( "<a href=\"" + url3 + "\">Open in JOSM (via remote control)</a><br><br>\n" );
-      File fixedEntry = new File( "fixedsuspects/" + id );
+      bw.write( "Overpass: <a href=\"" + url4a + "\">minus one week</a> &nbsp;&nbsp; <a href=\"" + url4b + "\">node context</a><br><br>\n" );
-      if ( fixedEntry.exists() )
+      bw.write( "<a href=\"" + url5 + "\">Open in Latest-Changes / last week</a><br><br>\n" );
+      bw.write( "<br>\n" );
+      if ( isFixed( id, suspectFile ) )
       {
-        bw.write( "<br><br><a href=\"/brouter/suspects/" + country + "/watchlist\">back to watchlist</a><br><br>\n" );
+        bw.write( "<br><br><a href=\"/brouter/suspects/" + country + "/" + filter + "/watchlist\">back to watchlist</a><br><br>\n" );
       }
       else
       {
-        bw.write( "<a href=\"/brouter/suspects/" + countryId + "/falsepositive\">mark false positive (=not an issue)</a><br><br>\n" );
+        bw.write( "<a href=\"/brouter/suspects" + countryId + "/falsepositive\">mark false positive (=not an issue)</a><br><br>\n" );
         File confirmedEntry = new File( "confirmednegatives/" + id );
         if ( confirmedEntry.exists() )
         {
-          bw.write( "<a href=\"/brouter/suspects/" + countryId + "/fixed\">mark as fixed</a><br><br>\n" );
+          String prefix = "<a href=\"/brouter/suspects" + countryId + "/fixed";
+          String prefix2 = " &nbsp;&nbsp;" + prefix;
+          bw.write( prefix + "\">mark as fixed</a><br><br>\n" );
+          bw.write( "hide for " );
+          bw.write( prefix2 + "/7\">1 week</a>" );
+          bw.write( prefix2 + "/30\">1 month</a>" );
+          bw.write( prefix2 + "/91\">3 months</a>" );
+          bw.write( prefix2 + "/182\">6 months</a><br><br>\n" );
         }
         else
         {
-          bw.write( "<a href=\"/brouter/suspects/" + countryId + "/confirm\">mark as a confirmed issue</a><br><br>\n" );
+          bw.write( "<a href=\"/brouter/suspects" + countryId + "/confirm\">mark as a confirmed issue</a><br><br>\n" );
         }
-        bw.write( "<br><br><a href=\"/brouter/suspects/" + country + "\">back to issue list</a><br><br>\n" );
+        bw.write( "<br><br><a href=\"/brouter/suspects" + country + "/" + filter + "\">back to issue list</a><br><br>\n" );
       }
     }
     else
     {
-      File suspects = new File( "suspects/suspects_" + country + ".txt" );
+      bw.write( filter + " suspect list for " + country + "\n" );
-      bw.write( "suspect list for " + country + "\n" );
+      bw.write( "<br><a href=\"/brouter/suspects" + country + "/" + filter + "/watchlist\">see watchlist</a>\n" );
-      bw.write( "<br><a href=\"/brouter/suspects/" + country + "/watchlist\">see watchlist</a>\n" );
       bw.write( "<br><a href=\"/brouter/suspects\">back to country list</a><br><br>\n" );
-      if ( suspects.exists() )
+      int maxprio = 0;
+      for ( int pass = 1; pass <= 2; pass++ )
       {
-        int maxprio = 0;
+        if ( pass == 2 )
-        for ( int pass = 1; pass <= 2; pass++ )
         {
-          if ( pass == 2 )
+          bw.write( "current level: " + getLevelDecsription( maxprio ) + "<br><br>\n" );
-          {
-            bw.write( "current level: " + getLevelDecsription( maxprio ) + "<br><br>\n" );
-          }
-
-          BufferedReader r = new BufferedReader( new FileReader( suspects ) );
-          for ( ;; )
-          {
-            String line = r.readLine();
-            if ( line == null )
-              break;
-            StringTokenizer tk2 = new StringTokenizer( line );
-            id = Long.parseLong( tk2.nextToken() );
-            int prio = Integer.parseInt( tk2.nextToken() );
-            prio = ( ( prio + 1 ) / 2 ) * 2; // normalize (no link prios)
-            String countryId = country + "/" + id;
-
-            String hint = "";
-
-            if ( new File( "falsepositives/" + id ).exists() )
-            {
-              continue; // known false positive
-            }
-            if ( new File( "fixedsuspects/" + id ).exists() )
-            {
-              continue; // known fixed
-            }
-            if ( pass == 1 )
-            {
-              if ( prio > maxprio )
-                maxprio = prio;
-              continue;
-            }
-            else
-            {
-              if ( prio < maxprio )
-                continue;
-            }
-            File confirmedEntry = new File( "confirmednegatives/" + id );
-            if ( confirmedEntry.exists() )
-            {
-              hint = "&nbsp;&nbsp;&nbsp;confirmed " + formatAge( confirmedEntry ) + " ago";
-            }
-            int ilon = (int) ( id >> 32 );
-            int ilat = (int) ( id & 0xffffffff );
-            double dlon = ( ilon - 180000000 ) / 1000000.;
-            double dlat = ( ilat - 90000000 ) / 1000000.;
-            String url2 = "/brouter/suspects/" + countryId;
-            bw.write( "<a href=\"" + url2 + "\">" + dlon + "," + dlat + "</a>" + hint + "<br>\n" );
-          }
-          r.close();
         }
+
+        BufferedReader r = new BufferedReader( new FileReader( suspectFile ) );
+        for ( ;; )
+        {
+          String line = r.readLine();
+          if ( line == null )
+            break;
+          StringTokenizer tk2 = new StringTokenizer( line );
+          String idString = tk2.nextToken();
+
+          int prio = Integer.parseInt( tk2.nextToken() );
+          prio = ( ( prio + 1 ) / 2 ) * 2; // normalize (no link prios)
+
+          if ( pass == 1 )
+          {
+            if ( prio <= maxprio )
+              continue;
+          }
+          else
+          {
+            if ( prio < maxprio )
+              continue;
+          }
+
+          id = Long.parseLong( idString );
+
+          if ( !polygon.isInBoundingBox( id ) )
+          {
+            continue; // not in selected polygon (pre-check)
+          }
+          if ( new File( "falsepositives/" + id ).exists() )
+          {
+            continue; // known false positive
+          }
+          if ( isFixed( id, suspectFile ) )
+          {
+            continue; // known fixed
+          }
+          if ( "new".equals( filter ) && new File( "suspectarchive/" + id ).exists() )
+          {
+            continue; // known fixed
+          }
+          if ( !polygon.isInArea( id ) )
+          {
+            continue; // not in selected polygon
+          }
+          if ( pass == 1 )
+          {
+            maxprio = prio;
+            continue;
+          }
+          String countryId = country + "/" + filter + "/" + id;
+          File confirmedEntry = new File( "confirmednegatives/" + id );
+          String hint = "";
+          if ( confirmedEntry.exists() )
+          {
+            hint = "&nbsp;&nbsp;&nbsp;confirmed " + formatAge( confirmedEntry ) + " ago";
+          }
+          int ilon = (int) ( id >> 32 );
+          int ilat = (int) ( id & 0xffffffff );
+          double dlon = ( ilon - 180000000 ) / 1000000.;
+          double dlat = ( ilat - 90000000 ) / 1000000.;
+          String url2 = "/brouter/suspects" + countryId;
+          bw.write( "<a href=\"" + url2 + "\">" + dlon + "," + dlat + "</a>" + hint + "<br>\n" );
+        }
+        r.close();
       }
     }
     bw.write( "</body></html>\n" );
@@ -304,4 +397,11 @@ public class SuspectManager extends Thread
     return;
   }
   
+
+  private static boolean isFixed( long id, File suspectFile )
+  {
+    File fixedEntry = new File( "fixedsuspects/" + id );
+    return fixedEntry.exists() && fixedEntry.lastModified() > suspectFile.lastModified();
+  }
+  
 }