Merge branch 'abrensch:master' into master

brouter-core/src/main/java/btools/router/MessageData.java

@@ -55,7 +55,9 @@ final class MessageData implements Cloneable
          + "\t" + linknodecost
          + "\t" + linkinitcost
          + "\t" + wayKeyValues
-         + "\t" + ( nodeKeyValues == null ? "" : nodeKeyValues );
+         + "\t" + ( nodeKeyValues == null ? "" : nodeKeyValues )
+            + "\t" + ((int)time)
+            + "\t" + ((int)energy);
   }
 
   void add( MessageData d )

brouter-core/src/main/java/btools/router/OsmPath.java

@@ -139,7 +139,10 @@ abstract class OsmPath implements OsmLinkHolder
   protected void addAddionalPenalty(OsmTrack refTrack, boolean detailMode, OsmPath origin, OsmLink link, RoutingContext rc )
   {
     byte[] description = link.descriptionBitmap;
-    if ( description == null ) throw new IllegalArgumentException( "null description for: " + link );
+    if ( description == null )
+    {
+      return; // could be a beeline path
+    }
 
     boolean recordTransferNodes = detailMode || rc.countTraffic;
 

brouter-core/src/main/java/btools/router/OsmTrack.java

@@ -19,8 +19,11 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.StringWriter;
+import java.text.DecimalFormat;
+import java.text.NumberFormat;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 
 import btools.mapaccess.MatchedWaypoint;
 import btools.mapaccess.OsmPos;
@@ -31,7 +34,7 @@ import btools.util.StringUtils;
 public final class OsmTrack
 {
   // csv-header-line
-  private static final String MESSAGES_HEADER = "Longitude\tLatitude\tElevation\tDistance\tCostPerKm\tElevCost\tTurnCost\tNodeCost\tInitialCost\tWayTags\tNodeTags";
+  private static final String MESSAGES_HEADER = "Longitude\tLatitude\tElevation\tDistance\tCostPerKm\tElevCost\tTurnCost\tNodeCost\tInitialCost\tWayTags\tNodeTags\tTime\tEnergy";
 
   public MatchedWaypoint endPoint;
   public long[] nogoChecksums;
@@ -719,6 +722,8 @@ public final class OsmTrack
 
   public String formatAsGeoJson()
   {
+    int turnInstructionMode = voiceHints != null ? voiceHints.turnInstructionMode : 0;
+
     StringBuilder sb = new StringBuilder( 8192 );
 
     sb.append( "{\n" );
@@ -740,7 +745,20 @@ public final class OsmTrack
       sb.append( "        \"voicehints\": [\n" );
       for( VoiceHint hint: voiceHints.list )
       {
-        sb.append( "          [" ).append( hint.indexInTrack ).append( ',' ).append( hint.getCommand() ).append( ',' ).append( hint.getExitNumber() ).append( "],\n" );
+        sb.append( "          [" );
+        sb.append( hint.indexInTrack );
+        sb.append( ',' ).append( hint.getCommand() );
+        sb.append( ',' ).append( hint.getExitNumber() );
+        sb.append( ',' ).append( hint.distanceToNext );
+        sb.append( ',' ).append( (int) hint.angle );
+
+        // not always include geometry because longer and only needed for comment style
+        if ( turnInstructionMode == 4 ) // comment style
+        {
+          sb.append( ",\"" ).append( hint.formatGeometry() ).append( "\"" );
+        }
+
+        sb.append( "],\n" );
       }
       sb.deleteCharAt( sb.lastIndexOf( "," ) );
       sb.append( "        ],\n" );
@@ -755,7 +773,7 @@ public final class OsmTrack
         {
           sb.append( "          [" ).append( sp.get(i) ).append( i> 0 ? "],\n" : "]\n" );
         }
-        sb.append( "        ]\n" );
+        sb.append( "        ],\n" );
       }
     }
     else // ... otherwise traditional message list
@@ -767,9 +785,24 @@ public final class OsmTrack
         sb.append( "          [\"" ).append( m.replaceAll( "\t", "\", \"" ) ).append( "\"],\n" );
       }
       sb.deleteCharAt( sb.lastIndexOf( "," ) );
-      sb.append( "        ]\n" );
+      sb.append( "        ],\n" );
     }
+
+    if ( getTotalSeconds() > 0 ) {
+      sb.append( "        \"times\": [" );
+      DecimalFormat decimalFormat = (DecimalFormat) NumberFormat.getInstance( Locale.ENGLISH );
+      decimalFormat.applyPattern( "0.###" );
+      for ( OsmPathElement n : nodes ) {
+        sb.append( decimalFormat.format( n.getTime() ) ).append( "," );
+      }
+      sb.deleteCharAt( sb.lastIndexOf( "," ) );
+      sb.append( "]\n" );
+    } else {
+      sb.deleteCharAt( sb.lastIndexOf( "," ) );
+    }
+
     sb.append( "      },\n" );
+
     if ( iternity != null )
     {
       sb.append( "      \"iternity\": [\n" );

brouter-core/src/main/java/btools/router/RoutingContext.java

@@ -16,6 +16,7 @@ import btools.expressions.BExpressionContextNode;
 import btools.expressions.BExpressionContextWay;
 import btools.mapaccess.GeometryDecoder;
 import btools.mapaccess.OsmLink;
+import btools.mapaccess.OsmNode;
 import btools.util.CheapAngleMeter;
 import btools.util.CheapRuler;
 
@@ -191,6 +192,7 @@ public final class RoutingContext
   public List<OsmNodeNamed> poipoints;
 
   public List<OsmNodeNamed> nogopoints = null;
+  private List<OsmNodeNamed> nogopoints_all = null; // full list not filtered for wayoints-in-nogos
   private List<OsmNodeNamed> keepnogopoints = null;
   private OsmNodeNamed pendingEndpoint = null;
 
@@ -258,14 +260,29 @@ public final class RoutingContext
     }
   }
 
-  public void cleanNogolist( List<OsmNodeNamed> waypoints )
+  /**
+   * restore the full nogolist previously saved by cleanNogoList
+   */
+  public void restoreNogoList()
   {
+    nogopoints = nogopoints_all;
+  }
+
+  /**
+   * clean the nogolist (previoulsy saved by saveFullNogolist())
+   * by removing nogos with waypoints within
+   * 
+   * @return true if all wayoints are all in the same (full-weigth) nogo area (triggering bee-line-mode)
+   */
+  public void cleanNogoList( List<OsmNode> waypoints )
+  {
+    nogopoints_all = nogopoints;
     if ( nogopoints == null ) return;
     List<OsmNodeNamed> nogos = new ArrayList<OsmNodeNamed>();
     for( OsmNodeNamed nogo : nogopoints )
     {
       boolean goodGuy = true;
-      for( OsmNodeNamed wp : waypoints )
+      for( OsmNode wp : waypoints )
       {
         if ( wp.calcDistance( nogo ) < nogo.radius
             && (!(nogo instanceof OsmNogoPolygon)
@@ -274,7 +291,6 @@ public final class RoutingContext
                         : ((OsmNogoPolygon)nogo).isOnPolyline(wp.ilon, wp.ilat))))
         {
           goodGuy = false;
-          break;
         }
       }
       if ( goodGuy ) nogos.add( nogo );
@@ -282,6 +298,31 @@ public final class RoutingContext
     nogopoints = nogos.isEmpty() ? null : nogos;
   }
 
+  public boolean allInOneNogo( List<OsmNode> waypoints )
+  {
+    if ( nogopoints == null ) return false;
+    boolean allInTotal = false;
+    for( OsmNodeNamed nogo : nogopoints )
+    {
+      boolean allIn = Double.isNaN( nogo.nogoWeight );
+      for( OsmNode wp : waypoints )
+      {
+        int dist = wp.calcDistance( nogo );
+        if ( dist < nogo.radius
+            && (!(nogo instanceof OsmNogoPolygon)
+                || (((OsmNogoPolygon)nogo).isClosed
+                    ? ((OsmNogoPolygon)nogo).isWithin(wp.ilon, wp.ilat)
+                        : ((OsmNogoPolygon)nogo).isOnPolyline(wp.ilon, wp.ilat))))
+        {
+          continue;
+        }
+        allIn = false;
+      }
+      allInTotal |= allIn;
+    }
+    return allInTotal;
+  }
+
   public long[] getNogoChecksums()
   {
     long[] cs = new long[3];

brouter-core/src/main/java/btools/router/RoutingEngine.java

@@ -151,9 +151,6 @@ public class RoutingEngine extends Thread
   {
     try
     {
-      // delete nogos with waypoints in them
-      routingContext.cleanNogolist( waypoints );
-
       startTime = System.currentTimeMillis();
       long startTime0 = startTime;
       this.maxRunningTime = maxRunningTime;
@@ -459,6 +456,29 @@ public class RoutingEngine extends Thread
   }
 
   private OsmTrack searchTrack( MatchedWaypoint startWp, MatchedWaypoint endWp, OsmTrack nearbyTrack, OsmTrack refTrack )
+  {
+    // remove nogos with waypoints inside
+    try
+    {
+      List<OsmNode> wpts2 = new ArrayList<OsmNode>();
+      wpts2.add( startWp.waypoint );
+      wpts2.add( endWp.waypoint );
+      boolean calcBeeline = routingContext.allInOneNogo(wpts2);
+      
+      if ( !calcBeeline ) return searchRoutedTrack( startWp, endWp, nearbyTrack, refTrack );
+      
+      // we want a beeline-segment
+      OsmPath path = routingContext.createPath( new OsmLink( null, startWp.crosspoint ) );
+      path = routingContext.createPath( path, new OsmLink( startWp.crosspoint, endWp.crosspoint ), null, false );
+      return compileTrack( path, false );
+    }
+    finally
+    {
+      routingContext.restoreNogoList();
+    }
+  }
+
+  private OsmTrack searchRoutedTrack( MatchedWaypoint startWp, MatchedWaypoint endWp, OsmTrack nearbyTrack, OsmTrack refTrack )
   {
     OsmTrack track = null;
     double[] airDistanceCostFactors = new double[]{ routingContext.pass1coefficient, routingContext.pass2coefficient };
@@ -650,6 +670,11 @@ public class RoutingEngine extends Thread
   {
     try
     {
+      List<OsmNode> wpts2 = new ArrayList<OsmNode>();
+      if ( startWp != null ) wpts2.add( startWp.waypoint );
+      if ( endWp != null ) wpts2.add( endWp.waypoint );
+      routingContext.cleanNogoList(wpts2);
+    
       boolean detailed = guideTrack != null;
       resetCache( detailed );
       nodesCache.nodesMap.cleanupMode = detailed ? 0 : ( routingContext.considerTurnRestrictions ? 2 : 1 );
@@ -657,6 +682,7 @@ public class RoutingEngine extends Thread
     }
     finally
     {
+      routingContext.restoreNogoList();
       nodesCache.clean( false ); // clean only non-virgin caches
     }
   }

brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java

@@ -1,5 +1,5 @@
 /**
- * Proof of concept for delta rd5's
+ * Manage rd5 diff-file creation
  *
  * @author ab
  */
@@ -56,10 +56,10 @@ final public class Rd5DiffManager
       File specificNewDiffs = new File( newDiffDir, basename );
       specificNewDiffs.mkdirs();
       
-      String diffFileName = md5 + ".rd5diff";
+      String diffFileName = md5 + ".df5";
       File diffFile = new File( specificNewDiffs, diffFileName );
       
-      String dummyDiffFileName = md5New + ".rd5diff";
+      String dummyDiffFileName = md5New + ".df5";
       File dummyDiffFile = new File( specificNewDiffs, dummyDiffFileName );
       dummyDiffFile.createNewFile();
 
@@ -73,7 +73,7 @@ final public class Rd5DiffManager
         File[] oldDiffs = specificOldDiffs.listFiles();
         for( File od : oldDiffs )
         {
-          if ( !od.getName().endsWith( ".rd5diff" ) )
+          if ( !od.getName().endsWith( ".df5" ) )
           {
             continue;
           }

brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java

@@ -1,5 +1,5 @@
 /**
- * Proof of concept for delta rd5's
+ * Calculate, add or merge rd5 delta files
  *
  * @author ab
  */
@@ -31,9 +31,9 @@ final public class Rd5DiffTool implements ProgressListener
       return;
     }
 
-    if ( args[1].endsWith( ".rd5diff" ) )
+    if ( args[1].endsWith( ".df5" ) )
     {
-      if ( args[0].endsWith( ".rd5diff" ) )
+      if ( args[0].endsWith( ".df5" ) )
       {
         addDeltas( new File( args[0] ),new File( args[1] ), new File( args[2] ) );
       }
@@ -152,6 +152,7 @@ final public class Rd5DiffTool implements ProgressListener
     DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
     DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) );
     DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
+    MCOutputStream mcOut = new MCOutputStream( dos, abBuf1 );
 
     // copy header to outfile
     long[] fileIndex1 = readFileIndex( dis1, null );
@@ -175,10 +176,6 @@ final public class Rd5DiffTool implements ProgressListener
            byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false );
            byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, false );
 
-           if ( ab2 == null )
-           {
-             continue; // empty target tile
-           }
            MicroCache mc;
            if ( Arrays.equals( ab1, ab2 ) )
            {
@@ -191,17 +188,10 @@ final public class Rd5DiffTool implements ProgressListener
              mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 );
              mc.calcDelta( mc1, mc2 );
            }
-              
-           if ( mc.getSize() == 0 )
-           {
-             dos.writeInt( 0 );
-           }
-           else
-           {
-             int len = mc.encodeMicroCache( abBuf1 );
-             dos.writeInt( len );
-             dos.write( abBuf1, 0, len );
 
+           int len = mcOut.writeMC( mc );
+           if ( len > 0 )
+           {
              bytesDiff += len;
              nodesDiff += mc.getSize();
              diffedTiles++;
@@ -244,7 +234,9 @@ final public class Rd5DiffTool implements ProgressListener
              */
           }
         }
+        mcOut.finish();
       }
+
       // write any remaining data to the output file
       for(;;)
       {
@@ -324,10 +316,12 @@ final public class Rd5DiffTool implements ProgressListener
     try
     {
       DataBuffers dataBuffers = new DataBuffers();
+      MCInputStream mcIn = new MCInputStream( dis2, dataBuffers );
+
       for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ )
       {
-         boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx );
+         boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx ); // has the basefile data
-         boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx );
+         boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx ); // has the *result* data
 
 //         boolean hasDataCmp = getTileStart( fileIndexCmp, subFileIdx ) < getTileEnd( fileIndexCmp, subFileIdx );
 
@@ -352,11 +346,8 @@ final public class Rd5DiffTool implements ProgressListener
            }
 
            byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false );
-           byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true );
+           MicroCache mc2 = mcIn.readMC();
-           if ( ab2 == null )
+           int targetSize = posIdx2 == null ? 0 : getPosIdx( posIdx2, tileIdx ) - getPosIdx( posIdx2, tileIdx-1 );
-           {
-             continue; // no target tile expected
-           }
 
 /*         int targetSizeCmp = getPosIdx( posIdxCmp, tileIdx ) - getPosIdx( posIdxCmp, tileIdx-1 );
            if ( targetSizeCmp != targetSize ) throw new IllegalArgumentException( "target size mismatch: "+ targetSize + "," + targetSizeCmp );
@@ -365,21 +356,35 @@ final public class Rd5DiffTool implements ProgressListener
 */
 
            // no-delta shortcut: just copy base data
-           if ( ab2.length == 0 )
+           if ( mc2.getSize() == 0 )
            {
              if ( ab1 != null )
              {
                dos.write( ab1 );
              }
+             int newTargetSize = ab1 == null ? 0 : ab1.length;
+             if ( targetSize != newTargetSize )
+             {
+               throw new RuntimeException( "size mismatch at " +  subFileIdx + "/" + tileIdx + " " + targetSize + "!=" + newTargetSize );
+             }
              continue;
            }
 
            // this is the real delta case (using decode->delta->encode )
+
            MicroCache mc1 = createMicroCache( ab1, dataBuffers );
-           MicroCache mc2 = createMicroCache( ab2, dataBuffers );
 
            MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 );
            mc.addDelta( mc1, mc2, false );
+           
+           if ( mc.size() == 0 )
+           {
+             if ( targetSize != 0 )
+             {
+               throw new RuntimeException( "size mismatch at " +  subFileIdx + "/" + tileIdx + " " + targetSize + ">0" );
+             }
+             continue;
+           }
 
            int len = mc.encodeMicroCache( abBuf1 );
 
@@ -403,8 +408,13 @@ final public class Rd5DiffTool implements ProgressListener
 */
            dos.write( abBuf1, 0, len );
            dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 );
+           if ( targetSize != len+4 )
+           {
+             throw new RuntimeException( "size mismatch at " +  subFileIdx + "/" + tileIdx + " " + targetSize + "<>" + (len+4) );
+           }
 
         }
+        mcIn.finish();
       }
       // write any remaining data to the output file
       for(;;)
@@ -529,7 +539,7 @@ final public class Rd5DiffTool implements ProgressListener
     DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) );
     DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
 
-    // copy header to outfile
+    // copy subfile-header to outfile
     long[] fileIndex1 = readFileIndex( dis1, null );
     long[] fileIndex2 = readFileIndex( dis2, dos );
 
@@ -538,51 +548,37 @@ final public class Rd5DiffTool implements ProgressListener
     try
     {
       DataBuffers dataBuffers = new DataBuffers();
+      MCInputStream mcIn1 = new MCInputStream( dis1, dataBuffers );
+      MCInputStream mcIn2 = new MCInputStream( dis2, dataBuffers );
+      MCOutputStream mcOut = new MCOutputStream( dos, abBuf1 );
+
       for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ )
       {
+         // copy tile-header to outfile
          boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx );
          boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx );
-
          int[] posIdx1 = hasData1 ? readPosIndex( dis1, null ) : null;
          int[] posIdx2 = hasData2 ? readPosIndex( dis2, dos ) : null;
 
          for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ )
          {
-           byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, true );
+           MicroCache mc1 = mcIn1.readMC();
-           byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true );
+           MicroCache mc2 = mcIn2.readMC();
-           if ( ab2 == null )
+           MicroCache mc;
+           if ( mc1.getSize() == 0 && mc2.getSize() == 0 )
            {
-             continue; // no target tile expected
+             mc = mc1;
            }
-
+           else
-           // no-delta shortcut: just copy base data
-           if ( ab2.length == 0 )
            {
-             if ( ab1 == null )
+             mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 );
-             {
+             mc.addDelta( mc1, mc2, true );
-               dos.writeInt( 0 );
-             }
-             else
-             {
-               dos.writeInt( ab1.length );
-               dos.write( ab1 );
-             }
-             continue;
            }
-
+           mcOut.writeMC( mc );
-           // this is the real delta case (using decode->add->encode )
-           MicroCache mc1 = createMicroCache( ab1, dataBuffers );
-           MicroCache mc2 = createMicroCache( ab2, dataBuffers );
-
-           MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 );
-           mc.addDelta( mc1, mc2, true );
-
-           int len = mc.encodeMicroCache( abBuf1 );
-
-           dos.writeInt( len+4 );
-           dos.write( abBuf1, 0, len );
-           dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 );
         }
+        mcIn1.finish();
+        mcIn2.finish();
+        mcOut.finish();
       }
       // write any remaining data to the output file
       for(;;)
@@ -706,5 +702,83 @@ final public class Rd5DiffTool implements ProgressListener
     }
   }
 
+  private static class MCOutputStream
+  {
+    private DataOutputStream dos;
+    private byte[] buffer;
+    private short skips = 0;
+
+    public MCOutputStream(DataOutputStream dos, byte[] buffer)
+    {
+      this.dos = dos;
+      this.buffer = buffer;
+    }
+
+    public int writeMC(MicroCache mc) throws Exception
+    {
+      if ( mc.getSize() == 0 )
+      {
+        skips++;
+        return 0;
+      }
+      dos.writeShort( skips );
+      skips = 0;
+      int len = mc.encodeMicroCache( buffer );
+      if ( len == 0 )
+      {
+        throw new IllegalArgumentException( "encoded buffer of non-empty micro-cache cannot be empty" );
+      }
+      dos.writeInt( len );
+      dos.write( buffer, 0, len );
+      return len;
+    }
+
+    public void finish() throws Exception
+    {
+      if ( skips > 0 )
+      {
+        dos.writeShort( skips );
+        skips = 0;
+      }
+    }
+  }
+
+  private static class MCInputStream
+  {
+    private short skips = -1;
+    private DataInputStream dis;
+    private DataBuffers dataBuffers;
+    private MicroCache empty = MicroCache.emptyCache();
+
+    public MCInputStream( DataInputStream dis, DataBuffers dataBuffers )
+    {
+      this.dis = dis;
+      this.dataBuffers = dataBuffers;
+    }
+
+    public MicroCache readMC() throws Exception
+    {
+      if (skips < 0 )
+      {
+        skips = dis.readShort();
+      }
+      MicroCache mc = empty;
+      if ( skips == 0 )
+      {
+        int size = dis.readInt();
+        byte[] ab = new byte[size];
+        dis.readFully( ab );
+        StatCoderContext bc = new StatCoderContext( ab );
+        mc = new MicroCache2( bc, dataBuffers, 0, 0, 32, null, null );
+      }
+      skips--;
+      return mc;
+    }
+
+    public void finish() throws Exception
+    {
+      skips = -1;
+    }
+  }
 
 }

brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffValidator.java

@@ -0,0 +1,70 @@
+/**
+ * Manage rd5 diff-file creation
+ *
+ * @author ab
+ */
+package btools.mapaccess;
+
+import java.io.File;
+
+final public class Rd5DiffValidator
+{
+  public static void main( String[] args ) throws Exception
+  {
+    validateDiffs( new File( args[0] ),new File( args[1] ) );
+  }
+
+  /**
+   * Validate diffs for all DF5 files
+   */
+  public static void validateDiffs( File oldDir, File newDir ) throws Exception
+  {
+    File oldDiffDir = new File( oldDir, "diff" );
+    File newDiffDir = new File( newDir, "diff" );
+
+    File[] filesNew = newDir.listFiles();
+    
+    for( File fn : filesNew )
+    {
+      String name = fn.getName();
+      if ( !name.endsWith( ".rd5" ) )
+      {
+        continue;
+      }
+      if ( fn.length() < 1024*1024 )
+      {
+        continue; // expecting no diff for small files
+      }
+      String basename = name.substring( 0, name.length() - 4 );
+      File fo = new File( oldDir, name );
+      if ( !fo.isFile() )
+      {
+        continue;
+      }
+
+      // calculate MD5 of old file
+      String md5 = Rd5DiffManager.getMD5( fo );
+
+      String md5New = Rd5DiffManager.getMD5( fn );
+
+      System.out.println( "name=" + name + " md5=" + md5 );
+
+      File specificNewDiffs = new File( newDiffDir, basename );
+
+      String diffFileName = md5 + ".df5";
+      File diffFile = new File( specificNewDiffs, diffFileName );
+
+      File fcmp = new File( oldDir, name + "_tmp" );
+
+      // merge old file and diff
+      Rd5DiffTool.recoverFromDelta( fo, diffFile, fcmp, new Rd5DiffTool() );
+      String md5Cmp = Rd5DiffManager.getMD5( fcmp );
+
+      if ( !md5Cmp.equals( md5New ) )
+      {
+        throw new RuntimeException( "**************** md5 mismatch!! *****************" );
+      }
+    }
+  }
+
+}

brouter-routing-app/src/main/java/btools/routingapp/BInstallerView.java

@@ -687,7 +687,7 @@ float tx, ty;
                     
                       // first check for a delta file
                       String md5 = Rd5DiffManager.getMD5( targetFile );
-                      String surlDelta = surlBase + "diff/" + name.replace( ".rd5", "/" + md5 + ".rd5diff" );
+                      String surlDelta = surlBase + "diff/" + name.replace( ".rd5", "/" + md5 + ".df5" );
                       
                       URL urlDelta = new URL(surlDelta);
 

brouter-server/src/main/java/btools/server/IpAccessMonitor.java

@@ -0,0 +1,52 @@
+package btools.server;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class IpAccessMonitor 
+{
+  private static Object sync = new Object();
+  private static HashMap<String,Long> ipAccess = new HashMap<String,Long>();
+  private static long MAX_IDLE = 900000; // 15 minutes
+  private static long CLEANUP_INTERVAL = 10000; // 10 seconds
+  private static long lastCleanup;
+
+  public static boolean touchIpAccess( String ip )
+  {
+    long t = System.currentTimeMillis();
+    synchronized( sync )
+    {
+      Long lastTime = ipAccess.get( ip );
+      ipAccess.put( ip, Long.valueOf( t ) );
+      return lastTime == null || t - lastTime.longValue() > MAX_IDLE;
+    }
+  }
+
+  public static int getSessionCount()
+  {
+    long t = System.currentTimeMillis();
+    synchronized( sync )
+    {
+      if ( t - lastCleanup > CLEANUP_INTERVAL )
+      {
+        cleanup( t );
+        lastCleanup = t;
+      }
+      return ipAccess.size();
+    }
+  }
+  
+  private static void cleanup( long t )
+  {
+    HashMap<String,Long> newMap = new HashMap<String,Long>(ipAccess.size());
+    for( Map.Entry<String,Long> e : ipAccess.entrySet() )
+    {
+      if ( t - e.getValue().longValue() <= MAX_IDLE )
+      {
+        newMap.put( e.getKey(), e.getValue() );
+      }
+    }
+    ipAccess = newMap;
+  }
+    
+}

brouter-server/src/main/java/btools/server/RouteServer.java

@@ -61,7 +61,7 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
 
   private static DateFormat tsFormat = new SimpleDateFormat( "dd.MM.yy HH:mm", new Locale( "en", "US" ) );
 
-  private static String formattedTimestamp()
+  private static String formattedTimeStamp( long t )
   {
     synchronized( tsFormat )
     {
@@ -73,15 +73,21 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
   {
           BufferedReader br = null;
           BufferedWriter bw = null;
+
+          // first line
+          String getline = null;
+          String sessionInfo = null;
+          String sIp = null;
+
           try
           {
             br = new BufferedReader( new InputStreamReader( clientSocket.getInputStream() , "UTF-8") );
             bw = new BufferedWriter( new OutputStreamWriter( clientSocket.getOutputStream(), "UTF-8" ) );
 
-            // first line
-            String getline = null;
             String agent = null;
             String encodings = null;
+            String xff = null; // X-Forwarded-For
+            String referer = null;
 
             // more headers until first empty line
             for(;;)
@@ -102,16 +108,40 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
               {
                 getline = line;
               }
-              if ( line.startsWith( "User-Agent: " ) )
+              line = line.toLowerCase();
+              if ( line.startsWith( "user-agent: " ) )
               {
-                agent = line.substring( "User-Agent: ".length() );
+                agent = line.substring( "user-agent: ".length() );
               }
-              if ( line.startsWith( "Accept-Encoding: " ) )
+              if ( line.startsWith( "accept-encoding: " ) )
               {
-                encodings = line.substring( "Accept-Encoding: ".length() );
+                encodings = line.substring( "accept-encoding: ".length() );
+              }
+              if ( line.startsWith( "x-forwarded-for: " ) )
+              {
+                xff = line.substring( "x-forwarded-for: ".length() );
+              }
+              if ( line.startsWith( "Referer: " ) )
+              {
+                referer = line.substring( "Referer: ".length() );
+              }
+              if ( line.startsWith( "Referrer: " ) )
+              {
+                referer = line.substring( "Referrer: ".length() );
               }
             }
             
+            InetAddress ip = clientSocket.getInetAddress();
+            sIp = xff == null ? (ip==null ? "null" : ip.toString() ) : xff;
+            boolean newSession = IpAccessMonitor.touchIpAccess( sIp );
+            sessionInfo = " new";
+            if ( !newSession )
+            {
+              int sessionCount = IpAccessMonitor.getSessionCount();
+              sessionInfo = "    " + Math.min( sessionCount, 999 );
+              sessionInfo = sessionInfo.substring( sessionInfo.length() - 4 );
+            }
+
             String excludedAgents = System.getProperty( "excludedAgents" );
             if ( agent != null && excludedAgents != null )
             {
@@ -128,6 +158,17 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
               }
             }
 
+            if ( referer != null && referer.indexOf( "brouter.de/brouter-web" ) >= 0 )
+            {
+              if ( getline.indexOf( "%7C" ) >= 0 && getline.indexOf( "%2C" ) >= 0 )
+              {
+                writeHttpHeader( bw, HTTP_STATUS_FORBIDDEN );
+                bw.write( "Spam? please stop" );
+                bw.flush();
+                return;
+              }
+            }
+
             if ( getline.startsWith("GET /favicon.ico") )
             {
               writeHttpHeader( bw, HTTP_STATUS_NOT_FOUND );
@@ -143,9 +184,6 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
               return;
             }
 
-            InetAddress ip = clientSocket.getInetAddress();
-            System.out.println( formattedTimestamp() + " ip=" + (ip==null ? "null" : ip.toString() ) + " -> " + getline );
-
             String url = getline.split(" ")[1];
             HashMap<String,String> params = getUrlParams(url);
 
@@ -280,8 +318,12 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
               {
                 threadPoolSync.notifyAll();
               }
+              long t = System.currentTimeMillis();
+              long ms = t - starttime;
+              System.out.println( formattedTimeStamp(t) + sessionInfo + " ip=" + sIp + " ms=" + ms + " -> " + getline );
           }
   }
+  
 
   public static void main(String[] args) throws Exception
   {
@@ -355,13 +397,18 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
                {
                  threadPoolSync.wait( maxWaitTime );
                }
+               long t = System.currentTimeMillis();
+               System.out.println( formattedTimeStamp(t) + " contention! ms waited " + (t - server.starttime) );
              }
              cleanupThreadQueue( threadQueue );
              if ( threadQueue.size() >= maxthreads )
              {
                if ( debug ) System.out.println( "stopping oldest thread..." );
                // no way... stop the oldest thread
-               threadQueue.poll().stopRouter();
+               RouteServer oldest = threadQueue.poll();
+               oldest.stopRouter();
+               long t = System.currentTimeMillis();
+               System.out.println( formattedTimeStamp(t) + " contention! ms killed " + (t - oldest.starttime) );
              }
           }
 

brouter-server/src/main/java/btools/server/SuspectManager.java

@@ -450,6 +450,8 @@ public class SuspectManager extends Thread
 
       String url5 = "https://tyrasd.github.io/latest-changes/#16/" + dlat + "/" + dlon;
 
+      String url6 = "https://apps.sentinel-hub.com/sentinel-playground/?source=S2L2A&lat=" + dlat + "&lng=" + dlon + "&zoom=15";
+
       if ( message != null )
       {
         bw.write( "<strong>" + message + "</strong><br><br>\n" );
@@ -459,6 +461,7 @@ public class SuspectManager extends Thread
       bw.write( "<a href=\"" + url3 + "\">Open in JOSM (via remote control)</a><br><br>\n" );
       bw.write( "Overpass: <a href=\"" + url4a + "\">minus one week</a> &nbsp;&nbsp; <a href=\"" + url4b + "\">node context</a><br><br>\n" );
       bw.write( "<a href=\"" + url5 + "\">Open in Latest-Changes / last week</a><br><br>\n" );
+      bw.write( "<a href=\"" + url6 + "\">Current Sentinel-2 imagary</a><br><br>\n" );
       bw.write( "<br>\n" );
       if ( isFixed( id, suspects.timestamp ) )
       {

misc/profiles2/lookups.dat

@@ -27,10 +27,10 @@ highway;0000079637 tertiary_link
 highway;0000070238 construction
 highway;0000058257 bridleway
 highway;0000039003 platform
-highway;0000037192 proposed
+highway;0000037192 proposed planned virtual
 highway;0000010307 raceway
 highway;0000003152 rest_area
-highway;0000002942 abandoned
+highway;0000002942 abandoned disused razed demolished dismantled
 highway;0000002631 services
 highway;0000002133 corridor
 highway;0000002093 crossing
@@ -227,7 +227,7 @@ cycleway;0000000892 left
 cycleway;0000000399 street
 cycleway;0000000344 shoulder
 cycleway;0000000326 designated
-cycleway;0000000247 proposed
+cycleway;0000000247 proposed planned virtual
 cycleway;0000000224 cyclestreet
 cycleway;0000000172 path
 cycleway;0000000154 sidewalk