Merge branch 'abrensch:master' into master
This commit is contained in:
afischerdev
GitHub
commit
f59eb0de1e
13 changed files with 440 additions and 89 deletions
|
|
@ -55,7 +55,9 @@ final class MessageData implements Cloneable
|
|||
+ "\t" + linknodecost
|
||||
+ "\t" + linkinitcost
|
||||
+ "\t" + wayKeyValues
|
||||
+ "\t" + ( nodeKeyValues == null ? "" : nodeKeyValues );
|
||||
+ "\t" + ( nodeKeyValues == null ? "" : nodeKeyValues )
|
||||
+ "\t" + ((int)time)
|
||||
+ "\t" + ((int)energy);
|
||||
}
|
||||
|
||||
void add( MessageData d )
|
||||
|
|
|
|||
|
|
@ -139,7 +139,10 @@ abstract class OsmPath implements OsmLinkHolder
|
|||
protected void addAddionalPenalty(OsmTrack refTrack, boolean detailMode, OsmPath origin, OsmLink link, RoutingContext rc )
|
||||
{
|
||||
byte[] description = link.descriptionBitmap;
|
||||
if ( description == null ) throw new IllegalArgumentException( "null description for: " + link );
|
||||
if ( description == null )
|
||||
{
|
||||
return; // could be a beeline path
|
||||
}
|
||||
|
||||
boolean recordTransferNodes = detailMode || rc.countTraffic;
|
||||
|
||||
|
|
|
|||
|
|
@ -19,8 +19,11 @@ import java.io.FileWriter;
|
|||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.StringWriter;
|
||||
import java.text.DecimalFormat;
|
||||
import java.text.NumberFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
import btools.mapaccess.MatchedWaypoint;
|
||||
import btools.mapaccess.OsmPos;
|
||||
|
|
@ -31,7 +34,7 @@ import btools.util.StringUtils;
|
|||
public final class OsmTrack
|
||||
{
|
||||
// csv-header-line
|
||||
private static final String MESSAGES_HEADER = "Longitude\tLatitude\tElevation\tDistance\tCostPerKm\tElevCost\tTurnCost\tNodeCost\tInitialCost\tWayTags\tNodeTags";
|
||||
private static final String MESSAGES_HEADER = "Longitude\tLatitude\tElevation\tDistance\tCostPerKm\tElevCost\tTurnCost\tNodeCost\tInitialCost\tWayTags\tNodeTags\tTime\tEnergy";
|
||||
|
||||
public MatchedWaypoint endPoint;
|
||||
public long[] nogoChecksums;
|
||||
|
|
@ -719,6 +722,8 @@ public final class OsmTrack
|
|||
|
||||
public String formatAsGeoJson()
|
||||
{
|
||||
int turnInstructionMode = voiceHints != null ? voiceHints.turnInstructionMode : 0;
|
||||
|
||||
StringBuilder sb = new StringBuilder( 8192 );
|
||||
|
||||
sb.append( "{\n" );
|
||||
|
|
@ -740,7 +745,20 @@ public final class OsmTrack
|
|||
sb.append( " \"voicehints\": [\n" );
|
||||
for( VoiceHint hint: voiceHints.list )
|
||||
{
|
||||
sb.append( " [" ).append( hint.indexInTrack ).append( ',' ).append( hint.getCommand() ).append( ',' ).append( hint.getExitNumber() ).append( "],\n" );
|
||||
sb.append( " [" );
|
||||
sb.append( hint.indexInTrack );
|
||||
sb.append( ',' ).append( hint.getCommand() );
|
||||
sb.append( ',' ).append( hint.getExitNumber() );
|
||||
sb.append( ',' ).append( hint.distanceToNext );
|
||||
sb.append( ',' ).append( (int) hint.angle );
|
||||
|
||||
// not always include geometry because longer and only needed for comment style
|
||||
if ( turnInstructionMode == 4 ) // comment style
|
||||
{
|
||||
sb.append( ",\"" ).append( hint.formatGeometry() ).append( "\"" );
|
||||
}
|
||||
|
||||
sb.append( "],\n" );
|
||||
}
|
||||
sb.deleteCharAt( sb.lastIndexOf( "," ) );
|
||||
sb.append( " ],\n" );
|
||||
|
|
@ -755,7 +773,7 @@ public final class OsmTrack
|
|||
{
|
||||
sb.append( " [" ).append( sp.get(i) ).append( i> 0 ? "],\n" : "]\n" );
|
||||
}
|
||||
sb.append( " ]\n" );
|
||||
sb.append( " ],\n" );
|
||||
}
|
||||
}
|
||||
else // ... otherwise traditional message list
|
||||
|
|
@ -767,9 +785,24 @@ public final class OsmTrack
|
|||
sb.append( " [\"" ).append( m.replaceAll( "\t", "\", \"" ) ).append( "\"],\n" );
|
||||
}
|
||||
sb.deleteCharAt( sb.lastIndexOf( "," ) );
|
||||
sb.append( " ]\n" );
|
||||
sb.append( " ],\n" );
|
||||
}
|
||||
|
||||
if ( getTotalSeconds() > 0 ) {
|
||||
sb.append( " \"times\": [" );
|
||||
DecimalFormat decimalFormat = (DecimalFormat) NumberFormat.getInstance( Locale.ENGLISH );
|
||||
decimalFormat.applyPattern( "0.###" );
|
||||
for ( OsmPathElement n : nodes ) {
|
||||
sb.append( decimalFormat.format( n.getTime() ) ).append( "," );
|
||||
}
|
||||
sb.deleteCharAt( sb.lastIndexOf( "," ) );
|
||||
sb.append( "]\n" );
|
||||
} else {
|
||||
sb.deleteCharAt( sb.lastIndexOf( "," ) );
|
||||
}
|
||||
|
||||
sb.append( " },\n" );
|
||||
|
||||
if ( iternity != null )
|
||||
{
|
||||
sb.append( " \"iternity\": [\n" );
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import btools.expressions.BExpressionContextNode;
|
|||
import btools.expressions.BExpressionContextWay;
|
||||
import btools.mapaccess.GeometryDecoder;
|
||||
import btools.mapaccess.OsmLink;
|
||||
import btools.mapaccess.OsmNode;
|
||||
import btools.util.CheapAngleMeter;
|
||||
import btools.util.CheapRuler;
|
||||
|
||||
|
|
@ -191,6 +192,7 @@ public final class RoutingContext
|
|||
public List<OsmNodeNamed> poipoints;
|
||||
|
||||
public List<OsmNodeNamed> nogopoints = null;
|
||||
private List<OsmNodeNamed> nogopoints_all = null; // full list not filtered for wayoints-in-nogos
|
||||
private List<OsmNodeNamed> keepnogopoints = null;
|
||||
private OsmNodeNamed pendingEndpoint = null;
|
||||
|
||||
|
|
@ -258,14 +260,29 @@ public final class RoutingContext
|
|||
}
|
||||
}
|
||||
|
||||
public void cleanNogolist( List<OsmNodeNamed> waypoints )
|
||||
/**
|
||||
* restore the full nogolist previously saved by cleanNogoList
|
||||
*/
|
||||
public void restoreNogoList()
|
||||
{
|
||||
nogopoints = nogopoints_all;
|
||||
}
|
||||
|
||||
/**
|
||||
* clean the nogolist (previoulsy saved by saveFullNogolist())
|
||||
* by removing nogos with waypoints within
|
||||
*
|
||||
* @return true if all wayoints are all in the same (full-weigth) nogo area (triggering bee-line-mode)
|
||||
*/
|
||||
public void cleanNogoList( List<OsmNode> waypoints )
|
||||
{
|
||||
nogopoints_all = nogopoints;
|
||||
if ( nogopoints == null ) return;
|
||||
List<OsmNodeNamed> nogos = new ArrayList<OsmNodeNamed>();
|
||||
for( OsmNodeNamed nogo : nogopoints )
|
||||
{
|
||||
boolean goodGuy = true;
|
||||
for( OsmNodeNamed wp : waypoints )
|
||||
for( OsmNode wp : waypoints )
|
||||
{
|
||||
if ( wp.calcDistance( nogo ) < nogo.radius
|
||||
&& (!(nogo instanceof OsmNogoPolygon)
|
||||
|
|
@ -274,7 +291,6 @@ public final class RoutingContext
|
|||
: ((OsmNogoPolygon)nogo).isOnPolyline(wp.ilon, wp.ilat))))
|
||||
{
|
||||
goodGuy = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if ( goodGuy ) nogos.add( nogo );
|
||||
|
|
@ -282,6 +298,31 @@ public final class RoutingContext
|
|||
nogopoints = nogos.isEmpty() ? null : nogos;
|
||||
}
|
||||
|
||||
public boolean allInOneNogo( List<OsmNode> waypoints )
|
||||
{
|
||||
if ( nogopoints == null ) return false;
|
||||
boolean allInTotal = false;
|
||||
for( OsmNodeNamed nogo : nogopoints )
|
||||
{
|
||||
boolean allIn = Double.isNaN( nogo.nogoWeight );
|
||||
for( OsmNode wp : waypoints )
|
||||
{
|
||||
int dist = wp.calcDistance( nogo );
|
||||
if ( dist < nogo.radius
|
||||
&& (!(nogo instanceof OsmNogoPolygon)
|
||||
|| (((OsmNogoPolygon)nogo).isClosed
|
||||
? ((OsmNogoPolygon)nogo).isWithin(wp.ilon, wp.ilat)
|
||||
: ((OsmNogoPolygon)nogo).isOnPolyline(wp.ilon, wp.ilat))))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
allIn = false;
|
||||
}
|
||||
allInTotal |= allIn;
|
||||
}
|
||||
return allInTotal;
|
||||
}
|
||||
|
||||
public long[] getNogoChecksums()
|
||||
{
|
||||
long[] cs = new long[3];
|
||||
|
|
|
|||
|
|
@ -151,9 +151,6 @@ public class RoutingEngine extends Thread
|
|||
{
|
||||
try
|
||||
{
|
||||
// delete nogos with waypoints in them
|
||||
routingContext.cleanNogolist( waypoints );
|
||||
|
||||
startTime = System.currentTimeMillis();
|
||||
long startTime0 = startTime;
|
||||
this.maxRunningTime = maxRunningTime;
|
||||
|
|
@ -459,6 +456,29 @@ public class RoutingEngine extends Thread
|
|||
}
|
||||
|
||||
private OsmTrack searchTrack( MatchedWaypoint startWp, MatchedWaypoint endWp, OsmTrack nearbyTrack, OsmTrack refTrack )
|
||||
{
|
||||
// remove nogos with waypoints inside
|
||||
try
|
||||
{
|
||||
List<OsmNode> wpts2 = new ArrayList<OsmNode>();
|
||||
wpts2.add( startWp.waypoint );
|
||||
wpts2.add( endWp.waypoint );
|
||||
boolean calcBeeline = routingContext.allInOneNogo(wpts2);
|
||||
|
||||
if ( !calcBeeline ) return searchRoutedTrack( startWp, endWp, nearbyTrack, refTrack );
|
||||
|
||||
// we want a beeline-segment
|
||||
OsmPath path = routingContext.createPath( new OsmLink( null, startWp.crosspoint ) );
|
||||
path = routingContext.createPath( path, new OsmLink( startWp.crosspoint, endWp.crosspoint ), null, false );
|
||||
return compileTrack( path, false );
|
||||
}
|
||||
finally
|
||||
{
|
||||
routingContext.restoreNogoList();
|
||||
}
|
||||
}
|
||||
|
||||
private OsmTrack searchRoutedTrack( MatchedWaypoint startWp, MatchedWaypoint endWp, OsmTrack nearbyTrack, OsmTrack refTrack )
|
||||
{
|
||||
OsmTrack track = null;
|
||||
double[] airDistanceCostFactors = new double[]{ routingContext.pass1coefficient, routingContext.pass2coefficient };
|
||||
|
|
@ -650,6 +670,11 @@ public class RoutingEngine extends Thread
|
|||
{
|
||||
try
|
||||
{
|
||||
List<OsmNode> wpts2 = new ArrayList<OsmNode>();
|
||||
if ( startWp != null ) wpts2.add( startWp.waypoint );
|
||||
if ( endWp != null ) wpts2.add( endWp.waypoint );
|
||||
routingContext.cleanNogoList(wpts2);
|
||||
|
||||
boolean detailed = guideTrack != null;
|
||||
resetCache( detailed );
|
||||
nodesCache.nodesMap.cleanupMode = detailed ? 0 : ( routingContext.considerTurnRestrictions ? 2 : 1 );
|
||||
|
|
@ -657,6 +682,7 @@ public class RoutingEngine extends Thread
|
|||
}
|
||||
finally
|
||||
{
|
||||
routingContext.restoreNogoList();
|
||||
nodesCache.clean( false ); // clean only non-virgin caches
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Proof of concept for delta rd5's
|
||||
* Manage rd5 diff-file creation
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
|
|
@ -56,10 +56,10 @@ final public class Rd5DiffManager
|
|||
File specificNewDiffs = new File( newDiffDir, basename );
|
||||
specificNewDiffs.mkdirs();
|
||||
|
||||
String diffFileName = md5 + ".rd5diff";
|
||||
String diffFileName = md5 + ".df5";
|
||||
File diffFile = new File( specificNewDiffs, diffFileName );
|
||||
|
||||
String dummyDiffFileName = md5New + ".rd5diff";
|
||||
String dummyDiffFileName = md5New + ".df5";
|
||||
File dummyDiffFile = new File( specificNewDiffs, dummyDiffFileName );
|
||||
dummyDiffFile.createNewFile();
|
||||
|
||||
|
|
@ -73,7 +73,7 @@ final public class Rd5DiffManager
|
|||
File[] oldDiffs = specificOldDiffs.listFiles();
|
||||
for( File od : oldDiffs )
|
||||
{
|
||||
if ( !od.getName().endsWith( ".rd5diff" ) )
|
||||
if ( !od.getName().endsWith( ".df5" ) )
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Proof of concept for delta rd5's
|
||||
* Calculate, add or merge rd5 delta files
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
|
|
@ -31,9 +31,9 @@ final public class Rd5DiffTool implements ProgressListener
|
|||
return;
|
||||
}
|
||||
|
||||
if ( args[1].endsWith( ".rd5diff" ) )
|
||||
if ( args[1].endsWith( ".df5" ) )
|
||||
{
|
||||
if ( args[0].endsWith( ".rd5diff" ) )
|
||||
if ( args[0].endsWith( ".df5" ) )
|
||||
{
|
||||
addDeltas( new File( args[0] ),new File( args[1] ), new File( args[2] ) );
|
||||
}
|
||||
|
|
@ -152,6 +152,7 @@ final public class Rd5DiffTool implements ProgressListener
|
|||
DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
|
||||
DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) );
|
||||
DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
|
||||
MCOutputStream mcOut = new MCOutputStream( dos, abBuf1 );
|
||||
|
||||
// copy header to outfile
|
||||
long[] fileIndex1 = readFileIndex( dis1, null );
|
||||
|
|
@ -175,10 +176,6 @@ final public class Rd5DiffTool implements ProgressListener
|
|||
byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false );
|
||||
byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, false );
|
||||
|
||||
if ( ab2 == null )
|
||||
{
|
||||
continue; // empty target tile
|
||||
}
|
||||
MicroCache mc;
|
||||
if ( Arrays.equals( ab1, ab2 ) )
|
||||
{
|
||||
|
|
@ -192,16 +189,9 @@ final public class Rd5DiffTool implements ProgressListener
|
|||
mc.calcDelta( mc1, mc2 );
|
||||
}
|
||||
|
||||
if ( mc.getSize() == 0 )
|
||||
int len = mcOut.writeMC( mc );
|
||||
if ( len > 0 )
|
||||
{
|
||||
dos.writeInt( 0 );
|
||||
}
|
||||
else
|
||||
{
|
||||
int len = mc.encodeMicroCache( abBuf1 );
|
||||
dos.writeInt( len );
|
||||
dos.write( abBuf1, 0, len );
|
||||
|
||||
bytesDiff += len;
|
||||
nodesDiff += mc.getSize();
|
||||
diffedTiles++;
|
||||
|
|
@ -244,7 +234,9 @@ final public class Rd5DiffTool implements ProgressListener
|
|||
*/
|
||||
}
|
||||
}
|
||||
mcOut.finish();
|
||||
}
|
||||
|
||||
// write any remaining data to the output file
|
||||
for(;;)
|
||||
{
|
||||
|
|
@ -324,10 +316,12 @@ final public class Rd5DiffTool implements ProgressListener
|
|||
try
|
||||
{
|
||||
DataBuffers dataBuffers = new DataBuffers();
|
||||
MCInputStream mcIn = new MCInputStream( dis2, dataBuffers );
|
||||
|
||||
for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ )
|
||||
{
|
||||
boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx );
|
||||
boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx );
|
||||
boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx ); // has the basefile data
|
||||
boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx ); // has the *result* data
|
||||
|
||||
// boolean hasDataCmp = getTileStart( fileIndexCmp, subFileIdx ) < getTileEnd( fileIndexCmp, subFileIdx );
|
||||
|
||||
|
|
@ -352,11 +346,8 @@ final public class Rd5DiffTool implements ProgressListener
|
|||
}
|
||||
|
||||
byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false );
|
||||
byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true );
|
||||
if ( ab2 == null )
|
||||
{
|
||||
continue; // no target tile expected
|
||||
}
|
||||
MicroCache mc2 = mcIn.readMC();
|
||||
int targetSize = posIdx2 == null ? 0 : getPosIdx( posIdx2, tileIdx ) - getPosIdx( posIdx2, tileIdx-1 );
|
||||
|
||||
/* int targetSizeCmp = getPosIdx( posIdxCmp, tileIdx ) - getPosIdx( posIdxCmp, tileIdx-1 );
|
||||
if ( targetSizeCmp != targetSize ) throw new IllegalArgumentException( "target size mismatch: "+ targetSize + "," + targetSizeCmp );
|
||||
|
|
@ -365,22 +356,36 @@ final public class Rd5DiffTool implements ProgressListener
|
|||
*/
|
||||
|
||||
// no-delta shortcut: just copy base data
|
||||
if ( ab2.length == 0 )
|
||||
if ( mc2.getSize() == 0 )
|
||||
{
|
||||
if ( ab1 != null )
|
||||
{
|
||||
dos.write( ab1 );
|
||||
}
|
||||
int newTargetSize = ab1 == null ? 0 : ab1.length;
|
||||
if ( targetSize != newTargetSize )
|
||||
{
|
||||
throw new RuntimeException( "size mismatch at " + subFileIdx + "/" + tileIdx + " " + targetSize + "!=" + newTargetSize );
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// this is the real delta case (using decode->delta->encode )
|
||||
|
||||
MicroCache mc1 = createMicroCache( ab1, dataBuffers );
|
||||
MicroCache mc2 = createMicroCache( ab2, dataBuffers );
|
||||
|
||||
MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 );
|
||||
mc.addDelta( mc1, mc2, false );
|
||||
|
||||
if ( mc.size() == 0 )
|
||||
{
|
||||
if ( targetSize != 0 )
|
||||
{
|
||||
throw new RuntimeException( "size mismatch at " + subFileIdx + "/" + tileIdx + " " + targetSize + ">0" );
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
int len = mc.encodeMicroCache( abBuf1 );
|
||||
|
||||
/* System.out.println( "comparing for subFileIdx=" + subFileIdx + " tileIdx=" + tileIdx );
|
||||
|
|
@ -403,8 +408,13 @@ final public class Rd5DiffTool implements ProgressListener
|
|||
*/
|
||||
dos.write( abBuf1, 0, len );
|
||||
dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 );
|
||||
if ( targetSize != len+4 )
|
||||
{
|
||||
throw new RuntimeException( "size mismatch at " + subFileIdx + "/" + tileIdx + " " + targetSize + "<>" + (len+4) );
|
||||
}
|
||||
|
||||
}
|
||||
mcIn.finish();
|
||||
}
|
||||
// write any remaining data to the output file
|
||||
for(;;)
|
||||
|
|
@ -529,7 +539,7 @@ final public class Rd5DiffTool implements ProgressListener
|
|||
DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) );
|
||||
DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
|
||||
|
||||
// copy header to outfile
|
||||
// copy subfile-header to outfile
|
||||
long[] fileIndex1 = readFileIndex( dis1, null );
|
||||
long[] fileIndex2 = readFileIndex( dis2, dos );
|
||||
|
||||
|
|
@ -538,51 +548,37 @@ final public class Rd5DiffTool implements ProgressListener
|
|||
try
|
||||
{
|
||||
DataBuffers dataBuffers = new DataBuffers();
|
||||
MCInputStream mcIn1 = new MCInputStream( dis1, dataBuffers );
|
||||
MCInputStream mcIn2 = new MCInputStream( dis2, dataBuffers );
|
||||
MCOutputStream mcOut = new MCOutputStream( dos, abBuf1 );
|
||||
|
||||
for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ )
|
||||
{
|
||||
// copy tile-header to outfile
|
||||
boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx );
|
||||
boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx );
|
||||
|
||||
int[] posIdx1 = hasData1 ? readPosIndex( dis1, null ) : null;
|
||||
int[] posIdx2 = hasData2 ? readPosIndex( dis2, dos ) : null;
|
||||
|
||||
for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ )
|
||||
{
|
||||
byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, true );
|
||||
byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true );
|
||||
if ( ab2 == null )
|
||||
MicroCache mc1 = mcIn1.readMC();
|
||||
MicroCache mc2 = mcIn2.readMC();
|
||||
MicroCache mc;
|
||||
if ( mc1.getSize() == 0 && mc2.getSize() == 0 )
|
||||
{
|
||||
continue; // no target tile expected
|
||||
}
|
||||
|
||||
// no-delta shortcut: just copy base data
|
||||
if ( ab2.length == 0 )
|
||||
{
|
||||
if ( ab1 == null )
|
||||
{
|
||||
dos.writeInt( 0 );
|
||||
mc = mc1;
|
||||
}
|
||||
else
|
||||
{
|
||||
dos.writeInt( ab1.length );
|
||||
dos.write( ab1 );
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// this is the real delta case (using decode->add->encode )
|
||||
MicroCache mc1 = createMicroCache( ab1, dataBuffers );
|
||||
MicroCache mc2 = createMicroCache( ab2, dataBuffers );
|
||||
|
||||
MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 );
|
||||
mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 );
|
||||
mc.addDelta( mc1, mc2, true );
|
||||
|
||||
int len = mc.encodeMicroCache( abBuf1 );
|
||||
|
||||
dos.writeInt( len+4 );
|
||||
dos.write( abBuf1, 0, len );
|
||||
dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 );
|
||||
}
|
||||
mcOut.writeMC( mc );
|
||||
}
|
||||
mcIn1.finish();
|
||||
mcIn2.finish();
|
||||
mcOut.finish();
|
||||
}
|
||||
// write any remaining data to the output file
|
||||
for(;;)
|
||||
|
|
@ -706,5 +702,83 @@ final public class Rd5DiffTool implements ProgressListener
|
|||
}
|
||||
}
|
||||
|
||||
private static class MCOutputStream
|
||||
{
|
||||
private DataOutputStream dos;
|
||||
private byte[] buffer;
|
||||
private short skips = 0;
|
||||
|
||||
public MCOutputStream(DataOutputStream dos, byte[] buffer)
|
||||
{
|
||||
this.dos = dos;
|
||||
this.buffer = buffer;
|
||||
}
|
||||
|
||||
public int writeMC(MicroCache mc) throws Exception
|
||||
{
|
||||
if ( mc.getSize() == 0 )
|
||||
{
|
||||
skips++;
|
||||
return 0;
|
||||
}
|
||||
dos.writeShort( skips );
|
||||
skips = 0;
|
||||
int len = mc.encodeMicroCache( buffer );
|
||||
if ( len == 0 )
|
||||
{
|
||||
throw new IllegalArgumentException( "encoded buffer of non-empty micro-cache cannot be empty" );
|
||||
}
|
||||
dos.writeInt( len );
|
||||
dos.write( buffer, 0, len );
|
||||
return len;
|
||||
}
|
||||
|
||||
public void finish() throws Exception
|
||||
{
|
||||
if ( skips > 0 )
|
||||
{
|
||||
dos.writeShort( skips );
|
||||
skips = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static class MCInputStream
|
||||
{
|
||||
private short skips = -1;
|
||||
private DataInputStream dis;
|
||||
private DataBuffers dataBuffers;
|
||||
private MicroCache empty = MicroCache.emptyCache();
|
||||
|
||||
public MCInputStream( DataInputStream dis, DataBuffers dataBuffers )
|
||||
{
|
||||
this.dis = dis;
|
||||
this.dataBuffers = dataBuffers;
|
||||
}
|
||||
|
||||
public MicroCache readMC() throws Exception
|
||||
{
|
||||
if (skips < 0 )
|
||||
{
|
||||
skips = dis.readShort();
|
||||
}
|
||||
MicroCache mc = empty;
|
||||
if ( skips == 0 )
|
||||
{
|
||||
int size = dis.readInt();
|
||||
byte[] ab = new byte[size];
|
||||
dis.readFully( ab );
|
||||
StatCoderContext bc = new StatCoderContext( ab );
|
||||
mc = new MicroCache2( bc, dataBuffers, 0, 0, 32, null, null );
|
||||
}
|
||||
skips--;
|
||||
return mc;
|
||||
}
|
||||
|
||||
public void finish() throws Exception
|
||||
{
|
||||
skips = -1;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,70 @@
|
|||
/**
|
||||
* Manage rd5 diff-file creation
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
package btools.mapaccess;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
final public class Rd5DiffValidator
|
||||
{
|
||||
public static void main( String[] args ) throws Exception
|
||||
{
|
||||
validateDiffs( new File( args[0] ),new File( args[1] ) );
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate diffs for all DF5 files
|
||||
*/
|
||||
public static void validateDiffs( File oldDir, File newDir ) throws Exception
|
||||
{
|
||||
File oldDiffDir = new File( oldDir, "diff" );
|
||||
File newDiffDir = new File( newDir, "diff" );
|
||||
|
||||
File[] filesNew = newDir.listFiles();
|
||||
|
||||
for( File fn : filesNew )
|
||||
{
|
||||
String name = fn.getName();
|
||||
if ( !name.endsWith( ".rd5" ) )
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if ( fn.length() < 1024*1024 )
|
||||
{
|
||||
continue; // expecting no diff for small files
|
||||
}
|
||||
String basename = name.substring( 0, name.length() - 4 );
|
||||
File fo = new File( oldDir, name );
|
||||
if ( !fo.isFile() )
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// calculate MD5 of old file
|
||||
String md5 = Rd5DiffManager.getMD5( fo );
|
||||
|
||||
String md5New = Rd5DiffManager.getMD5( fn );
|
||||
|
||||
System.out.println( "name=" + name + " md5=" + md5 );
|
||||
|
||||
File specificNewDiffs = new File( newDiffDir, basename );
|
||||
|
||||
String diffFileName = md5 + ".df5";
|
||||
File diffFile = new File( specificNewDiffs, diffFileName );
|
||||
|
||||
File fcmp = new File( oldDir, name + "_tmp" );
|
||||
|
||||
// merge old file and diff
|
||||
Rd5DiffTool.recoverFromDelta( fo, diffFile, fcmp, new Rd5DiffTool() );
|
||||
String md5Cmp = Rd5DiffManager.getMD5( fcmp );
|
||||
|
||||
if ( !md5Cmp.equals( md5New ) )
|
||||
{
|
||||
throw new RuntimeException( "**************** md5 mismatch!! *****************" );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -687,7 +687,7 @@ float tx, ty;
|
|||
|
||||
// first check for a delta file
|
||||
String md5 = Rd5DiffManager.getMD5( targetFile );
|
||||
String surlDelta = surlBase + "diff/" + name.replace( ".rd5", "/" + md5 + ".rd5diff" );
|
||||
String surlDelta = surlBase + "diff/" + name.replace( ".rd5", "/" + md5 + ".df5" );
|
||||
|
||||
URL urlDelta = new URL(surlDelta);
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
package btools.server;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class IpAccessMonitor
|
||||
{
|
||||
private static Object sync = new Object();
|
||||
private static HashMap<String,Long> ipAccess = new HashMap<String,Long>();
|
||||
private static long MAX_IDLE = 900000; // 15 minutes
|
||||
private static long CLEANUP_INTERVAL = 10000; // 10 seconds
|
||||
private static long lastCleanup;
|
||||
|
||||
public static boolean touchIpAccess( String ip )
|
||||
{
|
||||
long t = System.currentTimeMillis();
|
||||
synchronized( sync )
|
||||
{
|
||||
Long lastTime = ipAccess.get( ip );
|
||||
ipAccess.put( ip, Long.valueOf( t ) );
|
||||
return lastTime == null || t - lastTime.longValue() > MAX_IDLE;
|
||||
}
|
||||
}
|
||||
|
||||
public static int getSessionCount()
|
||||
{
|
||||
long t = System.currentTimeMillis();
|
||||
synchronized( sync )
|
||||
{
|
||||
if ( t - lastCleanup > CLEANUP_INTERVAL )
|
||||
{
|
||||
cleanup( t );
|
||||
lastCleanup = t;
|
||||
}
|
||||
return ipAccess.size();
|
||||
}
|
||||
}
|
||||
|
||||
private static void cleanup( long t )
|
||||
{
|
||||
HashMap<String,Long> newMap = new HashMap<String,Long>(ipAccess.size());
|
||||
for( Map.Entry<String,Long> e : ipAccess.entrySet() )
|
||||
{
|
||||
if ( t - e.getValue().longValue() <= MAX_IDLE )
|
||||
{
|
||||
newMap.put( e.getKey(), e.getValue() );
|
||||
}
|
||||
}
|
||||
ipAccess = newMap;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -61,7 +61,7 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
|
|||
|
||||
private static DateFormat tsFormat = new SimpleDateFormat( "dd.MM.yy HH:mm", new Locale( "en", "US" ) );
|
||||
|
||||
private static String formattedTimestamp()
|
||||
private static String formattedTimeStamp( long t )
|
||||
{
|
||||
synchronized( tsFormat )
|
||||
{
|
||||
|
|
@ -73,15 +73,21 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
|
|||
{
|
||||
BufferedReader br = null;
|
||||
BufferedWriter bw = null;
|
||||
|
||||
// first line
|
||||
String getline = null;
|
||||
String sessionInfo = null;
|
||||
String sIp = null;
|
||||
|
||||
try
|
||||
{
|
||||
br = new BufferedReader( new InputStreamReader( clientSocket.getInputStream() , "UTF-8") );
|
||||
bw = new BufferedWriter( new OutputStreamWriter( clientSocket.getOutputStream(), "UTF-8" ) );
|
||||
|
||||
// first line
|
||||
String getline = null;
|
||||
String agent = null;
|
||||
String encodings = null;
|
||||
String xff = null; // X-Forwarded-For
|
||||
String referer = null;
|
||||
|
||||
// more headers until first empty line
|
||||
for(;;)
|
||||
|
|
@ -102,14 +108,38 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
|
|||
{
|
||||
getline = line;
|
||||
}
|
||||
if ( line.startsWith( "User-Agent: " ) )
|
||||
line = line.toLowerCase();
|
||||
if ( line.startsWith( "user-agent: " ) )
|
||||
{
|
||||
agent = line.substring( "User-Agent: ".length() );
|
||||
agent = line.substring( "user-agent: ".length() );
|
||||
}
|
||||
if ( line.startsWith( "Accept-Encoding: " ) )
|
||||
if ( line.startsWith( "accept-encoding: " ) )
|
||||
{
|
||||
encodings = line.substring( "Accept-Encoding: ".length() );
|
||||
encodings = line.substring( "accept-encoding: ".length() );
|
||||
}
|
||||
if ( line.startsWith( "x-forwarded-for: " ) )
|
||||
{
|
||||
xff = line.substring( "x-forwarded-for: ".length() );
|
||||
}
|
||||
if ( line.startsWith( "Referer: " ) )
|
||||
{
|
||||
referer = line.substring( "Referer: ".length() );
|
||||
}
|
||||
if ( line.startsWith( "Referrer: " ) )
|
||||
{
|
||||
referer = line.substring( "Referrer: ".length() );
|
||||
}
|
||||
}
|
||||
|
||||
InetAddress ip = clientSocket.getInetAddress();
|
||||
sIp = xff == null ? (ip==null ? "null" : ip.toString() ) : xff;
|
||||
boolean newSession = IpAccessMonitor.touchIpAccess( sIp );
|
||||
sessionInfo = " new";
|
||||
if ( !newSession )
|
||||
{
|
||||
int sessionCount = IpAccessMonitor.getSessionCount();
|
||||
sessionInfo = " " + Math.min( sessionCount, 999 );
|
||||
sessionInfo = sessionInfo.substring( sessionInfo.length() - 4 );
|
||||
}
|
||||
|
||||
String excludedAgents = System.getProperty( "excludedAgents" );
|
||||
|
|
@ -128,6 +158,17 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
|
|||
}
|
||||
}
|
||||
|
||||
if ( referer != null && referer.indexOf( "brouter.de/brouter-web" ) >= 0 )
|
||||
{
|
||||
if ( getline.indexOf( "%7C" ) >= 0 && getline.indexOf( "%2C" ) >= 0 )
|
||||
{
|
||||
writeHttpHeader( bw, HTTP_STATUS_FORBIDDEN );
|
||||
bw.write( "Spam? please stop" );
|
||||
bw.flush();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if ( getline.startsWith("GET /favicon.ico") )
|
||||
{
|
||||
writeHttpHeader( bw, HTTP_STATUS_NOT_FOUND );
|
||||
|
|
@ -143,9 +184,6 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
|
|||
return;
|
||||
}
|
||||
|
||||
InetAddress ip = clientSocket.getInetAddress();
|
||||
System.out.println( formattedTimestamp() + " ip=" + (ip==null ? "null" : ip.toString() ) + " -> " + getline );
|
||||
|
||||
String url = getline.split(" ")[1];
|
||||
HashMap<String,String> params = getUrlParams(url);
|
||||
|
||||
|
|
@ -280,9 +318,13 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
|
|||
{
|
||||
threadPoolSync.notifyAll();
|
||||
}
|
||||
long t = System.currentTimeMillis();
|
||||
long ms = t - starttime;
|
||||
System.out.println( formattedTimeStamp(t) + sessionInfo + " ip=" + sIp + " ms=" + ms + " -> " + getline );
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
System.out.println("BRouter 1.6.1 / 01032020");
|
||||
|
|
@ -355,13 +397,18 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
|
|||
{
|
||||
threadPoolSync.wait( maxWaitTime );
|
||||
}
|
||||
long t = System.currentTimeMillis();
|
||||
System.out.println( formattedTimeStamp(t) + " contention! ms waited " + (t - server.starttime) );
|
||||
}
|
||||
cleanupThreadQueue( threadQueue );
|
||||
if ( threadQueue.size() >= maxthreads )
|
||||
{
|
||||
if ( debug ) System.out.println( "stopping oldest thread..." );
|
||||
// no way... stop the oldest thread
|
||||
threadQueue.poll().stopRouter();
|
||||
RouteServer oldest = threadQueue.poll();
|
||||
oldest.stopRouter();
|
||||
long t = System.currentTimeMillis();
|
||||
System.out.println( formattedTimeStamp(t) + " contention! ms killed " + (t - oldest.starttime) );
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -450,6 +450,8 @@ public class SuspectManager extends Thread
|
|||
|
||||
String url5 = "https://tyrasd.github.io/latest-changes/#16/" + dlat + "/" + dlon;
|
||||
|
||||
String url6 = "https://apps.sentinel-hub.com/sentinel-playground/?source=S2L2A&lat=" + dlat + "&lng=" + dlon + "&zoom=15";
|
||||
|
||||
if ( message != null )
|
||||
{
|
||||
bw.write( "<strong>" + message + "</strong><br><br>\n" );
|
||||
|
|
@ -459,6 +461,7 @@ public class SuspectManager extends Thread
|
|||
bw.write( "<a href=\"" + url3 + "\">Open in JOSM (via remote control)</a><br><br>\n" );
|
||||
bw.write( "Overpass: <a href=\"" + url4a + "\">minus one week</a> <a href=\"" + url4b + "\">node context</a><br><br>\n" );
|
||||
bw.write( "<a href=\"" + url5 + "\">Open in Latest-Changes / last week</a><br><br>\n" );
|
||||
bw.write( "<a href=\"" + url6 + "\">Current Sentinel-2 imagary</a><br><br>\n" );
|
||||
bw.write( "<br>\n" );
|
||||
if ( isFixed( id, suspects.timestamp ) )
|
||||
{
|
||||
|
|
|
|||
|
|
@ -27,10 +27,10 @@ highway;0000079637 tertiary_link
|
|||
highway;0000070238 construction
|
||||
highway;0000058257 bridleway
|
||||
highway;0000039003 platform
|
||||
highway;0000037192 proposed
|
||||
highway;0000037192 proposed planned virtual
|
||||
highway;0000010307 raceway
|
||||
highway;0000003152 rest_area
|
||||
highway;0000002942 abandoned
|
||||
highway;0000002942 abandoned disused razed demolished dismantled
|
||||
highway;0000002631 services
|
||||
highway;0000002133 corridor
|
||||
highway;0000002093 crossing
|
||||
|
|
@ -227,7 +227,7 @@ cycleway;0000000892 left
|
|||
cycleway;0000000399 street
|
||||
cycleway;0000000344 shoulder
|
||||
cycleway;0000000326 designated
|
||||
cycleway;0000000247 proposed
|
||||
cycleway;0000000247 proposed planned virtual
|
||||
cycleway;0000000224 cyclestreet
|
||||
cycleway;0000000172 path
|
||||
cycleway;0000000154 sidewalk
|
||||
|
|
|
|||
Loading…
Reference in a new issue