From ca48edda04b51a682af56b61ee5d5101784d2244 Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Sat, 14 Nov 2020 09:39:10 +0100 Subject: [PATCH 01/12] new diff file format, fixing empty-microtile-bug --- .../java/btools/mapaccess/Rd5DiffManager.java | 8 +- .../java/btools/mapaccess/Rd5DiffTool.java | 190 ++++++++++++------ .../btools/mapaccess/Rd5DiffValidator.java | 70 +++++++ .../btools/routingapp/BInstallerView.java | 2 +- 4 files changed, 207 insertions(+), 63 deletions(-) create mode 100644 brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffValidator.java diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java index faa860b..86ff392 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java @@ -1,5 +1,5 @@ /** - * Proof of concept for delta rd5's + * Manage rd5 diff-file creation * * @author ab */ @@ -56,10 +56,10 @@ final public class Rd5DiffManager File specificNewDiffs = new File( newDiffDir, basename ); specificNewDiffs.mkdirs(); - String diffFileName = md5 + ".rd5diff"; + String diffFileName = md5 + ".df5"; File diffFile = new File( specificNewDiffs, diffFileName ); - String dummyDiffFileName = md5New + ".rd5diff"; + String dummyDiffFileName = md5New + ".df5"; File dummyDiffFile = new File( specificNewDiffs, dummyDiffFileName ); dummyDiffFile.createNewFile(); @@ -73,7 +73,7 @@ final public class Rd5DiffManager File[] oldDiffs = specificOldDiffs.listFiles(); for( File od : oldDiffs ) { - if ( !od.getName().endsWith( ".rd5diff" ) ) + if ( !od.getName().endsWith( ".df5" ) ) { continue; } diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java index 1f796f7..426e40a 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java @@ -1,5 +1,5 @@ /** - * Proof of concept for delta rd5's + * Calculate, add or merge rd5 delta files * * @author ab */ @@ -31,9 +31,9 @@ final public class Rd5DiffTool implements ProgressListener return; } - if ( args[1].endsWith( ".rd5diff" ) ) + if ( args[1].endsWith( ".df5" ) ) { - if ( args[0].endsWith( ".rd5diff" ) ) + if ( args[0].endsWith( ".df5" ) ) { addDeltas( new File( args[0] ),new File( args[1] ), new File( args[2] ) ); } @@ -152,6 +152,7 @@ final public class Rd5DiffTool implements ProgressListener DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) ); DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) ); DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) ); + MCOutputStream mcOut = new MCOutputStream( dos, abBuf1 ); // copy header to outfile long[] fileIndex1 = readFileIndex( dis1, null ); @@ -175,10 +176,6 @@ final public class Rd5DiffTool implements ProgressListener byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false ); byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, false ); - if ( ab2 == null ) - { - continue; // empty target tile - } MicroCache mc; if ( Arrays.equals( ab1, ab2 ) ) { @@ -191,17 +188,10 @@ final public class Rd5DiffTool implements ProgressListener mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 ); mc.calcDelta( mc1, mc2 ); } - - if ( mc.getSize() == 0 ) - { - dos.writeInt( 0 ); - } - else - { - int len = mc.encodeMicroCache( abBuf1 ); - dos.writeInt( len ); - dos.write( abBuf1, 0, len ); + int len = mcOut.writeMC( mc ); + if ( len > 0 ) + { bytesDiff += len; nodesDiff += mc.getSize(); diffedTiles++; @@ -244,7 +234,9 @@ final public class Rd5DiffTool implements ProgressListener */ } } + mcOut.finish(); } + // write any remaining data to the output file for(;;) { @@ -324,10 +316,12 @@ final public class Rd5DiffTool implements ProgressListener try { DataBuffers dataBuffers = new DataBuffers(); + MCInputStream mcIn = new MCInputStream( dis2, dataBuffers ); + for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ ) { - boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx ); - boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx ); + boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx ); // has the basefile data + boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx ); // has the *result* data // boolean hasDataCmp = getTileStart( fileIndexCmp, subFileIdx ) < getTileEnd( fileIndexCmp, subFileIdx ); @@ -352,11 +346,8 @@ final public class Rd5DiffTool implements ProgressListener } byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false ); - byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true ); - if ( ab2 == null ) - { - continue; // no target tile expected - } + MicroCache mc2 = mcIn.readMC(); + int targetSize = getPosIdx( posIdx2, tileIdx ) - getPosIdx( posIdx2, tileIdx-1 ); /* int targetSizeCmp = getPosIdx( posIdxCmp, tileIdx ) - getPosIdx( posIdxCmp, tileIdx-1 ); if ( targetSizeCmp != targetSize ) throw new IllegalArgumentException( "target size mismatch: "+ targetSize + "," + targetSizeCmp ); @@ -365,21 +356,35 @@ final public class Rd5DiffTool implements ProgressListener */ // no-delta shortcut: just copy base data - if ( ab2.length == 0 ) + if ( mc2.getSize() == 0 ) { if ( ab1 != null ) { dos.write( ab1 ); } + int newTargetSize = ab1 == null ? 0 : ab1.length; + if ( targetSize != newTargetSize ) + { + throw new RuntimeException( "size mismatch at " + subFileIdx + "/" + tileIdx + " " + targetSize + "!=" + newTargetSize ); + } continue; } // this is the real delta case (using decode->delta->encode ) + MicroCache mc1 = createMicroCache( ab1, dataBuffers ); - MicroCache mc2 = createMicroCache( ab2, dataBuffers ); MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 ); mc.addDelta( mc1, mc2, false ); + + if ( mc.size() == 0 ) + { + if ( targetSize != 0 ) + { + throw new RuntimeException( "size mismatch at " + subFileIdx + "/" + tileIdx + " " + targetSize + ">0" ); + } + continue; + } int len = mc.encodeMicroCache( abBuf1 ); @@ -403,8 +408,13 @@ final public class Rd5DiffTool implements ProgressListener */ dos.write( abBuf1, 0, len ); dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 ); + if ( targetSize != len+4 ) + { + throw new RuntimeException( "size mismatch at " + subFileIdx + "/" + tileIdx + " " + targetSize + "<>" + (len+4) ); + } } + mcIn.finish(); } // write any remaining data to the output file for(;;) @@ -529,7 +539,7 @@ final public class Rd5DiffTool implements ProgressListener DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) ); DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) ); - // copy header to outfile + // copy subfile-header to outfile long[] fileIndex1 = readFileIndex( dis1, null ); long[] fileIndex2 = readFileIndex( dis2, dos ); @@ -538,51 +548,37 @@ final public class Rd5DiffTool implements ProgressListener try { DataBuffers dataBuffers = new DataBuffers(); + MCInputStream mcIn1 = new MCInputStream( dis1, dataBuffers ); + MCInputStream mcIn2 = new MCInputStream( dis2, dataBuffers ); + MCOutputStream mcOut = new MCOutputStream( dos, abBuf1 ); + for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ ) { + // copy tile-header to outfile boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx ); boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx ); - int[] posIdx1 = hasData1 ? readPosIndex( dis1, null ) : null; int[] posIdx2 = hasData2 ? readPosIndex( dis2, dos ) : null; for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ ) { - byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, true ); - byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true ); - if ( ab2 == null ) + MicroCache mc1 = mcIn1.readMC(); + MicroCache mc2 = mcIn2.readMC(); + MicroCache mc; + if ( mc1.getSize() == 0 && mc2.getSize() == 0 ) { - continue; // no target tile expected + mc = mc1; } - - // no-delta shortcut: just copy base data - if ( ab2.length == 0 ) + else { - if ( ab1 == null ) - { - dos.writeInt( 0 ); - } - else - { - dos.writeInt( ab1.length ); - dos.write( ab1 ); - } - continue; + mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 ); + mc.addDelta( mc1, mc2, true ); } - - // this is the real delta case (using decode->add->encode ) - MicroCache mc1 = createMicroCache( ab1, dataBuffers ); - MicroCache mc2 = createMicroCache( ab2, dataBuffers ); - - MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 ); - mc.addDelta( mc1, mc2, true ); - - int len = mc.encodeMicroCache( abBuf1 ); - - dos.writeInt( len+4 ); - dos.write( abBuf1, 0, len ); - dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 ); + mcOut.writeMC( mc ); } + mcIn1.finish(); + mcIn2.finish(); + mcOut.finish(); } // write any remaining data to the output file for(;;) @@ -706,5 +702,83 @@ final public class Rd5DiffTool implements ProgressListener } } + private static class MCOutputStream + { + private DataOutputStream dos; + private byte[] buffer; + private short skips = 0; + + public MCOutputStream(DataOutputStream dos, byte[] buffer) + { + this.dos = dos; + this.buffer = buffer; + } + + public int writeMC(MicroCache mc) throws Exception + { + if ( mc.getSize() == 0 ) + { + skips++; + return 0; + } + dos.writeShort( skips ); + skips = 0; + int len = mc.encodeMicroCache( buffer ); + if ( len == 0 ) + { + throw new IllegalArgumentException( "encoded buffer of non-empty micro-cache cannot be empty" ); + } + dos.writeInt( len ); + dos.write( buffer, 0, len ); + return len; + } + + public void finish() throws Exception + { + if ( skips > 0 ) + { + dos.writeShort( skips ); + skips = 0; + } + } + } + + private static class MCInputStream + { + private short skips = -1; + private DataInputStream dis; + private DataBuffers dataBuffers; + private MicroCache empty = MicroCache.emptyCache(); + + public MCInputStream( DataInputStream dis, DataBuffers dataBuffers ) + { + this.dis = dis; + this.dataBuffers = dataBuffers; + } + + public MicroCache readMC() throws Exception + { + if (skips < 0 ) + { + skips = dis.readShort(); + } + MicroCache mc = empty; + if ( skips == 0 ) + { + int size = dis.readInt(); + byte[] ab = new byte[size]; + dis.readFully( ab ); + StatCoderContext bc = new StatCoderContext( ab ); + mc = new MicroCache2( bc, dataBuffers, 0, 0, 32, null, null ); + } + skips--; + return mc; + } + + public void finish() throws Exception + { + skips = -1; + } + } } diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffValidator.java b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffValidator.java new file mode 100644 index 0000000..99bca62 --- /dev/null +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffValidator.java @@ -0,0 +1,70 @@ +/** + * Manage rd5 diff-file creation + * + * @author ab + */ +package btools.mapaccess; + +import java.io.File; + +final public class Rd5DiffValidator +{ + public static void main( String[] args ) throws Exception + { + validateDiffs( new File( args[0] ),new File( args[1] ) ); + } + + /** + * Validate diffs for all DF5 files + */ + public static void validateDiffs( File oldDir, File newDir ) throws Exception + { + File oldDiffDir = new File( oldDir, "diff" ); + File newDiffDir = new File( newDir, "diff" ); + + File[] filesNew = newDir.listFiles(); + + for( File fn : filesNew ) + { + String name = fn.getName(); + if ( !name.endsWith( ".rd5" ) ) + { + continue; + } + if ( fn.length() < 1024*1024 ) + { + continue; // expecting no diff for small files + } + String basename = name.substring( 0, name.length() - 4 ); + File fo = new File( oldDir, name ); + if ( !fo.isFile() ) + { + continue; + } + + // calculate MD5 of old file + String md5 = Rd5DiffManager.getMD5( fo ); + + String md5New = Rd5DiffManager.getMD5( fn ); + + System.out.println( "name=" + name + " md5=" + md5 ); + + File specificNewDiffs = new File( newDiffDir, basename ); + + String diffFileName = md5 + ".df5"; + File diffFile = new File( specificNewDiffs, diffFileName ); + + File fcmp = new File( oldDir, name + "_tmp" ); + + // merge old file and diff + Rd5DiffTool.recoverFromDelta( fo, diffFile, fcmp, new Rd5DiffTool() ); + String md5Cmp = Rd5DiffManager.getMD5( fcmp ); + + if ( !md5Cmp.equals( md5New ) ) + { + throw new RuntimeException( "**************** md5 mismatch!! *****************" ); + } + } + } + +} diff --git a/brouter-routing-app/src/main/java/btools/routingapp/BInstallerView.java b/brouter-routing-app/src/main/java/btools/routingapp/BInstallerView.java index 6c43fb7..b5e0422 100644 --- a/brouter-routing-app/src/main/java/btools/routingapp/BInstallerView.java +++ b/brouter-routing-app/src/main/java/btools/routingapp/BInstallerView.java @@ -687,7 +687,7 @@ float tx, ty; // first check for a delta file String md5 = Rd5DiffManager.getMD5( targetFile ); - String surlDelta = surlBase + "diff/" + name.replace( ".rd5", "/" + md5 + ".rd5diff" ); + String surlDelta = surlBase + "diff/" + name.replace( ".rd5", "/" + md5 + ".df5" ); URL urlDelta = new URL(surlDelta); From 372a04a6cf4608cf14bc7045aed9499012a23f52 Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Sat, 14 Nov 2020 10:10:08 +0100 Subject: [PATCH 02/12] fixed nullpointer in Rd5DiffTool --- .../src/main/java/btools/mapaccess/Rd5DiffTool.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java index 426e40a..ac2f1d9 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java @@ -347,7 +347,7 @@ final public class Rd5DiffTool implements ProgressListener byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false ); MicroCache mc2 = mcIn.readMC(); - int targetSize = getPosIdx( posIdx2, tileIdx ) - getPosIdx( posIdx2, tileIdx-1 ); + int targetSize = posIdx2 == null ? 0 : getPosIdx( posIdx2, tileIdx ) - getPosIdx( posIdx2, tileIdx-1 ); /* int targetSizeCmp = getPosIdx( posIdxCmp, tileIdx ) - getPosIdx( posIdxCmp, tileIdx-1 ); if ( targetSizeCmp != targetSize ) throw new IllegalArgumentException( "target size mismatch: "+ targetSize + "," + targetSizeCmp ); From eb716506fd934989177e8c3933b22c58b5a41574 Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Wed, 31 Mar 2021 16:07:21 +0200 Subject: [PATCH 03/12] bee-line patch: Nogo covering 2 Waypoints triggers a beeline segment --- .../src/main/java/btools/router/OsmPath.java | 5 +- .../java/btools/router/RoutingContext.java | 47 +++++++++++++++++-- .../java/btools/router/RoutingEngine.java | 32 +++++++++++-- 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/brouter-core/src/main/java/btools/router/OsmPath.java b/brouter-core/src/main/java/btools/router/OsmPath.java index c76089b..4a42b0c 100644 --- a/brouter-core/src/main/java/btools/router/OsmPath.java +++ b/brouter-core/src/main/java/btools/router/OsmPath.java @@ -139,7 +139,10 @@ abstract class OsmPath implements OsmLinkHolder protected void addAddionalPenalty(OsmTrack refTrack, boolean detailMode, OsmPath origin, OsmLink link, RoutingContext rc ) { byte[] description = link.descriptionBitmap; - if ( description == null ) throw new IllegalArgumentException( "null description for: " + link ); + if ( description == null ) + { + return; // could be a beeline path + } boolean recordTransferNodes = detailMode || rc.countTraffic; diff --git a/brouter-core/src/main/java/btools/router/RoutingContext.java b/brouter-core/src/main/java/btools/router/RoutingContext.java index 3e207fd..204a072 100644 --- a/brouter-core/src/main/java/btools/router/RoutingContext.java +++ b/brouter-core/src/main/java/btools/router/RoutingContext.java @@ -16,6 +16,7 @@ import btools.expressions.BExpressionContextNode; import btools.expressions.BExpressionContextWay; import btools.mapaccess.GeometryDecoder; import btools.mapaccess.OsmLink; +import btools.mapaccess.OsmNode; import btools.util.CheapAngleMeter; import btools.util.CheapRuler; @@ -191,6 +192,7 @@ public final class RoutingContext public List poipoints; public List nogopoints = null; + private List nogopoints_all = null; // full list not filtered for wayoints-in-nogos private List keepnogopoints = null; private OsmNodeNamed pendingEndpoint = null; @@ -258,14 +260,29 @@ public final class RoutingContext } } - public void cleanNogolist( List waypoints ) + /** + * restore the full nogolist previously saved by cleanNogoList + */ + public void restoreNogoList() { + nogopoints = nogopoints_all; + } + + /** + * clean the nogolist (previoulsy saved by saveFullNogolist()) + * by removing nogos with waypoints within + * + * @return true if all wayoints are all in the same (full-weigth) nogo area (triggering bee-line-mode) + */ + public void cleanNogoList( List waypoints ) + { + nogopoints_all = nogopoints; if ( nogopoints == null ) return; List nogos = new ArrayList(); for( OsmNodeNamed nogo : nogopoints ) { boolean goodGuy = true; - for( OsmNodeNamed wp : waypoints ) + for( OsmNode wp : waypoints ) { if ( wp.calcDistance( nogo ) < nogo.radius && (!(nogo instanceof OsmNogoPolygon) @@ -274,7 +291,6 @@ public final class RoutingContext : ((OsmNogoPolygon)nogo).isOnPolyline(wp.ilon, wp.ilat)))) { goodGuy = false; - break; } } if ( goodGuy ) nogos.add( nogo ); @@ -282,6 +298,31 @@ public final class RoutingContext nogopoints = nogos.isEmpty() ? null : nogos; } + public boolean allInOneNogo( List waypoints ) + { + if ( nogopoints == null ) return false; + boolean allInTotal = false; + for( OsmNodeNamed nogo : nogopoints ) + { + boolean allIn = Double.isNaN( nogo.nogoWeight ); + for( OsmNode wp : waypoints ) + { + int dist = wp.calcDistance( nogo ); + if ( dist < nogo.radius + && (!(nogo instanceof OsmNogoPolygon) + || (((OsmNogoPolygon)nogo).isClosed + ? ((OsmNogoPolygon)nogo).isWithin(wp.ilon, wp.ilat) + : ((OsmNogoPolygon)nogo).isOnPolyline(wp.ilon, wp.ilat)))) + { + continue; + } + allIn = false; + } + allInTotal |= allIn; + } + return allInTotal; + } + public long[] getNogoChecksums() { long[] cs = new long[3]; diff --git a/brouter-core/src/main/java/btools/router/RoutingEngine.java b/brouter-core/src/main/java/btools/router/RoutingEngine.java index 9fcea0b..1f9c99b 100644 --- a/brouter-core/src/main/java/btools/router/RoutingEngine.java +++ b/brouter-core/src/main/java/btools/router/RoutingEngine.java @@ -151,9 +151,6 @@ public class RoutingEngine extends Thread { try { - // delete nogos with waypoints in them - routingContext.cleanNogolist( waypoints ); - startTime = System.currentTimeMillis(); long startTime0 = startTime; this.maxRunningTime = maxRunningTime; @@ -459,6 +456,29 @@ public class RoutingEngine extends Thread } private OsmTrack searchTrack( MatchedWaypoint startWp, MatchedWaypoint endWp, OsmTrack nearbyTrack, OsmTrack refTrack ) + { + // remove nogos with waypoints inside + try + { + List wpts2 = new ArrayList(); + wpts2.add( startWp.waypoint ); + wpts2.add( endWp.waypoint ); + boolean calcBeeline = routingContext.allInOneNogo(wpts2); + + if ( !calcBeeline ) return searchRoutedTrack( startWp, endWp, nearbyTrack, refTrack ); + + // we want a beeline-segment + OsmPath path = routingContext.createPath( new OsmLink( null, startWp.crosspoint ) ); + path = routingContext.createPath( path, new OsmLink( startWp.crosspoint, endWp.crosspoint ), null, false ); + return compileTrack( path, false ); + } + finally + { + routingContext.restoreNogoList(); + } + } + + private OsmTrack searchRoutedTrack( MatchedWaypoint startWp, MatchedWaypoint endWp, OsmTrack nearbyTrack, OsmTrack refTrack ) { OsmTrack track = null; double[] airDistanceCostFactors = new double[]{ routingContext.pass1coefficient, routingContext.pass2coefficient }; @@ -650,6 +670,11 @@ public class RoutingEngine extends Thread { try { + List wpts2 = new ArrayList(); + if ( startWp != null ) wpts2.add( startWp.waypoint ); + if ( endWp != null ) wpts2.add( endWp.waypoint ); + routingContext.cleanNogoList(wpts2); + boolean detailed = guideTrack != null; resetCache( detailed ); nodesCache.nodesMap.cleanupMode = detailed ? 0 : ( routingContext.considerTurnRestrictions ? 2 : 1 ); @@ -657,6 +682,7 @@ public class RoutingEngine extends Thread } finally { + routingContext.restoreNogoList(); nodesCache.clean( false ); // clean only non-virgin caches } } From 1e542d19b7c45a4ee685acf20a858934bb4e2f6c Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Mon, 5 Apr 2021 20:58:15 +0200 Subject: [PATCH 04/12] more aliases for highway=proposed|abandoned --- misc/profiles2/lookups.dat | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misc/profiles2/lookups.dat b/misc/profiles2/lookups.dat index d8e935b..5a9599e 100644 --- a/misc/profiles2/lookups.dat +++ b/misc/profiles2/lookups.dat @@ -27,10 +27,10 @@ highway;0000079637 tertiary_link highway;0000070238 construction highway;0000058257 bridleway highway;0000039003 platform -highway;0000037192 proposed +highway;0000037192 proposed planned virtual highway;0000010307 raceway highway;0000003152 rest_area -highway;0000002942 abandoned +highway;0000002942 abandoned disused razed demolished dismantled highway;0000002631 services highway;0000002133 corridor highway;0000002093 crossing @@ -227,7 +227,7 @@ cycleway;0000000892 left cycleway;0000000399 street cycleway;0000000344 shoulder cycleway;0000000326 designated -cycleway;0000000247 proposed +cycleway;0000000247 proposed planned virtual cycleway;0000000224 cyclestreet cycleway;0000000172 path cycleway;0000000154 sidewalk From 3565f39914a05b04ef648a6a68b70ae2b5c10e91 Mon Sep 17 00:00:00 2001 From: Norbert Renner Date: Sat, 10 Apr 2021 10:04:50 +0200 Subject: [PATCH 05/12] Complete voice hints and add times to GeoJSON --- .../src/main/java/btools/router/OsmTrack.java | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/brouter-core/src/main/java/btools/router/OsmTrack.java b/brouter-core/src/main/java/btools/router/OsmTrack.java index c732a03..e2a6145 100644 --- a/brouter-core/src/main/java/btools/router/OsmTrack.java +++ b/brouter-core/src/main/java/btools/router/OsmTrack.java @@ -19,8 +19,11 @@ import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.io.StringWriter; +import java.text.DecimalFormat; +import java.text.NumberFormat; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import btools.mapaccess.MatchedWaypoint; import btools.mapaccess.OsmPos; @@ -710,6 +713,8 @@ public final class OsmTrack public String formatAsGeoJson() { + int turnInstructionMode = voiceHints != null ? voiceHints.turnInstructionMode : 0; + StringBuilder sb = new StringBuilder( 8192 ); sb.append( "{\n" ); @@ -731,7 +736,20 @@ public final class OsmTrack sb.append( " \"voicehints\": [\n" ); for( VoiceHint hint: voiceHints.list ) { - sb.append( " [" ).append( hint.indexInTrack ).append( ',' ).append( hint.getCommand() ).append( ',' ).append( hint.getExitNumber() ).append( "],\n" ); + sb.append( " [" ); + sb.append( hint.indexInTrack ); + sb.append( ',' ).append( hint.getCommand() ); + sb.append( ',' ).append( hint.getExitNumber() ); + sb.append( ',' ).append( hint.distanceToNext ); + sb.append( ',' ).append( (int) hint.angle ); + + // not always include geometry because longer and only needed for comment style + if ( turnInstructionMode == 4 ) // comment style + { + sb.append( ",\"" ).append( hint.formatGeometry() ).append( "\"" ); + } + + sb.append( "],\n" ); } sb.deleteCharAt( sb.lastIndexOf( "," ) ); sb.append( " ],\n" ); @@ -746,7 +764,7 @@ public final class OsmTrack { sb.append( " [" ).append( sp.get(i) ).append( i> 0 ? "],\n" : "]\n" ); } - sb.append( " ]\n" ); + sb.append( " ],\n" ); } } else // ... otherwise traditional message list @@ -758,9 +776,24 @@ public final class OsmTrack sb.append( " [\"" ).append( m.replaceAll( "\t", "\", \"" ) ).append( "\"],\n" ); } sb.deleteCharAt( sb.lastIndexOf( "," ) ); - sb.append( " ]\n" ); + sb.append( " ],\n" ); } + + if ( getTotalSeconds() > 0 ) { + sb.append( " \"times\": [" ); + DecimalFormat decimalFormat = (DecimalFormat) NumberFormat.getInstance( Locale.ENGLISH ); + decimalFormat.applyPattern( "0.###" ); + for ( OsmPathElement n : nodes ) { + sb.append( decimalFormat.format( n.getTime() ) ).append( "," ); + } + sb.deleteCharAt( sb.lastIndexOf( "," ) ); + sb.append( "]\n" ); + } else { + sb.deleteCharAt( sb.lastIndexOf( "," ) ); + } + sb.append( " },\n" ); + if ( iternity != null ) { sb.append( " \"iternity\": [\n" ); From 8206a1ae845982512f69c8de8fb0d28643552ee7 Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Sun, 18 Apr 2021 17:35:50 +0200 Subject: [PATCH 06/12] ip/session monitoring --- .../java/btools/server/IpAccessMonitor.java | 42 +++++++++++++++++++ .../main/java/btools/server/RouteServer.java | 19 ++++++--- 2 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 brouter-server/src/main/java/btools/server/IpAccessMonitor.java diff --git a/brouter-server/src/main/java/btools/server/IpAccessMonitor.java b/brouter-server/src/main/java/btools/server/IpAccessMonitor.java new file mode 100644 index 0000000..2aa1c8e --- /dev/null +++ b/brouter-server/src/main/java/btools/server/IpAccessMonitor.java @@ -0,0 +1,42 @@ +package btools.server; + +import java.util.HashMap; +import java.util.Map; + +public class IpAccessMonitor +{ + private static Object sync = new Object(); + private static HashMap ipAccess = new HashMap(); + private static long MAX_IDLE = 900000; // 15 minutes + + public static boolean touchIpAccess( String ip ) + { + long t = System.currentTimeMillis(); + synchronized( sync ) + { + Long lastTime = ipAccess.get( ip ); + if ( lastTime == null || t - lastTime.longValue() > MAX_IDLE ) + { + ipAccess.put( ip, Long.valueOf( t ) ); + cleanup(t); + return true; // new session detected + } + ipAccess.put( ip, Long.valueOf( t ) ); // touch existing session + return false; + } + } + + private static void cleanup( long t ) + { + HashMap newMap = new HashMap(ipAccess.size()); + for( Map.Entry e : ipAccess.entrySet() ) + { + if ( t - e.getValue().longValue() <= MAX_IDLE ) + { + newMap.put( e.getKey(), e.getValue() ); + } + } + ipAccess = newMap; + } + +} diff --git a/brouter-server/src/main/java/btools/server/RouteServer.java b/brouter-server/src/main/java/btools/server/RouteServer.java index dbb3624..ce79ec2 100644 --- a/brouter-server/src/main/java/btools/server/RouteServer.java +++ b/brouter-server/src/main/java/btools/server/RouteServer.java @@ -82,6 +82,7 @@ public class RouteServer extends Thread implements Comparable String getline = null; String agent = null; String encodings = null; + String xff = null; // X-Forwarded-For // more headers until first empty line for(;;) @@ -102,13 +103,18 @@ public class RouteServer extends Thread implements Comparable { getline = line; } - if ( line.startsWith( "User-Agent: " ) ) + line = line.toLowerCase(); + if ( line.startsWith( "user-agent: " ) ) { - agent = line.substring( "User-Agent: ".length() ); + agent = line.substring( "user-agent: ".length() ); } - if ( line.startsWith( "Accept-Encoding: " ) ) + if ( line.startsWith( "accept-encoding: " ) ) { - encodings = line.substring( "Accept-Encoding: ".length() ); + encodings = line.substring( "accept-encoding: ".length() ); + } + if ( line.startsWith( "x-forwarded-for: " ) ) + { + xff = line.substring( "x-forwarded-for: ".length() ); } } @@ -143,8 +149,11 @@ public class RouteServer extends Thread implements Comparable return; } + InetAddress ip = clientSocket.getInetAddress(); - System.out.println( formattedTimestamp() + " ip=" + (ip==null ? "null" : ip.toString() ) + " -> " + getline ); + String sIp = xff == null ? (ip==null ? "null" : ip.toString() ) : xff; + String sessionMode = IpAccessMonitor.touchIpAccess( sIp ) ? " new " : " "; + System.out.println( formattedTimestamp() + sessionMode + " ip=" + sIp + " -> " + getline ); String url = getline.split(" ")[1]; HashMap params = getUrlParams(url); From 31e761e731519df0fd2908e16d8d93492b7af9f5 Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Sun, 18 Apr 2021 19:32:07 +0200 Subject: [PATCH 07/12] session/ip info --- .../java/btools/server/IpAccessMonitor.java | 24 +++++++++++++------ .../main/java/btools/server/RouteServer.java | 11 +++++++-- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/brouter-server/src/main/java/btools/server/IpAccessMonitor.java b/brouter-server/src/main/java/btools/server/IpAccessMonitor.java index 2aa1c8e..3f3da83 100644 --- a/brouter-server/src/main/java/btools/server/IpAccessMonitor.java +++ b/brouter-server/src/main/java/btools/server/IpAccessMonitor.java @@ -8,21 +8,31 @@ public class IpAccessMonitor private static Object sync = new Object(); private static HashMap ipAccess = new HashMap(); private static long MAX_IDLE = 900000; // 15 minutes - + private static long CLEANUP_INTERVAL = 10000; // 10 seconds + private static long lastCleanup; + public static boolean touchIpAccess( String ip ) { long t = System.currentTimeMillis(); synchronized( sync ) { Long lastTime = ipAccess.get( ip ); - if ( lastTime == null || t - lastTime.longValue() > MAX_IDLE ) + ipAccess.put( ip, Long.valueOf( t ) ); + return lastTime == null || t - lastTime.longValue() > MAX_IDLE; + } + } + + public static int getSessionCount() + { + long t = System.currentTimeMillis(); + synchronized( sync ) + { + if ( t - lastCleanup > CLEANUP_INTERVAL ) { - ipAccess.put( ip, Long.valueOf( t ) ); - cleanup(t); - return true; // new session detected + cleanup( t ); + lastCleanup = t; } - ipAccess.put( ip, Long.valueOf( t ) ); // touch existing session - return false; + return ipAccess.size(); } } diff --git a/brouter-server/src/main/java/btools/server/RouteServer.java b/brouter-server/src/main/java/btools/server/RouteServer.java index ce79ec2..4d0887e 100644 --- a/brouter-server/src/main/java/btools/server/RouteServer.java +++ b/brouter-server/src/main/java/btools/server/RouteServer.java @@ -152,8 +152,15 @@ public class RouteServer extends Thread implements Comparable InetAddress ip = clientSocket.getInetAddress(); String sIp = xff == null ? (ip==null ? "null" : ip.toString() ) : xff; - String sessionMode = IpAccessMonitor.touchIpAccess( sIp ) ? " new " : " "; - System.out.println( formattedTimestamp() + sessionMode + " ip=" + sIp + " -> " + getline ); + boolean newSession = IpAccessMonitor.touchIpAccess( sIp ); + String sessionInfo = " new"; + if ( !newSession ) + { + int sessionCount = IpAccessMonitor.getSessionCount(); + sessionInfo = " " + Math.min( sessionCount, 999 ); + sessionInfo = sessionInfo.substring( sessionInfo.length() - 4 ); + } + System.out.println( formattedTimestamp() + sessionInfo + " ip=" + sIp + " -> " + getline ); String url = getline.split(" ")[1]; HashMap params = getUrlParams(url); From be71c0d86ed4e4a9b7679d1d92ef594e646c9006 Mon Sep 17 00:00:00 2001 From: radim-asamm Date: Thu, 22 Apr 2021 14:01:17 +0200 Subject: [PATCH 08/12] Added time and energy to response messages. --- brouter-core/src/main/java/btools/router/MessageData.java | 4 +++- brouter-core/src/main/java/btools/router/OsmTrack.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/brouter-core/src/main/java/btools/router/MessageData.java b/brouter-core/src/main/java/btools/router/MessageData.java index 10c3c8f..9e1fd0a 100644 --- a/brouter-core/src/main/java/btools/router/MessageData.java +++ b/brouter-core/src/main/java/btools/router/MessageData.java @@ -55,7 +55,9 @@ final class MessageData implements Cloneable + "\t" + linknodecost + "\t" + linkinitcost + "\t" + wayKeyValues - + "\t" + ( nodeKeyValues == null ? "" : nodeKeyValues ); + + "\t" + ( nodeKeyValues == null ? "" : nodeKeyValues ) + + "\t" + time + + "\t" + energy; } void add( MessageData d ) diff --git a/brouter-core/src/main/java/btools/router/OsmTrack.java b/brouter-core/src/main/java/btools/router/OsmTrack.java index e2a6145..3349cad 100644 --- a/brouter-core/src/main/java/btools/router/OsmTrack.java +++ b/brouter-core/src/main/java/btools/router/OsmTrack.java @@ -34,7 +34,7 @@ import btools.util.StringUtils; public final class OsmTrack { // csv-header-line - private static final String MESSAGES_HEADER = "Longitude\tLatitude\tElevation\tDistance\tCostPerKm\tElevCost\tTurnCost\tNodeCost\tInitialCost\tWayTags\tNodeTags"; + private static final String MESSAGES_HEADER = "Longitude\tLatitude\tElevation\tDistance\tCostPerKm\tElevCost\tTurnCost\tNodeCost\tInitialCost\tWayTags\tNodeTags\tTime\tEnergy"; public MatchedWaypoint endPoint; public long[] nogoChecksums; From 56b4263107296de29a851a0e7b0aff7027eb3c50 Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Sun, 2 May 2021 15:30:16 +0200 Subject: [PATCH 09/12] add response time and contention traces to logfile --- .../main/java/btools/server/RouteServer.java | 45 ++++++++++++------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/brouter-server/src/main/java/btools/server/RouteServer.java b/brouter-server/src/main/java/btools/server/RouteServer.java index 4d0887e..26ccfd3 100644 --- a/brouter-server/src/main/java/btools/server/RouteServer.java +++ b/brouter-server/src/main/java/btools/server/RouteServer.java @@ -61,7 +61,7 @@ public class RouteServer extends Thread implements Comparable private static DateFormat tsFormat = new SimpleDateFormat( "dd.MM.yy HH:mm", new Locale( "en", "US" ) ); - private static String formattedTimestamp() + private static String formattedTimeStamp( long t ) { synchronized( tsFormat ) { @@ -73,13 +73,17 @@ public class RouteServer extends Thread implements Comparable { BufferedReader br = null; BufferedWriter bw = null; + + // first line + String getline = null; + String sessionInfo = null; + String sIp = null; + try { br = new BufferedReader( new InputStreamReader( clientSocket.getInputStream() , "UTF-8") ); bw = new BufferedWriter( new OutputStreamWriter( clientSocket.getOutputStream(), "UTF-8" ) ); - // first line - String getline = null; String agent = null; String encodings = null; String xff = null; // X-Forwarded-For @@ -118,6 +122,17 @@ public class RouteServer extends Thread implements Comparable } } + InetAddress ip = clientSocket.getInetAddress(); + sIp = xff == null ? (ip==null ? "null" : ip.toString() ) : xff; + boolean newSession = IpAccessMonitor.touchIpAccess( sIp ); + sessionInfo = " new"; + if ( !newSession ) + { + int sessionCount = IpAccessMonitor.getSessionCount(); + sessionInfo = " " + Math.min( sessionCount, 999 ); + sessionInfo = sessionInfo.substring( sessionInfo.length() - 4 ); + } + String excludedAgents = System.getProperty( "excludedAgents" ); if ( agent != null && excludedAgents != null ) { @@ -149,19 +164,6 @@ public class RouteServer extends Thread implements Comparable return; } - - InetAddress ip = clientSocket.getInetAddress(); - String sIp = xff == null ? (ip==null ? "null" : ip.toString() ) : xff; - boolean newSession = IpAccessMonitor.touchIpAccess( sIp ); - String sessionInfo = " new"; - if ( !newSession ) - { - int sessionCount = IpAccessMonitor.getSessionCount(); - sessionInfo = " " + Math.min( sessionCount, 999 ); - sessionInfo = sessionInfo.substring( sessionInfo.length() - 4 ); - } - System.out.println( formattedTimestamp() + sessionInfo + " ip=" + sIp + " -> " + getline ); - String url = getline.split(" ")[1]; HashMap params = getUrlParams(url); @@ -296,8 +298,12 @@ public class RouteServer extends Thread implements Comparable { threadPoolSync.notifyAll(); } + long t = System.currentTimeMillis(); + long ms = t - starttime; + System.out.println( formattedTimeStamp(t) + sessionInfo + " ip=" + sIp + " ms=" + ms + " -> " + getline ); } } + public static void main(String[] args) throws Exception { @@ -371,13 +377,18 @@ public class RouteServer extends Thread implements Comparable { threadPoolSync.wait( maxWaitTime ); } + long t = System.currentTimeMillis(); + System.out.println( formattedTimeStamp(t) + " contention! ms waited " + (t - server.starttime) ); } cleanupThreadQueue( threadQueue ); if ( threadQueue.size() >= maxthreads ) { if ( debug ) System.out.println( "stopping oldest thread..." ); // no way... stop the oldest thread - threadQueue.poll().stopRouter(); + RouteServer oldest = threadQueue.poll(); + oldest.stopRouter(); + long t = System.currentTimeMillis(); + System.out.println( formattedTimeStamp(t) + " contention! ms killed " + (t - oldest.starttime) ); } } From 8234293e5aa3a35bb4b2cd50d13355eb1922af4f Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Sun, 2 May 2021 15:31:41 +0200 Subject: [PATCH 10/12] time/energy in message data only integer precision --- brouter-core/src/main/java/btools/router/MessageData.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brouter-core/src/main/java/btools/router/MessageData.java b/brouter-core/src/main/java/btools/router/MessageData.java index 9e1fd0a..3e3b6bb 100644 --- a/brouter-core/src/main/java/btools/router/MessageData.java +++ b/brouter-core/src/main/java/btools/router/MessageData.java @@ -56,8 +56,8 @@ final class MessageData implements Cloneable + "\t" + linkinitcost + "\t" + wayKeyValues + "\t" + ( nodeKeyValues == null ? "" : nodeKeyValues ) - + "\t" + time - + "\t" + energy; + + "\t" + ((int)time) + + "\t" + ((int)energy); } void add( MessageData d ) From 55a159af5d005ce94679f17639e0a14eaea34924 Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Sat, 22 May 2021 13:39:47 +0200 Subject: [PATCH 11/12] suspect-manager: sentinel-2 link --- brouter-server/src/main/java/btools/server/SuspectManager.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/brouter-server/src/main/java/btools/server/SuspectManager.java b/brouter-server/src/main/java/btools/server/SuspectManager.java index 5e8c929..f9a62f1 100644 --- a/brouter-server/src/main/java/btools/server/SuspectManager.java +++ b/brouter-server/src/main/java/btools/server/SuspectManager.java @@ -450,6 +450,8 @@ public class SuspectManager extends Thread String url5 = "https://tyrasd.github.io/latest-changes/#16/" + dlat + "/" + dlon; + String url6 = "https://apps.sentinel-hub.com/sentinel-playground/?source=S2L2A&lat=" + dlat + "&lng=" + dlon + "&zoom=15"; + if ( message != null ) { bw.write( "" + message + "

\n" ); @@ -459,6 +461,7 @@ public class SuspectManager extends Thread bw.write( "Open in JOSM (via remote control)

\n" ); bw.write( "Overpass: minus one week    node context

\n" ); bw.write( "Open in Latest-Changes / last week

\n" ); + bw.write( "Current Sentinel-2 imagary

\n" ); bw.write( "
\n" ); if ( isFixed( id, suspects.timestamp ) ) { From cdac1cc5aaa97d3bda16c9f2682f74560ab9b244 Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Wed, 26 May 2021 08:36:52 +0200 Subject: [PATCH 12/12] spam heuristics --- .../main/java/btools/server/RouteServer.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/brouter-server/src/main/java/btools/server/RouteServer.java b/brouter-server/src/main/java/btools/server/RouteServer.java index 26ccfd3..7125dc8 100644 --- a/brouter-server/src/main/java/btools/server/RouteServer.java +++ b/brouter-server/src/main/java/btools/server/RouteServer.java @@ -87,6 +87,7 @@ public class RouteServer extends Thread implements Comparable String agent = null; String encodings = null; String xff = null; // X-Forwarded-For + String referer = null; // more headers until first empty line for(;;) @@ -120,6 +121,14 @@ public class RouteServer extends Thread implements Comparable { xff = line.substring( "x-forwarded-for: ".length() ); } + if ( line.startsWith( "Referer: " ) ) + { + referer = line.substring( "Referer: ".length() ); + } + if ( line.startsWith( "Referrer: " ) ) + { + referer = line.substring( "Referrer: ".length() ); + } } InetAddress ip = clientSocket.getInetAddress(); @@ -149,6 +158,17 @@ public class RouteServer extends Thread implements Comparable } } + if ( referer != null && referer.indexOf( "brouter.de/brouter-web" ) >= 0 ) + { + if ( getline.indexOf( "%7C" ) >= 0 && getline.indexOf( "%2C" ) >= 0 ) + { + writeHttpHeader( bw, HTTP_STATUS_FORBIDDEN ); + bw.write( "Spam? please stop" ); + bw.flush(); + return; + } + } + if ( getline.startsWith("GET /favicon.ico") ) { writeHttpHeader( bw, HTTP_STATUS_NOT_FOUND );