From ab621d2b2e824e95998cc9e785d78c05337c6eb0 Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Sat, 28 Sep 2019 12:56:36 +0200 Subject: [PATCH] pre-process speed: restrictions file split --- .../java/btools/mapcreator/OsmCutter.java | 21 +++++++-- .../java/btools/mapcreator/OsmFastCutter.java | 14 +++++- .../btools/mapcreator/RestrictionCutter.java | 44 +++++++++++++++++++ .../btools/mapcreator/RestrictionCutter5.java | 43 ++++++++++++++++++ .../java/btools/mapcreator/WayCutter.java | 8 +++- .../java/btools/mapcreator/WayCutter5.java | 39 +++++++++++++++- .../java/btools/mapcreator/WayLinker.java | 42 ++++++++++-------- 7 files changed, 186 insertions(+), 25 deletions(-) create mode 100644 brouter-map-creator/src/main/java/btools/mapcreator/RestrictionCutter.java create mode 100644 brouter-map-creator/src/main/java/btools/mapcreator/RestrictionCutter5.java diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java b/brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java index fa76e58..749950c 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java @@ -31,6 +31,7 @@ public class OsmCutter extends MapCreatorBase private DataOutputStream restrictionsDos; public WayCutter wayCutter; + public RestrictionCutter restrictionCutter; public NodeFilter nodeFilter; public static void main(String[] args) throws Exception @@ -83,7 +84,10 @@ public class OsmCutter extends MapCreatorBase wayDos = wayFile == null ? null : new DataOutputStream( new BufferedOutputStream( new FileOutputStream( wayFile ) ) ); cyclewayDos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( relFile ) ) ); - restrictionsDos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( resFile ) ) ); + if ( resFile != null ) + { + restrictionsDos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( resFile ) ) ); + } // read the osm map into memory long t0 = System.currentTimeMillis(); @@ -99,7 +103,10 @@ public class OsmCutter extends MapCreatorBase wayDos.close(); } cyclewayDos.close(); - restrictionsDos.close(); + if ( restrictionsDos != null ) + { + restrictionsDos.close(); + } // System.out.println( "-------- way-statistics -------- " ); // _expctxWayStat.dumpStatistics(); @@ -296,7 +303,15 @@ public class OsmCutter extends MapCreatorBase res.fromWid = fromWid; res.toWid = toWid; res.viaNid = viaNid; - res.writeTo( restrictionsDos ); + + if ( restrictionsDos != null ) + { + res.writeTo( restrictionsDos ); + } + if ( restrictionCutter != null ) + { + restrictionCutter.nextRestriction( res ); + } } diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/OsmFastCutter.java b/brouter-map-creator/src/main/java/btools/mapcreator/OsmFastCutter.java index 69ca8fa..c222e4e 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/OsmFastCutter.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/OsmFastCutter.java @@ -48,13 +48,18 @@ public class OsmFastCutter extends MapCreatorBase cutter.wayCutter = new WayCutter(); cutter.wayCutter.init( wayDir ); + // ... inject RestrictionCutter + cutter.restrictionCutter = new RestrictionCutter(); + cutter.restrictionCutter.init( new File( nodeDir.getParentFile(), "restrictions" ), cutter.wayCutter ); + // ... inject NodeFilter NodeFilter nodeFilter = new NodeFilter(); nodeFilter.init(); cutter.nodeFilter = nodeFilter; - cutter.process( lookupFile, nodeDir, null, relFile, resFile, profileAll, mapFile ); + cutter.process( lookupFile, nodeDir, null, relFile, null, profileAll, mapFile ); cutter.wayCutter.finish(); + cutter.restrictionCutter.finish(); cutter = null; // ***** run WayCutter5 **** @@ -64,6 +69,10 @@ public class OsmFastCutter extends MapCreatorBase wayCut5.relMerger = new RelationMerger(); wayCut5.relMerger.init( relFile, lookupFile, profileReport, profileCheck ); + // ... inject RestrictionCutter5 + wayCut5.restrictionCutter5 = new RestrictionCutter5(); + wayCut5.restrictionCutter5.init( new File( nodeDir.getParentFile(), "restrictions55" ), wayCut5 ); + //... inject NodeFilter wayCut5.nodeFilter = nodeFilter; @@ -72,5 +81,8 @@ public class OsmFastCutter extends MapCreatorBase wayCut5.nodeCutter.init( node55Dir ); wayCut5.process( nodeDir, wayDir, way55Dir, borderFile ); + + wayCut5.restrictionCutter5.finish(); + } } diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/RestrictionCutter.java b/brouter-map-creator/src/main/java/btools/mapcreator/RestrictionCutter.java new file mode 100644 index 0000000..6b4e096 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapcreator/RestrictionCutter.java @@ -0,0 +1,44 @@ +package btools.mapcreator; + +import java.io.File; + +/** + * RestrictionCutter writes Restrictions to tiles + * + * - cut the way file into 45*30 - pieces + * - enrich ways with relation information + * + * @author ab + */ +public class RestrictionCutter extends MapCreatorBase +{ + private WayCutter wayCutter; + + public void init( File outTileDir, WayCutter wayCutter ) throws Exception + { + outTileDir.mkdir(); + this.outTileDir = outTileDir; + this.wayCutter = wayCutter; + } + + public void finish() throws Exception + { + closeTileOutStreams(); + } + + public void nextRestriction( RestrictionData data ) throws Exception + { + int tileIndex = wayCutter.getTileIndexForNid( data.viaNid ); + if ( tileIndex != -1 ) + { + data.writeTo( getOutStreamForTile( tileIndex ) ); + } + } + + protected String getNameForTile( int tileIndex ) + { + String name = wayCutter.getNameForTile( tileIndex ); + return name.substring( 0, name.length()-3 ) + "rtl"; + } + +} diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/RestrictionCutter5.java b/brouter-map-creator/src/main/java/btools/mapcreator/RestrictionCutter5.java new file mode 100644 index 0000000..83b74d8 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapcreator/RestrictionCutter5.java @@ -0,0 +1,43 @@ +package btools.mapcreator; + +import java.io.File; + +/** + * RestrictionCutter5 does 1 step in map-processing: + * + * - cut the 45*30 restriction files into 5*5 pieces + * + * @author ab + */ +public class RestrictionCutter5 extends MapCreatorBase +{ + private WayCutter5 wayCutter5; + + public void init( File outTileDir, WayCutter5 wayCutter5 ) throws Exception + { + outTileDir.mkdir(); + this.outTileDir = outTileDir; + this.wayCutter5 = wayCutter5; + } + + public void finish() throws Exception + { + closeTileOutStreams(); + } + + + public void nextRestriction( RestrictionData data ) throws Exception + { + int tileIndex = wayCutter5.getTileIndexForNid( data.viaNid ); + if ( tileIndex != -1 ) + { + data.writeTo( getOutStreamForTile( tileIndex ) ); + } + } + + protected String getNameForTile( int tileIndex ) + { + String name = wayCutter5.getNameForTile( tileIndex ); + return name.substring( 0, name.length()-3 ) + "rt5"; + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java b/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java index 8e3acc8..28529b7 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java @@ -86,6 +86,12 @@ public class WayCutter extends MapCreatorBase } } + + public int getTileIndexForNid( long nid ) + { + return tileIndexMap.getInt( nid ); + } + private int getTileIndex( int ilon, int ilat ) { int lon = ilon / 45000000; @@ -94,7 +100,7 @@ public class WayCutter extends MapCreatorBase return lon*6 + lat; } - protected String getNameForTile( int tileIndex ) + public String getNameForTile( int tileIndex ) { int lon = (tileIndex / 6 ) * 45 - 180; int lat = (tileIndex % 6 ) * 30 - 90; diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter5.java b/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter5.java index 21a587c..4d47331 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter5.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter5.java @@ -1,7 +1,11 @@ package btools.mapcreator; +import java.io.BufferedInputStream; +import java.io.DataInputStream; import java.io.DataOutputStream; +import java.io.EOFException; import java.io.File; +import java.io.FileInputStream; import btools.util.DenseLongMap; import btools.util.TinyDenseLongMap; @@ -25,6 +29,7 @@ public class WayCutter5 extends MapCreatorBase public RelationMerger relMerger; public NodeFilter nodeFilter; public NodeCutter nodeCutter; + public RestrictionCutter5 restrictionCutter5; public static void main(String[] args) throws Exception { @@ -65,7 +70,34 @@ public class WayCutter5 extends MapCreatorBase { nodeCutter.nodeFileStart( null ); } - new NodeIterator( this, false ).processFile( nodefile ); + new NodeIterator( this, nodeCutter != null ).processFile( nodefile ); + + if ( restrictionCutter5 != null ) + { + String resfilename = name.substring( 0, name.length()-3 ) + "rtl"; + File resfile = new File( "restrictions", resfilename ); + + if ( resfile.exists() ) + { + // read restrictions for nodes in nodesMap + DataInputStream di = new DataInputStream( new BufferedInputStream ( new FileInputStream( resfile ) ) ); + int ntr = 0; + try + { + for(;;) + { + RestrictionData res = new RestrictionData( di ); + restrictionCutter5.nextRestriction( res ); + ntr++; + } + } + catch( EOFException eof ) + { + di.close(); + } + System.out.println( "read " + ntr + " turn-restrictions" ); + } + } return true; } @@ -144,6 +176,11 @@ public class WayCutter5 extends MapCreatorBase } } + public int getTileIndexForNid( long nid ) + { + return tileIndexMap.getInt( nid ); + } + private int getTileIndex( int ilon, int ilat ) { int lonoff = (ilon / 45000000 ) * 45; diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java b/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java index 8e42e0e..3d45117 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java @@ -146,34 +146,38 @@ public class WayLinker extends MapCreatorBase FrozenLongMap nodesMapFrozen = new FrozenLongMap( nodesMap ); nodesMap = nodesMapFrozen; + File restrictionFile = fileFromTemplate( wayfile, new File( nodeTilesIn.getParentFile(), "restrictions55" ), "rt5" ); // read restrictions for nodes in nodesMap - DataInputStream di = new DataInputStream( new BufferedInputStream ( new FileInputStream( restrictionsFileIn ) ) ); - int ntr = 0; - try + if ( restrictionFile.exists() ) { - for(;;) + DataInputStream di = new DataInputStream( new BufferedInputStream ( new FileInputStream( restrictionFile ) ) ); + int ntr = 0; + try { - RestrictionData res = new RestrictionData( di ); - OsmNodeP n = nodesMap.get( res.viaNid ); - if ( n != null ) + for(;;) { - if ( ! ( n instanceof OsmNodePT ) ) + RestrictionData res = new RestrictionData( di ); + OsmNodeP n = nodesMap.get( res.viaNid ); + if ( n != null ) { - n = new OsmNodePT( n ); - nodesMap.put( res.viaNid, n ); + if ( ! ( n instanceof OsmNodePT ) ) + { + n = new OsmNodePT( n ); + nodesMap.put( res.viaNid, n ); + } + OsmNodePT nt = (OsmNodePT) n; + res.next = nt.firstRestriction; + nt.firstRestriction = res; + ntr++; } - OsmNodePT nt = (OsmNodePT) n; - res.next = nt.firstRestriction; - nt.firstRestriction = res; - ntr++; } } + catch( EOFException eof ) + { + di.close(); + } + System.out.println( "read " + ntr + " turn-restrictions" ); } - catch( EOFException eof ) - { - di.close(); - } - System.out.println( "read " + ntr + " turn-restrictions" ); nodesList = nodesMapFrozen.getValueList(); }