diff --git a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java index 9477abf..72540b5 100644 --- a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java +++ b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java @@ -24,6 +24,7 @@ public final class BExpressionContext { private static final String CONTEXT_TAG = "---context:"; private static final String VERSION_TAG = "---lookupversion:"; + private static final String VARLENGTH_TAG = "---readvarlength"; private String context; private boolean _inOurContext = false; @@ -77,6 +78,7 @@ public final class BExpressionContext private int linenr; public short lookupVersion = -1; + public boolean readVarLength = false; public BExpressionContext( String context ) { @@ -111,6 +113,8 @@ public final class BExpressionContext public byte[] encode( int[] ld ) { + if ( !readVarLength ) return encodeFix( ld ); + // start with first bit hardwired ("reversedirection") BitCoderContext ctx = new BitCoderContext( abBuf ); ctx.encodeBit( ld[0] != 0 ); @@ -126,11 +130,10 @@ public final class BExpressionContext skippedTags++; continue; } - int n = lookupValues.get(inum).length - 1; - if ( n == 2 ) { n = 1; d = d == 2 ? 1 : 0; } // 1-bit encoding for booleans ctx.encodeDistance( skippedTags ); skippedTags = 0; - ctx.encode( n, d ); + int n = lookupValues.get(inum).length - 2; + if ( n > 1 ) ctx.encode( n, d-1 ); // booleans are encoded just by presence... } ctx.encodeDistance( skippedTags ); int len = ctx.getEncodedLength(); @@ -139,6 +142,10 @@ public final class BExpressionContext return ab; } + private byte[] encodeFix( int[] ld ) + { + throw new IllegalArgumentException( "encoding fixed-length not supporte" ); + } /** @@ -146,7 +153,9 @@ public final class BExpressionContext */ public void decode( int[] ld, byte[] ab ) { - BitCoderContext ctx = new BitCoderContext(ab); + if ( ab.length == 8 ) { decodeFix( ld, ab ); return; } + + BitCoderContext ctx = new BitCoderContext(ab); // start with first bit hardwired ("reversedirection") ld[0] = ctx.decodeBit() ? 2 : 0; @@ -157,15 +166,47 @@ public final class BExpressionContext int skip = ctx.decodeDistance(); while ( skip-- > 0 ) ld[inum++] = 0; if ( inum >= lookupValues.size() ) break; - int nv = lookupValues.get(inum).length; - int n = nv == 3 ? 1 : nv-1; // 1-bit encoding for booleans - - int d = ctx.decode( n ); - if ( nv == 3 && d == 1 ) d = 2; // 1-bit encoding for booleans - ld[inum] = d; + int n = lookupValues.get(inum).length - 2; + if ( n > 1 ) + { + ld[inum] = ctx.decode( n ) + 1; + } + else + { + ld[inum] = 2; // boolean + } } } + /** + * decode old, 64-bit-fixed-length format + */ + public void decodeFix( int[] ld, byte[] ab ) + { + int idx = 0; + long i7 = ab[idx++]& 0xff; + long i6 = ab[idx++]& 0xff; + long i5 = ab[idx++]& 0xff; + long i4 = ab[idx++]& 0xff; + long i3 = ab[idx++]& 0xff; + long i2 = ab[idx++]& 0xff; + long i1 = ab[idx++]& 0xff; + long i0 = ab[idx++]& 0xff; + long w = (i7 << 56) + (i6 << 48) + (i5 << 40) + (i4 << 32) + (i3 << 24) + (i2 << 16) + (i1 << 8) + i0; + + for( int inum = lookupValues.size()-1; inum >= 0; inum-- ) // loop over lookup names + { + int nv = lookupValues.get(inum).length; + int n = nv == 3 ? 1 : nv-1; // 1-bit encoding for booleans + int m = 0; + long ww = w; + while( n != 0 ) { n >>= 1; ww >>= 1; m = m<<1 | 1; } + int d = (int)(w & m); + if ( nv == 3 && d == 1 ) d = 2; // 1-bit encoding for booleans + ld[inum] = d; + w = ww; + } + } public String getCsvDescription( byte[] ab ) { @@ -213,8 +254,7 @@ public final class BExpressionContext int parsedLines = 0; boolean ourContext = false; - if ( "way".equals( context ) ) addLookupValue( "reversedirection", "yes", null ); - else if ( "node".equals( context ) ) addLookupValue( "nodeaccessgranted", "yes", null ); + boolean fixTagsWritten = false; for(;;) { String line = br.readLine(); @@ -231,6 +271,11 @@ public final class BExpressionContext lookupVersion = Short.parseShort( line.substring( VERSION_TAG.length() ) ); continue; } + if ( line.startsWith( VARLENGTH_TAG ) ) + { + readVarLength = true; + continue; + } if ( !ourContext ) continue; parsedLines++; StringTokenizer tk = new StringTokenizer( line, " " ); @@ -238,8 +283,18 @@ public final class BExpressionContext String value = tk.nextToken(); int idx = name.indexOf( ';' ); if ( idx >= 0 ) name = name.substring( 0, idx ); - if ( "reversedirection".equals( name ) ) continue; // this is hardcoded - if ( "nodeaccessgranted".equals( name ) ) continue; // this is hardcoded + + if ( readVarLength ) + { + if ( !fixTagsWritten ) + { + fixTagsWritten = true; + if ( "way".equals( context ) ) addLookupValue( "reversedirection", "yes", null ); + else if ( "node".equals( context ) ) addLookupValue( "nodeaccessgranted", "yes", null ); + } + if ( "reversedirection".equals( name ) ) continue; // this is hardcoded + if ( "nodeaccessgranted".equals( name ) ) continue; // this is hardcoded + } BExpressionLookupValue newValue = addLookupValue( name, value, null ); // add aliases @@ -274,20 +329,20 @@ public final class BExpressionContext public void evaluate( boolean inverseDirection, byte[] ab, BExpressionReceiver receiver ) { + int inverseBitByteIndex = readVarLength ? 0 : 7; + _receiver = receiver; int abLen = ab.length; boolean equalsCurrent = currentHashBucket >= 0 && abLen == currentByteArray.length; if ( equalsCurrent ) { - if ( (inverseDirection ? ab[0] ^ 1 : ab[0] ) == currentByteArray[0] ) + for( int i=0; i { public static final int EXTERNAL_BITMASK = 0x80; - public static final int FIRSTFORWAY_BITMASK = 0x40; + public static final int VARIABLEDESC_BITMASK = 0x40; public static final int TRANSFERNODE_BITMASK = 0x20; public static final int WRITEDESC_BITMASK = 0x10; public static final int SKIPDETAILS_BITMASK = 0x08; @@ -144,14 +144,14 @@ public class OsmNodeP implements Comparable if ( targetLonIdx == lonIdx && targetLatIdx == latIdx ) { // reduced position for internal target - os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit ); + os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit | VARIABLEDESC_BITMASK ); os2.writeShort( (short)(target.ilon - lonIdx*62500 - 31250) ); os2.writeShort( (short)(target.ilat - latIdx*62500 - 31250) ); } else { // full position for external target - os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit | EXTERNAL_BITMASK ); + os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit | EXTERNAL_BITMASK | VARIABLEDESC_BITMASK ); os2.writeInt( target.ilon ); os2.writeInt( target.ilat ); } diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java b/brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java index db28f4c..808fe64 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java @@ -23,6 +23,7 @@ public class RelationMerger extends MapCreatorBase private CompactLongSet routesetall; private BExpressionContext expctxReport; private BExpressionContext expctxCheck; + private BExpressionContext expctxStat; private DataOutputStream wayOutStream; @@ -47,6 +48,7 @@ public class RelationMerger extends MapCreatorBase expctxCheck = new BExpressionContext("way"); expctxCheck.readMetaData( lookupFile ); expctxCheck.parseFile( checkProfile, "global" ); + expctxStat = new BExpressionContext("way"); // *** read the relation file into sets for each processed tag routesets = new HashMap(); @@ -76,6 +78,7 @@ public class RelationMerger extends MapCreatorBase { long wid = readId( dis ); if ( wid == -1 ) break; + expctxStat.addLookupValue( tagname, "yes", null ); if ( routeset != null && !routeset.contains( wid ) ) { routeset.add( wid ); @@ -92,14 +95,17 @@ public class RelationMerger extends MapCreatorBase { CompactLongSet routeset = new FrozenLongSet( routesets.get( tagname ) ); routesets.put( tagname, routeset ); - System.out.println( "marked " + routeset.size() + " ways for tag: " + tagname ); + System.out.println( "marked " + routeset.size() + " routes for tag: " + tagname ); } // *** finally process the way-file wayOutStream = createOutStream( wayFileOut ); new WayIterator( this, true ).processFile( wayFileIn ); wayOutStream.close(); - } + + System.out.println( "-------- route-statistics -------- " ); + expctxStat.dumpStatistics(); +} @Override public void nextWay( WayData data ) throws Exception diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java b/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java index dceea21..aa60509 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java @@ -20,16 +20,16 @@ public class WayCutter extends MapCreatorBase public static void main(String[] args) throws Exception { System.out.println("*** WayCutter: Soft-Cut way-data into tiles"); - if (args.length != 4) + if (args.length != 3) { - System.out.println("usage: java WayCutter " ); + System.out.println("usage: java WayCutter " ); return; } - new WayCutter().process( new File( args[0] ), new File( args[1] ), new File( args[2] ), new File( args[3] ) ); + new WayCutter().process( new File( args[0] ), new File( args[1] ), new File( args[2] ) ); } - public void process( File nodeTilesIn, File wayFileIn, File wayTilesOut, File relationFileIn ) throws Exception + public void process( File nodeTilesIn, File wayFileIn, File wayTilesOut ) throws Exception { this.outTileDir = wayTilesOut; diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java b/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java index 0aa69dd..c0abb39 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java @@ -8,6 +8,7 @@ import java.util.Collections; import java.util.List; import btools.expressions.BExpressionContext; +import btools.util.ByteArrayUnifier; import btools.util.CompactLongMap; import btools.util.CompactLongSet; import btools.util.Crc32; @@ -43,6 +44,8 @@ public class WayLinker extends MapCreatorBase private BExpressionContext expctxWay; + private ByteArrayUnifier abUnifier; + private int minLon; private int minLat; @@ -80,6 +83,8 @@ public class WayLinker extends MapCreatorBase creationTimeStamp = System.currentTimeMillis(); + abUnifier = new ByteArrayUnifier( 16384 ); + // then process all segments new WayIterator( this, true ).processDir( wayTilesIn, ".wt5" ); } @@ -140,7 +145,7 @@ public class WayLinker extends MapCreatorBase @Override public void nextWay( WayData way ) throws Exception { - byte[] description = way.description; + byte[] description = abUnifier.unify( way.description ); // filter according to profile expctxWay.evaluate( false, description, null ); diff --git a/brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java b/brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java index 2409f03..2e4bc3a 100644 --- a/brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java +++ b/brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java @@ -43,7 +43,7 @@ public class MapcreatorTest // run WayCutter File waytiles = new File( tmpdir, "waytiles" ); waytiles.mkdir(); - new WayCutter().process( ftiles, wayFile2, waytiles, relFile ); + new WayCutter().process( ftiles, wayFile2, waytiles ); // run WayCutter5 File waytiles55 = new File( tmpdir, "waytiles55" ); diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/MicroCache.java b/brouter-mapaccess/src/main/java/btools/mapaccess/MicroCache.java index e29b3ef..c53d40e 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/MicroCache.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/MicroCache.java @@ -22,7 +22,7 @@ final class MicroCache // the object parsing position and length private byte[] ab; private int aboffset; - private int ablength; + private int aboffsetEnd; // cache control: a virgin cache can be // put to ghost state for later recovery @@ -150,7 +150,8 @@ final class MicroCache if ( ( fapos[n] & 0x80000000 ) == 0 ) { aboffset = fapos[n]; - ablength = ( n+1 < size ? fapos[n+1] & 0x7fffffff : ab.length ) - aboffset; + int ablength = ( n+1 < size ? fapos[n+1] & 0x7fffffff : ab.length ) - aboffset; + aboffsetEnd = aboffset + ablength; fapos[n] |= 0x80000000; // mark deleted delbytes+= ablength; delcount++; @@ -172,7 +173,7 @@ final class MicroCache long id = node.getIdFromPos(); if ( getAndClear( id ) ) { - node.parseNodeBody( this, ablength, nodesMap, dc ); + node.parseNodeBody( this, nodesMap, dc ); } if ( doCollect && delcount > size / 2 ) // garbage collection @@ -206,7 +207,7 @@ final class MicroCache int pos = fapos[i]; if ( ( pos & 0x80000000 ) == 0 ) { - ablength = ( i+1 < size ? fapos[i+1] & 0x7fffffff : ab.length ) - pos; + int ablength = ( i+1 < size ? fapos[i+1] & 0x7fffffff : ab.length ) - pos; System.arraycopy( ab, pos, nab, nab_off, ablength ); nfaid[idx] = faid[i]; nfapos[idx] = nab_off; @@ -297,4 +298,9 @@ final class MicroCache System.arraycopy( ab, aboffset, ta, 0, ta.length ); aboffset += ta.length; } + + public boolean hasMoreData() + { + return aboffset < aboffsetEnd; + } } diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java b/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java index 1ecfcb3..6b9ca85 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java @@ -5,8 +5,11 @@ */ package btools.mapaccess; -import java.util.*; -import java.io.*; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; public final class NodesCache { diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java index 99093dd..3f75d2f 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java @@ -5,12 +5,14 @@ */ package btools.mapaccess; +import btools.util.ByteArrayUnifier; + public class OsmNode implements OsmPos { public static final int EXTERNAL_BITMASK = 0x80; - public static final int FIRSTFORWAY_BITMASK = 0x40; + public static final int VARIABLEDESC_BITMASK = 0x40; public static final int TRANSFERNODE_BITMASK = 0x20; public static final int WRITEDESC_BITMASK = 0x10; public static final int SKIPDETAILS_BITMASK = 0x08; @@ -107,17 +109,18 @@ public class OsmNode implements OsmPos } - public void parseNodeBody( MicroCache is, int bodySize, OsmNodesMap hollowNodes, DistanceChecker dc ) + public void parseNodeBody( MicroCache is, OsmNodesMap hollowNodes, DistanceChecker dc ) { - selev = is.readShort(); - bodySize -= 2; + ByteArrayUnifier abUnifier = hollowNodes.getByteArrayUnifier(); + + selev = is.readShort(); OsmLink lastlink = null; int lonIdx = ilon/62500; int latIdx = ilat/62500; - while( bodySize > 0 ) + while( is.hasMoreData() ) { OsmLink link = new OsmLink(); OsmTransferNode firstTransferNode = null; @@ -128,32 +131,33 @@ public class OsmNode implements OsmPos for(;;) { int bitField = is.readByte(); - bodySize -= 1; if ( (bitField & EXTERNAL_BITMASK) != 0 ) { // full position for external target - bodySize -= 8; linklon = is.readInt(); linklat = is.readInt(); } else { // reduced position for internal target - bodySize -= 4; linklon = is.readShort(); linklat = is.readShort(); linklon += lonIdx*62500 + 31250; linklat += latIdx*62500 + 31250; } + // read variable length or old 8 byte fixed, and ensure that 8 bytes is only fixed + boolean readFix8 = (bitField & VARIABLEDESC_BITMASK ) == 0; // old, fix length format if ( (bitField & WRITEDESC_BITMASK ) != 0 ) { - int dlen = is.readByte(); description = new byte[dlen]; is.readFully( description ); - bodySize -= 1 + dlen; + byte[] ab = new byte[readFix8 ? 8 : is.readByte()]; + is.readFully( ab ); + description = abUnifier.unify( ab ); } if ( (bitField & NODEDESC_BITMASK ) != 0 ) { - int dlen = is.readByte(); nodeDescription = new byte[dlen]; is.readFully( nodeDescription ); - bodySize -= 1 + dlen; + byte[] ab = new byte[readFix8 ? 8 : is.readByte()]; + is.readFully( ab ); + nodeDescription = abUnifier.unify( ab ); } if ( (bitField & SKIPDETAILS_BITMASK ) != 0 ) { @@ -169,7 +173,6 @@ public class OsmNode implements OsmPos trans.ilon = linklon; trans.ilat = linklat; trans.descriptionBitmap = description; - bodySize -= 2; trans.selev = is.readShort(); if ( lastTransferNode == null ) { diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodesMap.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodesMap.java index d9db767..3d9dfb3 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodesMap.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodesMap.java @@ -7,10 +7,19 @@ package btools.mapaccess; import java.util.*; +import btools.util.ByteArrayUnifier; + public final class OsmNodesMap { private HashMap hmap = new HashMap(); + + private ByteArrayUnifier abUnifier = new ByteArrayUnifier( 16384 ); + public ByteArrayUnifier getByteArrayUnifier() + { + return abUnifier; + } + private NodesList completedNodes = null; /** diff --git a/brouter-util/src/main/java/btools/util/ByteArrayUnifier.java b/brouter-util/src/main/java/btools/util/ByteArrayUnifier.java new file mode 100644 index 0000000..24e09f8 --- /dev/null +++ b/brouter-util/src/main/java/btools/util/ByteArrayUnifier.java @@ -0,0 +1,40 @@ +package btools.util; + +public final class ByteArrayUnifier +{ + private byte[][] byteArrayCache; + private int size; + + public ByteArrayUnifier( int size ) + { + this.size = size; + byteArrayCache = new byte[size][]; + } + + /** + * Unify a byte array in order to reuse instances when possible. + * The byte arrays are assumed to be treated as immutable, + * allowing the reuse + * @param the byte array to unify + * @return the cached instance or the input instanced if not cached + */ + public byte[] unify( byte[] ab ) + { + int n = ab.length; + int crc = Crc32.crc( ab, 0, n ); + int idx = (crc & 0xfffffff) % size; + byte[] abc = byteArrayCache[idx]; + if ( abc != null && abc.length == n ) + { + int i = 0; + while( i < n ) + { + if ( ab[i] != abc[i] ) break; + i++; + } + if ( i == n ) return abc; + } + byteArrayCache[idx] = ab; + return ab; + } +} diff --git a/misc/profiles2/lookups.dat b/misc/profiles2/lookups.dat index e0dcb02..626dc8d 100644 --- a/misc/profiles2/lookups.dat +++ b/misc/profiles2/lookups.dat @@ -1,4 +1,5 @@ ----lookupversion:2 +---lookupversion:10 +---readvarlength ---context:way diff --git a/misc/scripts/mapcreation/process_pbf_planet.sh b/misc/scripts/mapcreation/process_pbf_planet.sh index 56f762f..ec980cc 100644 --- a/misc/scripts/mapcreation/process_pbf_planet.sh +++ b/misc/scripts/mapcreation/process_pbf_planet.sh @@ -14,13 +14,15 @@ rm -rf /var/www/brouter/segments2_lastrun mkdir tmp cd tmp mkdir nodetiles -/java/bin/java -Xmx256m -Xms256m -Xmn32m -cp ../pbfparser.jar:../brouter.jar btools.mapcreator.OsmCutter ../lookups.dat nodetiles ways.dat cycleways.dat ../planet-latest.osm.pbf +/java/bin/java -Xmx256m -Xms256m -Xmn32m -cp ../pbfparser.jar:../brouter.jar btools.mapcreator.OsmCutter ../lookups.dat nodetiles ways.dat relations.dat ../planet-latest.osm.pbf + +/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.RelationMerger ways.dat ways2.dat relations.dat ../lookups.dat ../trekking.brf ../softaccess.brf mkdir ftiles /java/bin/java -Xmx512M -Xms512M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.NodeFilter nodetiles ways.dat ftiles mkdir waytiles -/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter ftiles ways.dat waytiles cycleways.dat ../lookups.dat ../trekking.brf ../softaccess.brf +/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter ftiles ways.dat waytiles mkdir waytiles55 /java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter5 ftiles waytiles waytiles55 bordernids.dat