Fabio/brouter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

variable length tag descriptions (downwards compatibility)

Browse source
This commit is contained in:
Arndt 2014-05-29 16:47:43 +02:00
parent c16c242a65
commit afa498637a
14 changed files with 195 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java
View file

@ -24,6 +24,7 @@ public final class BExpressionContext
{
private static final String CONTEXT_TAG = "---context:";
private static final String VERSION_TAG = "---lookupversion:";
private static final String VARLENGTH_TAG = "---readvarlength";
private String context;
private boolean _inOurContext = false;
@ -77,6 +78,7 @@ public final class BExpressionContext
private int linenr;
public short lookupVersion = -1;
public boolean readVarLength = false;
public BExpressionContext( String context )
{
@ -111,6 +113,8 @@ public final class BExpressionContext
public byte[] encode( int[] ld )
{
if ( !readVarLength ) return encodeFix( ld );
// start with first bit hardwired ("reversedirection")
BitCoderContext ctx = new BitCoderContext( abBuf );
ctx.encodeBit( ld[0] != 0 );
@ -126,11 +130,10 @@ public final class BExpressionContext
skippedTags++;
continue;
}
int n = lookupValues.get(inum).length - 1;
if ( n == 2 ) { n = 1; d = d == 2 ? 1 : 0; } // 1-bit encoding for booleans
ctx.encodeDistance( skippedTags );
skippedTags = 0;
ctx.encode( n, d );
int n = lookupValues.get(inum).length - 2;
if ( n > 1 ) ctx.encode( n, d-1 ); // booleans are encoded just by presence...
}
ctx.encodeDistance( skippedTags );
int len = ctx.getEncodedLength();
@ -139,6 +142,10 @@ public final class BExpressionContext
return ab;
}
private byte[] encodeFix( int[] ld )
{
throw new IllegalArgumentException( "encoding fixed-length not supporte" );
}
/**
@ -146,6 +153,8 @@ public final class BExpressionContext
*/
public void decode( int[] ld, byte[] ab )
{
if ( ab.length == 8 ) { decodeFix( ld, ab ); return; }
BitCoderContext ctx = new BitCoderContext(ab);
// start with first bit hardwired ("reversedirection")
@ -157,16 +166,48 @@ public final class BExpressionContext
int skip = ctx.decodeDistance();
while ( skip-- > 0 ) ld[inum++] = 0;
if ( inum >= lookupValues.size() ) break;
int n = lookupValues.get(inum).length - 2;
if ( n > 1 )
{
ld[inum] = ctx.decode( n ) + 1;
}
else
{
ld[inum] = 2; // boolean
}
}
}
/**
* decode old, 64-bit-fixed-length format
*/
public void decodeFix( int[] ld, byte[] ab )
{
int idx = 0;
long i7 = ab[idx++]& 0xff;
long i6 = ab[idx++]& 0xff;
long i5 = ab[idx++]& 0xff;
long i4 = ab[idx++]& 0xff;
long i3 = ab[idx++]& 0xff;
long i2 = ab[idx++]& 0xff;
long i1 = ab[idx++]& 0xff;
long i0 = ab[idx++]& 0xff;
long w = (i7 << 56) + (i6 << 48) + (i5 << 40) + (i4 << 32) + (i3 << 24) + (i2 << 16) + (i1 << 8) + i0;
for( int inum = lookupValues.size()-1; inum >= 0; inum-- ) // loop over lookup names
{
int nv = lookupValues.get(inum).length;
int n = nv == 3 ? 1 : nv-1; // 1-bit encoding for booleans
int d = ctx.decode( n );
int m = 0;
long ww = w;
while( n != 0 ) { n >>= 1; ww >>= 1; m = m<<1 | 1; }
int d = (int)(w & m);
if ( nv == 3 && d == 1 ) d = 2; // 1-bit encoding for booleans
ld[inum] = d;
w = ww;
}
}
public String getCsvDescription( byte[] ab )
{
StringBuilder sb = new StringBuilder( 200 );
@ -213,8 +254,7 @@ public final class BExpressionContext
int parsedLines = 0;
boolean ourContext = false;
if ( "way".equals( context ) ) addLookupValue( "reversedirection", "yes", null );
else if ( "node".equals( context ) ) addLookupValue( "nodeaccessgranted", "yes", null );
boolean fixTagsWritten = false;
for(;;)
{
String line = br.readLine();
@ -231,6 +271,11 @@ public final class BExpressionContext
lookupVersion = Short.parseShort( line.substring( VERSION_TAG.length() ) );
continue;
}
if ( line.startsWith( VARLENGTH_TAG ) )
{
readVarLength = true;
continue;
}
if ( !ourContext ) continue;
parsedLines++;
StringTokenizer tk = new StringTokenizer( line, " " );
@ -238,8 +283,18 @@ public final class BExpressionContext
String value = tk.nextToken();
int idx = name.indexOf( ';' );
if ( idx >= 0 ) name = name.substring( 0, idx );
if ( readVarLength )
{
if ( !fixTagsWritten )
{
fixTagsWritten = true;
if ( "way".equals( context ) ) addLookupValue( "reversedirection", "yes", null );
else if ( "node".equals( context ) ) addLookupValue( "nodeaccessgranted", "yes", null );
}
if ( "reversedirection".equals( name ) ) continue; // this is hardcoded
if ( "nodeaccessgranted".equals( name ) ) continue; // this is hardcoded
}
BExpressionLookupValue newValue = addLookupValue( name, value, null );
// add aliases
@ -274,21 +329,21 @@ public final class BExpressionContext
public void evaluate( boolean inverseDirection, byte[] ab, BExpressionReceiver receiver )
{
int inverseBitByteIndex = readVarLength ? 0 : 7;
_receiver = receiver;
int abLen = ab.length;
boolean equalsCurrent = currentHashBucket >= 0 && abLen == currentByteArray.length;
if ( equalsCurrent )
{
if ( (inverseDirection ? ab[0] ^ 1 : ab[0] ) == currentByteArray[0] )
for( int i=0; i<abLen; i++ )
{
for( int i=1; i<abLen; i++ )
{
if ( ab[i] != currentByteArray[i] ) { equalsCurrent = false; break; }
byte b = ab[i];
if ( i == inverseBitByteIndex && inverseDirection ) b ^= 1;
if ( b != currentByteArray[i] ) { equalsCurrent = false; break; }
}
}
else equalsCurrent = false;
}
if ( equalsCurrent )
@ -303,7 +358,7 @@ public final class BExpressionContext
currentHashBucket = (crc & 0xfffffff) % hashSize;
currentByteArray = new byte[abLen];
System.arraycopy( ab, 0, currentByteArray, 0 , abLen );
if ( inverseDirection ) currentByteArray[0] ^= 1;
if ( inverseDirection ) currentByteArray[inverseBitByteIndex] ^= 1;
}
boolean hashBucketEquals = false;

15
brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java
View file

@ -46,6 +46,9 @@ public class OsmCutter extends MapCreatorBase
private BExpressionContext _expctxWay;
private BExpressionContext _expctxNode;
private BExpressionContext _expctxWayStat;
private BExpressionContext _expctxNodeStat;
public void process (File lookupFile, File outTileDir, File wayFile, File relFile, File mapFile) throws Exception
{
if ( !lookupFile.exists() )
@ -55,9 +58,11 @@ public class OsmCutter extends MapCreatorBase
_expctxWay = new BExpressionContext("way");
_expctxWay.readMetaData( lookupFile );
_expctxWayStat = new BExpressionContext("way");
_expctxNode = new BExpressionContext("node");
_expctxNode.readMetaData( lookupFile );
_expctxNodeStat = new BExpressionContext("node");
this.outTileDir = outTileDir;
if ( !outTileDir.isDirectory() ) throw new RuntimeException( "out tile directory " + outTileDir + " does not exist" );
@ -77,11 +82,11 @@ public class OsmCutter extends MapCreatorBase
wayDos.close();
cyclewayDos.close();
/* System.out.println( "-------- way-statistics -------- " );
_expctxWay.dumpStatistics();
System.out.println( "-------- way-statistics -------- " );
_expctxWayStat.dumpStatistics();
System.out.println( "-------- node-statistics -------- " );
_expctxNode.dumpStatistics();
*/
_expctxNodeStat.dumpStatistics();
System.out.println( statsLine() );
}
@ -109,6 +114,7 @@ public class OsmCutter extends MapCreatorBase
{
String value = n.getTag( key );
_expctxNode.addLookupValue( key, value, lookupData );
_expctxNodeStat.addLookupValue( key, value, null );
}
n.description = _expctxNode.encode(lookupData);
}
@ -155,6 +161,7 @@ public class OsmCutter extends MapCreatorBase
{
String value = w.getTag( key );
_expctxWay.addLookupValue( key, value, lookupData );
_expctxWayStat.addLookupValue( key, value, null );
}
w.description = _expctxWay.encode(lookupData);
}

6
brouter-map-creator/src/main/java/btools/mapcreator/OsmNodeP.java
View file

@ -12,7 +12,7 @@ import java.io.IOException;
public class OsmNodeP implements Comparable<OsmNodeP>
{
public static final int EXTERNAL_BITMASK = 0x80;
public static final int FIRSTFORWAY_BITMASK = 0x40;
public static final int VARIABLEDESC_BITMASK = 0x40;
public static final int TRANSFERNODE_BITMASK = 0x20;
public static final int WRITEDESC_BITMASK = 0x10;
public static final int SKIPDETAILS_BITMASK = 0x08;
@ -144,14 +144,14 @@ public class OsmNodeP implements Comparable<OsmNodeP>
if ( targetLonIdx == lonIdx && targetLatIdx == latIdx )
{
// reduced position for internal target
os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit );
os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit | VARIABLEDESC_BITMASK );
os2.writeShort( (short)(target.ilon - lonIdx*62500 - 31250) );
os2.writeShort( (short)(target.ilat - latIdx*62500 - 31250) );
}
else
{
// full position for external target
os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit | EXTERNAL_BITMASK );
os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit | EXTERNAL_BITMASK | VARIABLEDESC_BITMASK );
os2.writeInt( target.ilon );
os2.writeInt( target.ilat );
}

10
brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java
View file

@ -23,6 +23,7 @@ public class RelationMerger extends MapCreatorBase
private CompactLongSet routesetall;
private BExpressionContext expctxReport;
private BExpressionContext expctxCheck;
private BExpressionContext expctxStat;
private DataOutputStream wayOutStream;
@ -47,6 +48,7 @@ public class RelationMerger extends MapCreatorBase
expctxCheck = new BExpressionContext("way");
expctxCheck.readMetaData( lookupFile );
expctxCheck.parseFile( checkProfile, "global" );
expctxStat = new BExpressionContext("way");
// *** read the relation file into sets for each processed tag
routesets = new HashMap<String,CompactLongSet>();
@ -76,6 +78,7 @@ public class RelationMerger extends MapCreatorBase
{
long wid = readId( dis );
if ( wid == -1 ) break;
expctxStat.addLookupValue( tagname, "yes", null );
if ( routeset != null && !routeset.contains( wid ) )
{
routeset.add( wid );
@ -92,14 +95,17 @@ public class RelationMerger extends MapCreatorBase
{
CompactLongSet routeset = new FrozenLongSet( routesets.get( tagname ) );
routesets.put( tagname, routeset );
System.out.println( "marked " + routeset.size() + " ways for tag: " + tagname );
System.out.println( "marked " + routeset.size() + " routes for tag: " + tagname );
}
// *** finally process the way-file
wayOutStream = createOutStream( wayFileOut );
new WayIterator( this, true ).processFile( wayFileIn );
wayOutStream.close();
}
System.out.println( "-------- route-statistics -------- " );
expctxStat.dumpStatistics();
}
@Override
public void nextWay( WayData data ) throws Exception

8
brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java
View file

@ -20,16 +20,16 @@ public class WayCutter extends MapCreatorBase
public static void main(String[] args) throws Exception
{
System.out.println("*** WayCutter: Soft-Cut way-data into tiles");
if (args.length != 4)
if (args.length != 3)
{
System.out.println("usage: java WayCutter <node-tiles-in> <way-file-in> <way-tiles-out> <relation-file>" );
System.out.println("usage: java WayCutter <node-tiles-in> <way-file-in> <way-tiles-out>" );
return;
}
new WayCutter().process( new File( args[0] ), new File( args[1] ), new File( args[2] ), new File( args[3] ) );
new WayCutter().process( new File( args[0] ), new File( args[1] ), new File( args[2] ) );
}
public void process( File nodeTilesIn, File wayFileIn, File wayTilesOut, File relationFileIn ) throws Exception
public void process( File nodeTilesIn, File wayFileIn, File wayTilesOut ) throws Exception
{
this.outTileDir = wayTilesOut;

7
brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java
View file

@ -8,6 +8,7 @@ import java.util.Collections;
import java.util.List;
import btools.expressions.BExpressionContext;
import btools.util.ByteArrayUnifier;
import btools.util.CompactLongMap;
import btools.util.CompactLongSet;
import btools.util.Crc32;
@ -43,6 +44,8 @@ public class WayLinker extends MapCreatorBase
private BExpressionContext expctxWay;
private ByteArrayUnifier abUnifier;
private int minLon;
private int minLat;
@ -80,6 +83,8 @@ public class WayLinker extends MapCreatorBase
creationTimeStamp = System.currentTimeMillis();
abUnifier = new ByteArrayUnifier( 16384 );
// then process all segments
new WayIterator( this, true ).processDir( wayTilesIn, ".wt5" );
}
@ -140,7 +145,7 @@ public class WayLinker extends MapCreatorBase
@Override
public void nextWay( WayData way ) throws Exception
{
byte[] description = way.description;
byte[] description = abUnifier.unify( way.description );
// filter according to profile
expctxWay.evaluate( false, description, null );

2
brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java
View file

@ -43,7 +43,7 @@ public class MapcreatorTest
// run WayCutter
File waytiles = new File( tmpdir, "waytiles" );
waytiles.mkdir();
new WayCutter().process( ftiles, wayFile2, waytiles, relFile );
new WayCutter().process( ftiles, wayFile2, waytiles );
// run WayCutter5
File waytiles55 = new File( tmpdir, "waytiles55" );

14
brouter-mapaccess/src/main/java/btools/mapaccess/MicroCache.java
View file

@ -22,7 +22,7 @@ final class MicroCache
// the object parsing position and length
private byte[] ab;
private int aboffset;
private int ablength;
private int aboffsetEnd;
// cache control: a virgin cache can be
// put to ghost state for later recovery
@ -150,7 +150,8 @@ final class MicroCache
if ( ( fapos[n] & 0x80000000 ) == 0 )
{
aboffset = fapos[n];
ablength = ( n+1 < size ? fapos[n+1] & 0x7fffffff : ab.length ) - aboffset;
int ablength = ( n+1 < size ? fapos[n+1] & 0x7fffffff : ab.length ) - aboffset;
aboffsetEnd = aboffset + ablength;
fapos[n] |= 0x80000000; // mark deleted
delbytes+= ablength;
delcount++;
@ -172,7 +173,7 @@ final class MicroCache
long id = node.getIdFromPos();
if ( getAndClear( id ) )
{
node.parseNodeBody( this, ablength, nodesMap, dc );
node.parseNodeBody( this, nodesMap, dc );
}
if ( doCollect && delcount > size / 2 ) // garbage collection
@ -206,7 +207,7 @@ final class MicroCache
int pos = fapos[i];
if ( ( pos & 0x80000000 ) == 0 )
{
ablength = ( i+1 < size ? fapos[i+1] & 0x7fffffff : ab.length ) - pos;
int ablength = ( i+1 < size ? fapos[i+1] & 0x7fffffff : ab.length ) - pos;
System.arraycopy( ab, pos, nab, nab_off, ablength );
nfaid[idx] = faid[i];
nfapos[idx] = nab_off;
@ -297,4 +298,9 @@ final class MicroCache
System.arraycopy( ab, aboffset, ta, 0, ta.length );
aboffset += ta.length;
}
public boolean hasMoreData()
{
return aboffset < aboffsetEnd;
}
}

7
brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java
View file

@ -5,8 +5,11 @@
*/
package btools.mapaccess;
import java.util.*;
import java.io.*;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
public final class NodesCache
{

27
brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java
View file

@ -5,12 +5,14 @@
*/
package btools.mapaccess;
import btools.util.ByteArrayUnifier;
public class OsmNode implements OsmPos
{
public static final int EXTERNAL_BITMASK = 0x80;
public static final int FIRSTFORWAY_BITMASK = 0x40;
public static final int VARIABLEDESC_BITMASK = 0x40;
public static final int TRANSFERNODE_BITMASK = 0x20;
public static final int WRITEDESC_BITMASK = 0x10;
public static final int SKIPDETAILS_BITMASK = 0x08;
@ -107,17 +109,18 @@ public class OsmNode implements OsmPos
}
public void parseNodeBody( MicroCache is, int bodySize, OsmNodesMap hollowNodes, DistanceChecker dc )
public void parseNodeBody( MicroCache is, OsmNodesMap hollowNodes, DistanceChecker dc )
{
ByteArrayUnifier abUnifier = hollowNodes.getByteArrayUnifier();
selev = is.readShort();
bodySize -= 2;
OsmLink lastlink = null;
int lonIdx = ilon/62500;
int latIdx = ilat/62500;
while( bodySize > 0 )
while( is.hasMoreData() )
{
OsmLink link = new OsmLink();
OsmTransferNode firstTransferNode = null;
@ -128,32 +131,33 @@ public class OsmNode implements OsmPos
for(;;)
{
int bitField = is.readByte();
bodySize -= 1;
if ( (bitField & EXTERNAL_BITMASK) != 0 )
{
// full position for external target
bodySize -= 8;
linklon = is.readInt();
linklat = is.readInt();
}
else
{
// reduced position for internal target
bodySize -= 4;
linklon = is.readShort();
linklat = is.readShort();
linklon += lonIdx*62500 + 31250;
linklat += latIdx*62500 + 31250;
}
// read variable length or old 8 byte fixed, and ensure that 8 bytes is only fixed
boolean readFix8 = (bitField & VARIABLEDESC_BITMASK ) == 0; // old, fix length format
if ( (bitField & WRITEDESC_BITMASK ) != 0 )
{
int dlen = is.readByte(); description = new byte[dlen]; is.readFully( description );
bodySize -= 1 + dlen;
byte[] ab = new byte[readFix8 ? 8 : is.readByte()];
is.readFully( ab );
description = abUnifier.unify( ab );
}
if ( (bitField & NODEDESC_BITMASK ) != 0 )
{
int dlen = is.readByte(); nodeDescription = new byte[dlen]; is.readFully( nodeDescription );
bodySize -= 1 + dlen;
byte[] ab = new byte[readFix8 ? 8 : is.readByte()];
is.readFully( ab );
nodeDescription = abUnifier.unify( ab );
}
if ( (bitField & SKIPDETAILS_BITMASK ) != 0 )
{
@ -169,7 +173,6 @@ public class OsmNode implements OsmPos
trans.ilon = linklon;
trans.ilat = linklat;
trans.descriptionBitmap = description;
bodySize -= 2;
trans.selev = is.readShort();
if ( lastTransferNode == null )
{

9
brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodesMap.java
View file

@ -7,10 +7,19 @@ package btools.mapaccess;
import java.util.*;
import btools.util.ByteArrayUnifier;
public final class OsmNodesMap
{
private HashMap<Long,OsmNode> hmap = new HashMap<Long,OsmNode>();
private ByteArrayUnifier abUnifier = new ByteArrayUnifier( 16384 );
public ByteArrayUnifier getByteArrayUnifier()
{
return abUnifier;
}
private NodesList completedNodes = null;
/**

40
brouter-util/src/main/java/btools/util/ByteArrayUnifier.java Normal file
View file

@ -0,0 +1,40 @@
package btools.util;
public final class ByteArrayUnifier
{
private byte[][] byteArrayCache;
private int size;
public ByteArrayUnifier( int size )
{
this.size = size;
byteArrayCache = new byte[size][];
}
/**
* Unify a byte array in order to reuse instances when possible.
* The byte arrays are assumed to be treated as immutable,
* allowing the reuse
* @param the byte array to unify
* @return the cached instance or the input instanced if not cached
*/
public byte[] unify( byte[] ab )
{
int n = ab.length;
int crc = Crc32.crc( ab, 0, n );
int idx = (crc & 0xfffffff) % size;
byte[] abc = byteArrayCache[idx];
if ( abc != null && abc.length == n )
{
int i = 0;
while( i < n )
{
if ( ab[i] != abc[i] ) break;
i++;
}
if ( i == n ) return abc;
}
byteArrayCache[idx] = ab;
return ab;
}
}

3
misc/profiles2/lookups.dat
View file

@ -1,4 +1,5 @@
---lookupversion:2
---lookupversion:10
---readvarlength
---context:way

6
misc/scripts/mapcreation/process_pbf_planet.sh
View file

@ -14,13 +14,15 @@ rm -rf /var/www/brouter/segments2_lastrun
mkdir tmp
cd tmp
mkdir nodetiles
/java/bin/java -Xmx256m -Xms256m -Xmn32m -cp ../pbfparser.jar:../brouter.jar btools.mapcreator.OsmCutter ../lookups.dat nodetiles ways.dat cycleways.dat ../planet-latest.osm.pbf
/java/bin/java -Xmx256m -Xms256m -Xmn32m -cp ../pbfparser.jar:../brouter.jar btools.mapcreator.OsmCutter ../lookups.dat nodetiles ways.dat relations.dat ../planet-latest.osm.pbf
/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.RelationMerger ways.dat ways2.dat relations.dat ../lookups.dat ../trekking.brf ../softaccess.brf
mkdir ftiles
/java/bin/java -Xmx512M -Xms512M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.NodeFilter nodetiles ways.dat ftiles
mkdir waytiles
/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter ftiles ways.dat waytiles cycleways.dat ../lookups.dat ../trekking.brf ../softaccess.brf
/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter ftiles ways.dat waytiles
mkdir waytiles55
/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter5 ftiles waytiles waytiles55 bordernids.dat