variable length tag descriptions (downwards compatibility)
This commit is contained in:
parent
c16c242a65
commit
afa498637a
14 changed files with 195 additions and 58 deletions
|
@ -24,6 +24,7 @@ public final class BExpressionContext
|
||||||
{
|
{
|
||||||
private static final String CONTEXT_TAG = "---context:";
|
private static final String CONTEXT_TAG = "---context:";
|
||||||
private static final String VERSION_TAG = "---lookupversion:";
|
private static final String VERSION_TAG = "---lookupversion:";
|
||||||
|
private static final String VARLENGTH_TAG = "---readvarlength";
|
||||||
|
|
||||||
private String context;
|
private String context;
|
||||||
private boolean _inOurContext = false;
|
private boolean _inOurContext = false;
|
||||||
|
@ -77,6 +78,7 @@ public final class BExpressionContext
|
||||||
private int linenr;
|
private int linenr;
|
||||||
|
|
||||||
public short lookupVersion = -1;
|
public short lookupVersion = -1;
|
||||||
|
public boolean readVarLength = false;
|
||||||
|
|
||||||
public BExpressionContext( String context )
|
public BExpressionContext( String context )
|
||||||
{
|
{
|
||||||
|
@ -111,6 +113,8 @@ public final class BExpressionContext
|
||||||
|
|
||||||
public byte[] encode( int[] ld )
|
public byte[] encode( int[] ld )
|
||||||
{
|
{
|
||||||
|
if ( !readVarLength ) return encodeFix( ld );
|
||||||
|
|
||||||
// start with first bit hardwired ("reversedirection")
|
// start with first bit hardwired ("reversedirection")
|
||||||
BitCoderContext ctx = new BitCoderContext( abBuf );
|
BitCoderContext ctx = new BitCoderContext( abBuf );
|
||||||
ctx.encodeBit( ld[0] != 0 );
|
ctx.encodeBit( ld[0] != 0 );
|
||||||
|
@ -126,11 +130,10 @@ public final class BExpressionContext
|
||||||
skippedTags++;
|
skippedTags++;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
int n = lookupValues.get(inum).length - 1;
|
|
||||||
if ( n == 2 ) { n = 1; d = d == 2 ? 1 : 0; } // 1-bit encoding for booleans
|
|
||||||
ctx.encodeDistance( skippedTags );
|
ctx.encodeDistance( skippedTags );
|
||||||
skippedTags = 0;
|
skippedTags = 0;
|
||||||
ctx.encode( n, d );
|
int n = lookupValues.get(inum).length - 2;
|
||||||
|
if ( n > 1 ) ctx.encode( n, d-1 ); // booleans are encoded just by presence...
|
||||||
}
|
}
|
||||||
ctx.encodeDistance( skippedTags );
|
ctx.encodeDistance( skippedTags );
|
||||||
int len = ctx.getEncodedLength();
|
int len = ctx.getEncodedLength();
|
||||||
|
@ -139,6 +142,10 @@ public final class BExpressionContext
|
||||||
return ab;
|
return ab;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private byte[] encodeFix( int[] ld )
|
||||||
|
{
|
||||||
|
throw new IllegalArgumentException( "encoding fixed-length not supporte" );
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -146,6 +153,8 @@ public final class BExpressionContext
|
||||||
*/
|
*/
|
||||||
public void decode( int[] ld, byte[] ab )
|
public void decode( int[] ld, byte[] ab )
|
||||||
{
|
{
|
||||||
|
if ( ab.length == 8 ) { decodeFix( ld, ab ); return; }
|
||||||
|
|
||||||
BitCoderContext ctx = new BitCoderContext(ab);
|
BitCoderContext ctx = new BitCoderContext(ab);
|
||||||
|
|
||||||
// start with first bit hardwired ("reversedirection")
|
// start with first bit hardwired ("reversedirection")
|
||||||
|
@ -157,16 +166,48 @@ public final class BExpressionContext
|
||||||
int skip = ctx.decodeDistance();
|
int skip = ctx.decodeDistance();
|
||||||
while ( skip-- > 0 ) ld[inum++] = 0;
|
while ( skip-- > 0 ) ld[inum++] = 0;
|
||||||
if ( inum >= lookupValues.size() ) break;
|
if ( inum >= lookupValues.size() ) break;
|
||||||
|
int n = lookupValues.get(inum).length - 2;
|
||||||
|
if ( n > 1 )
|
||||||
|
{
|
||||||
|
ld[inum] = ctx.decode( n ) + 1;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
ld[inum] = 2; // boolean
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* decode old, 64-bit-fixed-length format
|
||||||
|
*/
|
||||||
|
public void decodeFix( int[] ld, byte[] ab )
|
||||||
|
{
|
||||||
|
int idx = 0;
|
||||||
|
long i7 = ab[idx++]& 0xff;
|
||||||
|
long i6 = ab[idx++]& 0xff;
|
||||||
|
long i5 = ab[idx++]& 0xff;
|
||||||
|
long i4 = ab[idx++]& 0xff;
|
||||||
|
long i3 = ab[idx++]& 0xff;
|
||||||
|
long i2 = ab[idx++]& 0xff;
|
||||||
|
long i1 = ab[idx++]& 0xff;
|
||||||
|
long i0 = ab[idx++]& 0xff;
|
||||||
|
long w = (i7 << 56) + (i6 << 48) + (i5 << 40) + (i4 << 32) + (i3 << 24) + (i2 << 16) + (i1 << 8) + i0;
|
||||||
|
|
||||||
|
for( int inum = lookupValues.size()-1; inum >= 0; inum-- ) // loop over lookup names
|
||||||
|
{
|
||||||
int nv = lookupValues.get(inum).length;
|
int nv = lookupValues.get(inum).length;
|
||||||
int n = nv == 3 ? 1 : nv-1; // 1-bit encoding for booleans
|
int n = nv == 3 ? 1 : nv-1; // 1-bit encoding for booleans
|
||||||
|
int m = 0;
|
||||||
int d = ctx.decode( n );
|
long ww = w;
|
||||||
|
while( n != 0 ) { n >>= 1; ww >>= 1; m = m<<1 | 1; }
|
||||||
|
int d = (int)(w & m);
|
||||||
if ( nv == 3 && d == 1 ) d = 2; // 1-bit encoding for booleans
|
if ( nv == 3 && d == 1 ) d = 2; // 1-bit encoding for booleans
|
||||||
ld[inum] = d;
|
ld[inum] = d;
|
||||||
|
w = ww;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public String getCsvDescription( byte[] ab )
|
public String getCsvDescription( byte[] ab )
|
||||||
{
|
{
|
||||||
StringBuilder sb = new StringBuilder( 200 );
|
StringBuilder sb = new StringBuilder( 200 );
|
||||||
|
@ -213,8 +254,7 @@ public final class BExpressionContext
|
||||||
|
|
||||||
int parsedLines = 0;
|
int parsedLines = 0;
|
||||||
boolean ourContext = false;
|
boolean ourContext = false;
|
||||||
if ( "way".equals( context ) ) addLookupValue( "reversedirection", "yes", null );
|
boolean fixTagsWritten = false;
|
||||||
else if ( "node".equals( context ) ) addLookupValue( "nodeaccessgranted", "yes", null );
|
|
||||||
for(;;)
|
for(;;)
|
||||||
{
|
{
|
||||||
String line = br.readLine();
|
String line = br.readLine();
|
||||||
|
@ -231,6 +271,11 @@ public final class BExpressionContext
|
||||||
lookupVersion = Short.parseShort( line.substring( VERSION_TAG.length() ) );
|
lookupVersion = Short.parseShort( line.substring( VERSION_TAG.length() ) );
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if ( line.startsWith( VARLENGTH_TAG ) )
|
||||||
|
{
|
||||||
|
readVarLength = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if ( !ourContext ) continue;
|
if ( !ourContext ) continue;
|
||||||
parsedLines++;
|
parsedLines++;
|
||||||
StringTokenizer tk = new StringTokenizer( line, " " );
|
StringTokenizer tk = new StringTokenizer( line, " " );
|
||||||
|
@ -238,8 +283,18 @@ public final class BExpressionContext
|
||||||
String value = tk.nextToken();
|
String value = tk.nextToken();
|
||||||
int idx = name.indexOf( ';' );
|
int idx = name.indexOf( ';' );
|
||||||
if ( idx >= 0 ) name = name.substring( 0, idx );
|
if ( idx >= 0 ) name = name.substring( 0, idx );
|
||||||
|
|
||||||
|
if ( readVarLength )
|
||||||
|
{
|
||||||
|
if ( !fixTagsWritten )
|
||||||
|
{
|
||||||
|
fixTagsWritten = true;
|
||||||
|
if ( "way".equals( context ) ) addLookupValue( "reversedirection", "yes", null );
|
||||||
|
else if ( "node".equals( context ) ) addLookupValue( "nodeaccessgranted", "yes", null );
|
||||||
|
}
|
||||||
if ( "reversedirection".equals( name ) ) continue; // this is hardcoded
|
if ( "reversedirection".equals( name ) ) continue; // this is hardcoded
|
||||||
if ( "nodeaccessgranted".equals( name ) ) continue; // this is hardcoded
|
if ( "nodeaccessgranted".equals( name ) ) continue; // this is hardcoded
|
||||||
|
}
|
||||||
BExpressionLookupValue newValue = addLookupValue( name, value, null );
|
BExpressionLookupValue newValue = addLookupValue( name, value, null );
|
||||||
|
|
||||||
// add aliases
|
// add aliases
|
||||||
|
@ -274,21 +329,21 @@ public final class BExpressionContext
|
||||||
|
|
||||||
public void evaluate( boolean inverseDirection, byte[] ab, BExpressionReceiver receiver )
|
public void evaluate( boolean inverseDirection, byte[] ab, BExpressionReceiver receiver )
|
||||||
{
|
{
|
||||||
|
int inverseBitByteIndex = readVarLength ? 0 : 7;
|
||||||
|
|
||||||
_receiver = receiver;
|
_receiver = receiver;
|
||||||
|
|
||||||
int abLen = ab.length;
|
int abLen = ab.length;
|
||||||
boolean equalsCurrent = currentHashBucket >= 0 && abLen == currentByteArray.length;
|
boolean equalsCurrent = currentHashBucket >= 0 && abLen == currentByteArray.length;
|
||||||
if ( equalsCurrent )
|
if ( equalsCurrent )
|
||||||
{
|
{
|
||||||
if ( (inverseDirection ? ab[0] ^ 1 : ab[0] ) == currentByteArray[0] )
|
for( int i=0; i<abLen; i++ )
|
||||||
{
|
{
|
||||||
for( int i=1; i<abLen; i++ )
|
byte b = ab[i];
|
||||||
{
|
if ( i == inverseBitByteIndex && inverseDirection ) b ^= 1;
|
||||||
if ( ab[i] != currentByteArray[i] ) { equalsCurrent = false; break; }
|
if ( b != currentByteArray[i] ) { equalsCurrent = false; break; }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else equalsCurrent = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
if ( equalsCurrent )
|
if ( equalsCurrent )
|
||||||
|
@ -303,7 +358,7 @@ public final class BExpressionContext
|
||||||
currentHashBucket = (crc & 0xfffffff) % hashSize;
|
currentHashBucket = (crc & 0xfffffff) % hashSize;
|
||||||
currentByteArray = new byte[abLen];
|
currentByteArray = new byte[abLen];
|
||||||
System.arraycopy( ab, 0, currentByteArray, 0 , abLen );
|
System.arraycopy( ab, 0, currentByteArray, 0 , abLen );
|
||||||
if ( inverseDirection ) currentByteArray[0] ^= 1;
|
if ( inverseDirection ) currentByteArray[inverseBitByteIndex] ^= 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean hashBucketEquals = false;
|
boolean hashBucketEquals = false;
|
||||||
|
|
|
@ -46,6 +46,9 @@ public class OsmCutter extends MapCreatorBase
|
||||||
private BExpressionContext _expctxWay;
|
private BExpressionContext _expctxWay;
|
||||||
private BExpressionContext _expctxNode;
|
private BExpressionContext _expctxNode;
|
||||||
|
|
||||||
|
private BExpressionContext _expctxWayStat;
|
||||||
|
private BExpressionContext _expctxNodeStat;
|
||||||
|
|
||||||
public void process (File lookupFile, File outTileDir, File wayFile, File relFile, File mapFile) throws Exception
|
public void process (File lookupFile, File outTileDir, File wayFile, File relFile, File mapFile) throws Exception
|
||||||
{
|
{
|
||||||
if ( !lookupFile.exists() )
|
if ( !lookupFile.exists() )
|
||||||
|
@ -55,9 +58,11 @@ public class OsmCutter extends MapCreatorBase
|
||||||
|
|
||||||
_expctxWay = new BExpressionContext("way");
|
_expctxWay = new BExpressionContext("way");
|
||||||
_expctxWay.readMetaData( lookupFile );
|
_expctxWay.readMetaData( lookupFile );
|
||||||
|
_expctxWayStat = new BExpressionContext("way");
|
||||||
|
|
||||||
_expctxNode = new BExpressionContext("node");
|
_expctxNode = new BExpressionContext("node");
|
||||||
_expctxNode.readMetaData( lookupFile );
|
_expctxNode.readMetaData( lookupFile );
|
||||||
|
_expctxNodeStat = new BExpressionContext("node");
|
||||||
|
|
||||||
this.outTileDir = outTileDir;
|
this.outTileDir = outTileDir;
|
||||||
if ( !outTileDir.isDirectory() ) throw new RuntimeException( "out tile directory " + outTileDir + " does not exist" );
|
if ( !outTileDir.isDirectory() ) throw new RuntimeException( "out tile directory " + outTileDir + " does not exist" );
|
||||||
|
@ -77,11 +82,11 @@ public class OsmCutter extends MapCreatorBase
|
||||||
wayDos.close();
|
wayDos.close();
|
||||||
cyclewayDos.close();
|
cyclewayDos.close();
|
||||||
|
|
||||||
/* System.out.println( "-------- way-statistics -------- " );
|
System.out.println( "-------- way-statistics -------- " );
|
||||||
_expctxWay.dumpStatistics();
|
_expctxWayStat.dumpStatistics();
|
||||||
System.out.println( "-------- node-statistics -------- " );
|
System.out.println( "-------- node-statistics -------- " );
|
||||||
_expctxNode.dumpStatistics();
|
_expctxNodeStat.dumpStatistics();
|
||||||
*/
|
|
||||||
System.out.println( statsLine() );
|
System.out.println( statsLine() );
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -109,6 +114,7 @@ public class OsmCutter extends MapCreatorBase
|
||||||
{
|
{
|
||||||
String value = n.getTag( key );
|
String value = n.getTag( key );
|
||||||
_expctxNode.addLookupValue( key, value, lookupData );
|
_expctxNode.addLookupValue( key, value, lookupData );
|
||||||
|
_expctxNodeStat.addLookupValue( key, value, null );
|
||||||
}
|
}
|
||||||
n.description = _expctxNode.encode(lookupData);
|
n.description = _expctxNode.encode(lookupData);
|
||||||
}
|
}
|
||||||
|
@ -155,6 +161,7 @@ public class OsmCutter extends MapCreatorBase
|
||||||
{
|
{
|
||||||
String value = w.getTag( key );
|
String value = w.getTag( key );
|
||||||
_expctxWay.addLookupValue( key, value, lookupData );
|
_expctxWay.addLookupValue( key, value, lookupData );
|
||||||
|
_expctxWayStat.addLookupValue( key, value, null );
|
||||||
}
|
}
|
||||||
w.description = _expctxWay.encode(lookupData);
|
w.description = _expctxWay.encode(lookupData);
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,7 +12,7 @@ import java.io.IOException;
|
||||||
public class OsmNodeP implements Comparable<OsmNodeP>
|
public class OsmNodeP implements Comparable<OsmNodeP>
|
||||||
{
|
{
|
||||||
public static final int EXTERNAL_BITMASK = 0x80;
|
public static final int EXTERNAL_BITMASK = 0x80;
|
||||||
public static final int FIRSTFORWAY_BITMASK = 0x40;
|
public static final int VARIABLEDESC_BITMASK = 0x40;
|
||||||
public static final int TRANSFERNODE_BITMASK = 0x20;
|
public static final int TRANSFERNODE_BITMASK = 0x20;
|
||||||
public static final int WRITEDESC_BITMASK = 0x10;
|
public static final int WRITEDESC_BITMASK = 0x10;
|
||||||
public static final int SKIPDETAILS_BITMASK = 0x08;
|
public static final int SKIPDETAILS_BITMASK = 0x08;
|
||||||
|
@ -144,14 +144,14 @@ public class OsmNodeP implements Comparable<OsmNodeP>
|
||||||
if ( targetLonIdx == lonIdx && targetLatIdx == latIdx )
|
if ( targetLonIdx == lonIdx && targetLatIdx == latIdx )
|
||||||
{
|
{
|
||||||
// reduced position for internal target
|
// reduced position for internal target
|
||||||
os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit );
|
os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit | VARIABLEDESC_BITMASK );
|
||||||
os2.writeShort( (short)(target.ilon - lonIdx*62500 - 31250) );
|
os2.writeShort( (short)(target.ilon - lonIdx*62500 - 31250) );
|
||||||
os2.writeShort( (short)(target.ilat - latIdx*62500 - 31250) );
|
os2.writeShort( (short)(target.ilat - latIdx*62500 - 31250) );
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// full position for external target
|
// full position for external target
|
||||||
os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit | EXTERNAL_BITMASK );
|
os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit | EXTERNAL_BITMASK | VARIABLEDESC_BITMASK );
|
||||||
os2.writeInt( target.ilon );
|
os2.writeInt( target.ilon );
|
||||||
os2.writeInt( target.ilat );
|
os2.writeInt( target.ilat );
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,6 +23,7 @@ public class RelationMerger extends MapCreatorBase
|
||||||
private CompactLongSet routesetall;
|
private CompactLongSet routesetall;
|
||||||
private BExpressionContext expctxReport;
|
private BExpressionContext expctxReport;
|
||||||
private BExpressionContext expctxCheck;
|
private BExpressionContext expctxCheck;
|
||||||
|
private BExpressionContext expctxStat;
|
||||||
|
|
||||||
private DataOutputStream wayOutStream;
|
private DataOutputStream wayOutStream;
|
||||||
|
|
||||||
|
@ -47,6 +48,7 @@ public class RelationMerger extends MapCreatorBase
|
||||||
expctxCheck = new BExpressionContext("way");
|
expctxCheck = new BExpressionContext("way");
|
||||||
expctxCheck.readMetaData( lookupFile );
|
expctxCheck.readMetaData( lookupFile );
|
||||||
expctxCheck.parseFile( checkProfile, "global" );
|
expctxCheck.parseFile( checkProfile, "global" );
|
||||||
|
expctxStat = new BExpressionContext("way");
|
||||||
|
|
||||||
// *** read the relation file into sets for each processed tag
|
// *** read the relation file into sets for each processed tag
|
||||||
routesets = new HashMap<String,CompactLongSet>();
|
routesets = new HashMap<String,CompactLongSet>();
|
||||||
|
@ -76,6 +78,7 @@ public class RelationMerger extends MapCreatorBase
|
||||||
{
|
{
|
||||||
long wid = readId( dis );
|
long wid = readId( dis );
|
||||||
if ( wid == -1 ) break;
|
if ( wid == -1 ) break;
|
||||||
|
expctxStat.addLookupValue( tagname, "yes", null );
|
||||||
if ( routeset != null && !routeset.contains( wid ) )
|
if ( routeset != null && !routeset.contains( wid ) )
|
||||||
{
|
{
|
||||||
routeset.add( wid );
|
routeset.add( wid );
|
||||||
|
@ -92,14 +95,17 @@ public class RelationMerger extends MapCreatorBase
|
||||||
{
|
{
|
||||||
CompactLongSet routeset = new FrozenLongSet( routesets.get( tagname ) );
|
CompactLongSet routeset = new FrozenLongSet( routesets.get( tagname ) );
|
||||||
routesets.put( tagname, routeset );
|
routesets.put( tagname, routeset );
|
||||||
System.out.println( "marked " + routeset.size() + " ways for tag: " + tagname );
|
System.out.println( "marked " + routeset.size() + " routes for tag: " + tagname );
|
||||||
}
|
}
|
||||||
|
|
||||||
// *** finally process the way-file
|
// *** finally process the way-file
|
||||||
wayOutStream = createOutStream( wayFileOut );
|
wayOutStream = createOutStream( wayFileOut );
|
||||||
new WayIterator( this, true ).processFile( wayFileIn );
|
new WayIterator( this, true ).processFile( wayFileIn );
|
||||||
wayOutStream.close();
|
wayOutStream.close();
|
||||||
}
|
|
||||||
|
System.out.println( "-------- route-statistics -------- " );
|
||||||
|
expctxStat.dumpStatistics();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void nextWay( WayData data ) throws Exception
|
public void nextWay( WayData data ) throws Exception
|
||||||
|
|
|
@ -20,16 +20,16 @@ public class WayCutter extends MapCreatorBase
|
||||||
public static void main(String[] args) throws Exception
|
public static void main(String[] args) throws Exception
|
||||||
{
|
{
|
||||||
System.out.println("*** WayCutter: Soft-Cut way-data into tiles");
|
System.out.println("*** WayCutter: Soft-Cut way-data into tiles");
|
||||||
if (args.length != 4)
|
if (args.length != 3)
|
||||||
{
|
{
|
||||||
System.out.println("usage: java WayCutter <node-tiles-in> <way-file-in> <way-tiles-out> <relation-file>" );
|
System.out.println("usage: java WayCutter <node-tiles-in> <way-file-in> <way-tiles-out>" );
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
new WayCutter().process( new File( args[0] ), new File( args[1] ), new File( args[2] ), new File( args[3] ) );
|
new WayCutter().process( new File( args[0] ), new File( args[1] ), new File( args[2] ) );
|
||||||
}
|
}
|
||||||
|
|
||||||
public void process( File nodeTilesIn, File wayFileIn, File wayTilesOut, File relationFileIn ) throws Exception
|
public void process( File nodeTilesIn, File wayFileIn, File wayTilesOut ) throws Exception
|
||||||
{
|
{
|
||||||
this.outTileDir = wayTilesOut;
|
this.outTileDir = wayTilesOut;
|
||||||
|
|
||||||
|
|
|
@ -8,6 +8,7 @@ import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import btools.expressions.BExpressionContext;
|
import btools.expressions.BExpressionContext;
|
||||||
|
import btools.util.ByteArrayUnifier;
|
||||||
import btools.util.CompactLongMap;
|
import btools.util.CompactLongMap;
|
||||||
import btools.util.CompactLongSet;
|
import btools.util.CompactLongSet;
|
||||||
import btools.util.Crc32;
|
import btools.util.Crc32;
|
||||||
|
@ -43,6 +44,8 @@ public class WayLinker extends MapCreatorBase
|
||||||
|
|
||||||
private BExpressionContext expctxWay;
|
private BExpressionContext expctxWay;
|
||||||
|
|
||||||
|
private ByteArrayUnifier abUnifier;
|
||||||
|
|
||||||
private int minLon;
|
private int minLon;
|
||||||
private int minLat;
|
private int minLat;
|
||||||
|
|
||||||
|
@ -80,6 +83,8 @@ public class WayLinker extends MapCreatorBase
|
||||||
|
|
||||||
creationTimeStamp = System.currentTimeMillis();
|
creationTimeStamp = System.currentTimeMillis();
|
||||||
|
|
||||||
|
abUnifier = new ByteArrayUnifier( 16384 );
|
||||||
|
|
||||||
// then process all segments
|
// then process all segments
|
||||||
new WayIterator( this, true ).processDir( wayTilesIn, ".wt5" );
|
new WayIterator( this, true ).processDir( wayTilesIn, ".wt5" );
|
||||||
}
|
}
|
||||||
|
@ -140,7 +145,7 @@ public class WayLinker extends MapCreatorBase
|
||||||
@Override
|
@Override
|
||||||
public void nextWay( WayData way ) throws Exception
|
public void nextWay( WayData way ) throws Exception
|
||||||
{
|
{
|
||||||
byte[] description = way.description;
|
byte[] description = abUnifier.unify( way.description );
|
||||||
|
|
||||||
// filter according to profile
|
// filter according to profile
|
||||||
expctxWay.evaluate( false, description, null );
|
expctxWay.evaluate( false, description, null );
|
||||||
|
|
|
@ -43,7 +43,7 @@ public class MapcreatorTest
|
||||||
// run WayCutter
|
// run WayCutter
|
||||||
File waytiles = new File( tmpdir, "waytiles" );
|
File waytiles = new File( tmpdir, "waytiles" );
|
||||||
waytiles.mkdir();
|
waytiles.mkdir();
|
||||||
new WayCutter().process( ftiles, wayFile2, waytiles, relFile );
|
new WayCutter().process( ftiles, wayFile2, waytiles );
|
||||||
|
|
||||||
// run WayCutter5
|
// run WayCutter5
|
||||||
File waytiles55 = new File( tmpdir, "waytiles55" );
|
File waytiles55 = new File( tmpdir, "waytiles55" );
|
||||||
|
|
|
@ -22,7 +22,7 @@ final class MicroCache
|
||||||
// the object parsing position and length
|
// the object parsing position and length
|
||||||
private byte[] ab;
|
private byte[] ab;
|
||||||
private int aboffset;
|
private int aboffset;
|
||||||
private int ablength;
|
private int aboffsetEnd;
|
||||||
|
|
||||||
// cache control: a virgin cache can be
|
// cache control: a virgin cache can be
|
||||||
// put to ghost state for later recovery
|
// put to ghost state for later recovery
|
||||||
|
@ -150,7 +150,8 @@ final class MicroCache
|
||||||
if ( ( fapos[n] & 0x80000000 ) == 0 )
|
if ( ( fapos[n] & 0x80000000 ) == 0 )
|
||||||
{
|
{
|
||||||
aboffset = fapos[n];
|
aboffset = fapos[n];
|
||||||
ablength = ( n+1 < size ? fapos[n+1] & 0x7fffffff : ab.length ) - aboffset;
|
int ablength = ( n+1 < size ? fapos[n+1] & 0x7fffffff : ab.length ) - aboffset;
|
||||||
|
aboffsetEnd = aboffset + ablength;
|
||||||
fapos[n] |= 0x80000000; // mark deleted
|
fapos[n] |= 0x80000000; // mark deleted
|
||||||
delbytes+= ablength;
|
delbytes+= ablength;
|
||||||
delcount++;
|
delcount++;
|
||||||
|
@ -172,7 +173,7 @@ final class MicroCache
|
||||||
long id = node.getIdFromPos();
|
long id = node.getIdFromPos();
|
||||||
if ( getAndClear( id ) )
|
if ( getAndClear( id ) )
|
||||||
{
|
{
|
||||||
node.parseNodeBody( this, ablength, nodesMap, dc );
|
node.parseNodeBody( this, nodesMap, dc );
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( doCollect && delcount > size / 2 ) // garbage collection
|
if ( doCollect && delcount > size / 2 ) // garbage collection
|
||||||
|
@ -206,7 +207,7 @@ final class MicroCache
|
||||||
int pos = fapos[i];
|
int pos = fapos[i];
|
||||||
if ( ( pos & 0x80000000 ) == 0 )
|
if ( ( pos & 0x80000000 ) == 0 )
|
||||||
{
|
{
|
||||||
ablength = ( i+1 < size ? fapos[i+1] & 0x7fffffff : ab.length ) - pos;
|
int ablength = ( i+1 < size ? fapos[i+1] & 0x7fffffff : ab.length ) - pos;
|
||||||
System.arraycopy( ab, pos, nab, nab_off, ablength );
|
System.arraycopy( ab, pos, nab, nab_off, ablength );
|
||||||
nfaid[idx] = faid[i];
|
nfaid[idx] = faid[i];
|
||||||
nfapos[idx] = nab_off;
|
nfapos[idx] = nab_off;
|
||||||
|
@ -297,4 +298,9 @@ final class MicroCache
|
||||||
System.arraycopy( ab, aboffset, ta, 0, ta.length );
|
System.arraycopy( ab, aboffset, ta, 0, ta.length );
|
||||||
aboffset += ta.length;
|
aboffset += ta.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public boolean hasMoreData()
|
||||||
|
{
|
||||||
|
return aboffset < aboffsetEnd;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,8 +5,11 @@
|
||||||
*/
|
*/
|
||||||
package btools.mapaccess;
|
package btools.mapaccess;
|
||||||
|
|
||||||
import java.util.*;
|
import java.io.File;
|
||||||
import java.io.*;
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
public final class NodesCache
|
public final class NodesCache
|
||||||
{
|
{
|
||||||
|
|
|
@ -5,12 +5,14 @@
|
||||||
*/
|
*/
|
||||||
package btools.mapaccess;
|
package btools.mapaccess;
|
||||||
|
|
||||||
|
import btools.util.ByteArrayUnifier;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public class OsmNode implements OsmPos
|
public class OsmNode implements OsmPos
|
||||||
{
|
{
|
||||||
public static final int EXTERNAL_BITMASK = 0x80;
|
public static final int EXTERNAL_BITMASK = 0x80;
|
||||||
public static final int FIRSTFORWAY_BITMASK = 0x40;
|
public static final int VARIABLEDESC_BITMASK = 0x40;
|
||||||
public static final int TRANSFERNODE_BITMASK = 0x20;
|
public static final int TRANSFERNODE_BITMASK = 0x20;
|
||||||
public static final int WRITEDESC_BITMASK = 0x10;
|
public static final int WRITEDESC_BITMASK = 0x10;
|
||||||
public static final int SKIPDETAILS_BITMASK = 0x08;
|
public static final int SKIPDETAILS_BITMASK = 0x08;
|
||||||
|
@ -107,17 +109,18 @@ public class OsmNode implements OsmPos
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public void parseNodeBody( MicroCache is, int bodySize, OsmNodesMap hollowNodes, DistanceChecker dc )
|
public void parseNodeBody( MicroCache is, OsmNodesMap hollowNodes, DistanceChecker dc )
|
||||||
{
|
{
|
||||||
|
ByteArrayUnifier abUnifier = hollowNodes.getByteArrayUnifier();
|
||||||
|
|
||||||
selev = is.readShort();
|
selev = is.readShort();
|
||||||
bodySize -= 2;
|
|
||||||
|
|
||||||
OsmLink lastlink = null;
|
OsmLink lastlink = null;
|
||||||
|
|
||||||
int lonIdx = ilon/62500;
|
int lonIdx = ilon/62500;
|
||||||
int latIdx = ilat/62500;
|
int latIdx = ilat/62500;
|
||||||
|
|
||||||
while( bodySize > 0 )
|
while( is.hasMoreData() )
|
||||||
{
|
{
|
||||||
OsmLink link = new OsmLink();
|
OsmLink link = new OsmLink();
|
||||||
OsmTransferNode firstTransferNode = null;
|
OsmTransferNode firstTransferNode = null;
|
||||||
|
@ -128,32 +131,33 @@ public class OsmNode implements OsmPos
|
||||||
for(;;)
|
for(;;)
|
||||||
{
|
{
|
||||||
int bitField = is.readByte();
|
int bitField = is.readByte();
|
||||||
bodySize -= 1;
|
|
||||||
if ( (bitField & EXTERNAL_BITMASK) != 0 )
|
if ( (bitField & EXTERNAL_BITMASK) != 0 )
|
||||||
{
|
{
|
||||||
// full position for external target
|
// full position for external target
|
||||||
bodySize -= 8;
|
|
||||||
linklon = is.readInt();
|
linklon = is.readInt();
|
||||||
linklat = is.readInt();
|
linklat = is.readInt();
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// reduced position for internal target
|
// reduced position for internal target
|
||||||
bodySize -= 4;
|
|
||||||
linklon = is.readShort();
|
linklon = is.readShort();
|
||||||
linklat = is.readShort();
|
linklat = is.readShort();
|
||||||
linklon += lonIdx*62500 + 31250;
|
linklon += lonIdx*62500 + 31250;
|
||||||
linklat += latIdx*62500 + 31250;
|
linklat += latIdx*62500 + 31250;
|
||||||
}
|
}
|
||||||
|
// read variable length or old 8 byte fixed, and ensure that 8 bytes is only fixed
|
||||||
|
boolean readFix8 = (bitField & VARIABLEDESC_BITMASK ) == 0; // old, fix length format
|
||||||
if ( (bitField & WRITEDESC_BITMASK ) != 0 )
|
if ( (bitField & WRITEDESC_BITMASK ) != 0 )
|
||||||
{
|
{
|
||||||
int dlen = is.readByte(); description = new byte[dlen]; is.readFully( description );
|
byte[] ab = new byte[readFix8 ? 8 : is.readByte()];
|
||||||
bodySize -= 1 + dlen;
|
is.readFully( ab );
|
||||||
|
description = abUnifier.unify( ab );
|
||||||
}
|
}
|
||||||
if ( (bitField & NODEDESC_BITMASK ) != 0 )
|
if ( (bitField & NODEDESC_BITMASK ) != 0 )
|
||||||
{
|
{
|
||||||
int dlen = is.readByte(); nodeDescription = new byte[dlen]; is.readFully( nodeDescription );
|
byte[] ab = new byte[readFix8 ? 8 : is.readByte()];
|
||||||
bodySize -= 1 + dlen;
|
is.readFully( ab );
|
||||||
|
nodeDescription = abUnifier.unify( ab );
|
||||||
}
|
}
|
||||||
if ( (bitField & SKIPDETAILS_BITMASK ) != 0 )
|
if ( (bitField & SKIPDETAILS_BITMASK ) != 0 )
|
||||||
{
|
{
|
||||||
|
@ -169,7 +173,6 @@ public class OsmNode implements OsmPos
|
||||||
trans.ilon = linklon;
|
trans.ilon = linklon;
|
||||||
trans.ilat = linklat;
|
trans.ilat = linklat;
|
||||||
trans.descriptionBitmap = description;
|
trans.descriptionBitmap = description;
|
||||||
bodySize -= 2;
|
|
||||||
trans.selev = is.readShort();
|
trans.selev = is.readShort();
|
||||||
if ( lastTransferNode == null )
|
if ( lastTransferNode == null )
|
||||||
{
|
{
|
||||||
|
|
|
@ -7,10 +7,19 @@ package btools.mapaccess;
|
||||||
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
|
||||||
|
import btools.util.ByteArrayUnifier;
|
||||||
|
|
||||||
public final class OsmNodesMap
|
public final class OsmNodesMap
|
||||||
{
|
{
|
||||||
private HashMap<Long,OsmNode> hmap = new HashMap<Long,OsmNode>();
|
private HashMap<Long,OsmNode> hmap = new HashMap<Long,OsmNode>();
|
||||||
|
|
||||||
|
private ByteArrayUnifier abUnifier = new ByteArrayUnifier( 16384 );
|
||||||
|
|
||||||
|
public ByteArrayUnifier getByteArrayUnifier()
|
||||||
|
{
|
||||||
|
return abUnifier;
|
||||||
|
}
|
||||||
|
|
||||||
private NodesList completedNodes = null;
|
private NodesList completedNodes = null;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
40
brouter-util/src/main/java/btools/util/ByteArrayUnifier.java
Normal file
40
brouter-util/src/main/java/btools/util/ByteArrayUnifier.java
Normal file
|
@ -0,0 +1,40 @@
|
||||||
|
package btools.util;
|
||||||
|
|
||||||
|
public final class ByteArrayUnifier
|
||||||
|
{
|
||||||
|
private byte[][] byteArrayCache;
|
||||||
|
private int size;
|
||||||
|
|
||||||
|
public ByteArrayUnifier( int size )
|
||||||
|
{
|
||||||
|
this.size = size;
|
||||||
|
byteArrayCache = new byte[size][];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unify a byte array in order to reuse instances when possible.
|
||||||
|
* The byte arrays are assumed to be treated as immutable,
|
||||||
|
* allowing the reuse
|
||||||
|
* @param the byte array to unify
|
||||||
|
* @return the cached instance or the input instanced if not cached
|
||||||
|
*/
|
||||||
|
public byte[] unify( byte[] ab )
|
||||||
|
{
|
||||||
|
int n = ab.length;
|
||||||
|
int crc = Crc32.crc( ab, 0, n );
|
||||||
|
int idx = (crc & 0xfffffff) % size;
|
||||||
|
byte[] abc = byteArrayCache[idx];
|
||||||
|
if ( abc != null && abc.length == n )
|
||||||
|
{
|
||||||
|
int i = 0;
|
||||||
|
while( i < n )
|
||||||
|
{
|
||||||
|
if ( ab[i] != abc[i] ) break;
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
if ( i == n ) return abc;
|
||||||
|
}
|
||||||
|
byteArrayCache[idx] = ab;
|
||||||
|
return ab;
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,4 +1,5 @@
|
||||||
---lookupversion:2
|
---lookupversion:10
|
||||||
|
---readvarlength
|
||||||
|
|
||||||
---context:way
|
---context:way
|
||||||
|
|
||||||
|
|
|
@ -14,13 +14,15 @@ rm -rf /var/www/brouter/segments2_lastrun
|
||||||
mkdir tmp
|
mkdir tmp
|
||||||
cd tmp
|
cd tmp
|
||||||
mkdir nodetiles
|
mkdir nodetiles
|
||||||
/java/bin/java -Xmx256m -Xms256m -Xmn32m -cp ../pbfparser.jar:../brouter.jar btools.mapcreator.OsmCutter ../lookups.dat nodetiles ways.dat cycleways.dat ../planet-latest.osm.pbf
|
/java/bin/java -Xmx256m -Xms256m -Xmn32m -cp ../pbfparser.jar:../brouter.jar btools.mapcreator.OsmCutter ../lookups.dat nodetiles ways.dat relations.dat ../planet-latest.osm.pbf
|
||||||
|
|
||||||
|
/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.RelationMerger ways.dat ways2.dat relations.dat ../lookups.dat ../trekking.brf ../softaccess.brf
|
||||||
|
|
||||||
mkdir ftiles
|
mkdir ftiles
|
||||||
/java/bin/java -Xmx512M -Xms512M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.NodeFilter nodetiles ways.dat ftiles
|
/java/bin/java -Xmx512M -Xms512M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.NodeFilter nodetiles ways.dat ftiles
|
||||||
|
|
||||||
mkdir waytiles
|
mkdir waytiles
|
||||||
/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter ftiles ways.dat waytiles cycleways.dat ../lookups.dat ../trekking.brf ../softaccess.brf
|
/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter ftiles ways.dat waytiles
|
||||||
|
|
||||||
mkdir waytiles55
|
mkdir waytiles55
|
||||||
/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter5 ftiles waytiles waytiles55 bordernids.dat
|
/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter5 ftiles waytiles waytiles55 bordernids.dat
|
||||||
|
|
Loading…
Reference in a new issue