variable length tag descriptions (downwards compatibility)

This commit is contained in:
Arndt 2014-05-29 16:47:43 +02:00
parent c16c242a65
commit afa498637a
14 changed files with 195 additions and 58 deletions

View file

@ -24,6 +24,7 @@ public final class BExpressionContext
{ {
private static final String CONTEXT_TAG = "---context:"; private static final String CONTEXT_TAG = "---context:";
private static final String VERSION_TAG = "---lookupversion:"; private static final String VERSION_TAG = "---lookupversion:";
private static final String VARLENGTH_TAG = "---readvarlength";
private String context; private String context;
private boolean _inOurContext = false; private boolean _inOurContext = false;
@ -77,6 +78,7 @@ public final class BExpressionContext
private int linenr; private int linenr;
public short lookupVersion = -1; public short lookupVersion = -1;
public boolean readVarLength = false;
public BExpressionContext( String context ) public BExpressionContext( String context )
{ {
@ -111,6 +113,8 @@ public final class BExpressionContext
public byte[] encode( int[] ld ) public byte[] encode( int[] ld )
{ {
if ( !readVarLength ) return encodeFix( ld );
// start with first bit hardwired ("reversedirection") // start with first bit hardwired ("reversedirection")
BitCoderContext ctx = new BitCoderContext( abBuf ); BitCoderContext ctx = new BitCoderContext( abBuf );
ctx.encodeBit( ld[0] != 0 ); ctx.encodeBit( ld[0] != 0 );
@ -126,11 +130,10 @@ public final class BExpressionContext
skippedTags++; skippedTags++;
continue; continue;
} }
int n = lookupValues.get(inum).length - 1;
if ( n == 2 ) { n = 1; d = d == 2 ? 1 : 0; } // 1-bit encoding for booleans
ctx.encodeDistance( skippedTags ); ctx.encodeDistance( skippedTags );
skippedTags = 0; skippedTags = 0;
ctx.encode( n, d ); int n = lookupValues.get(inum).length - 2;
if ( n > 1 ) ctx.encode( n, d-1 ); // booleans are encoded just by presence...
} }
ctx.encodeDistance( skippedTags ); ctx.encodeDistance( skippedTags );
int len = ctx.getEncodedLength(); int len = ctx.getEncodedLength();
@ -139,6 +142,10 @@ public final class BExpressionContext
return ab; return ab;
} }
private byte[] encodeFix( int[] ld )
{
throw new IllegalArgumentException( "encoding fixed-length not supporte" );
}
/** /**
@ -146,7 +153,9 @@ public final class BExpressionContext
*/ */
public void decode( int[] ld, byte[] ab ) public void decode( int[] ld, byte[] ab )
{ {
BitCoderContext ctx = new BitCoderContext(ab); if ( ab.length == 8 ) { decodeFix( ld, ab ); return; }
BitCoderContext ctx = new BitCoderContext(ab);
// start with first bit hardwired ("reversedirection") // start with first bit hardwired ("reversedirection")
ld[0] = ctx.decodeBit() ? 2 : 0; ld[0] = ctx.decodeBit() ? 2 : 0;
@ -157,15 +166,47 @@ public final class BExpressionContext
int skip = ctx.decodeDistance(); int skip = ctx.decodeDistance();
while ( skip-- > 0 ) ld[inum++] = 0; while ( skip-- > 0 ) ld[inum++] = 0;
if ( inum >= lookupValues.size() ) break; if ( inum >= lookupValues.size() ) break;
int nv = lookupValues.get(inum).length; int n = lookupValues.get(inum).length - 2;
int n = nv == 3 ? 1 : nv-1; // 1-bit encoding for booleans if ( n > 1 )
{
int d = ctx.decode( n ); ld[inum] = ctx.decode( n ) + 1;
if ( nv == 3 && d == 1 ) d = 2; // 1-bit encoding for booleans }
ld[inum] = d; else
{
ld[inum] = 2; // boolean
}
} }
} }
/**
* decode old, 64-bit-fixed-length format
*/
public void decodeFix( int[] ld, byte[] ab )
{
int idx = 0;
long i7 = ab[idx++]& 0xff;
long i6 = ab[idx++]& 0xff;
long i5 = ab[idx++]& 0xff;
long i4 = ab[idx++]& 0xff;
long i3 = ab[idx++]& 0xff;
long i2 = ab[idx++]& 0xff;
long i1 = ab[idx++]& 0xff;
long i0 = ab[idx++]& 0xff;
long w = (i7 << 56) + (i6 << 48) + (i5 << 40) + (i4 << 32) + (i3 << 24) + (i2 << 16) + (i1 << 8) + i0;
for( int inum = lookupValues.size()-1; inum >= 0; inum-- ) // loop over lookup names
{
int nv = lookupValues.get(inum).length;
int n = nv == 3 ? 1 : nv-1; // 1-bit encoding for booleans
int m = 0;
long ww = w;
while( n != 0 ) { n >>= 1; ww >>= 1; m = m<<1 | 1; }
int d = (int)(w & m);
if ( nv == 3 && d == 1 ) d = 2; // 1-bit encoding for booleans
ld[inum] = d;
w = ww;
}
}
public String getCsvDescription( byte[] ab ) public String getCsvDescription( byte[] ab )
{ {
@ -213,8 +254,7 @@ public final class BExpressionContext
int parsedLines = 0; int parsedLines = 0;
boolean ourContext = false; boolean ourContext = false;
if ( "way".equals( context ) ) addLookupValue( "reversedirection", "yes", null ); boolean fixTagsWritten = false;
else if ( "node".equals( context ) ) addLookupValue( "nodeaccessgranted", "yes", null );
for(;;) for(;;)
{ {
String line = br.readLine(); String line = br.readLine();
@ -231,6 +271,11 @@ public final class BExpressionContext
lookupVersion = Short.parseShort( line.substring( VERSION_TAG.length() ) ); lookupVersion = Short.parseShort( line.substring( VERSION_TAG.length() ) );
continue; continue;
} }
if ( line.startsWith( VARLENGTH_TAG ) )
{
readVarLength = true;
continue;
}
if ( !ourContext ) continue; if ( !ourContext ) continue;
parsedLines++; parsedLines++;
StringTokenizer tk = new StringTokenizer( line, " " ); StringTokenizer tk = new StringTokenizer( line, " " );
@ -238,8 +283,18 @@ public final class BExpressionContext
String value = tk.nextToken(); String value = tk.nextToken();
int idx = name.indexOf( ';' ); int idx = name.indexOf( ';' );
if ( idx >= 0 ) name = name.substring( 0, idx ); if ( idx >= 0 ) name = name.substring( 0, idx );
if ( "reversedirection".equals( name ) ) continue; // this is hardcoded
if ( "nodeaccessgranted".equals( name ) ) continue; // this is hardcoded if ( readVarLength )
{
if ( !fixTagsWritten )
{
fixTagsWritten = true;
if ( "way".equals( context ) ) addLookupValue( "reversedirection", "yes", null );
else if ( "node".equals( context ) ) addLookupValue( "nodeaccessgranted", "yes", null );
}
if ( "reversedirection".equals( name ) ) continue; // this is hardcoded
if ( "nodeaccessgranted".equals( name ) ) continue; // this is hardcoded
}
BExpressionLookupValue newValue = addLookupValue( name, value, null ); BExpressionLookupValue newValue = addLookupValue( name, value, null );
// add aliases // add aliases
@ -274,20 +329,20 @@ public final class BExpressionContext
public void evaluate( boolean inverseDirection, byte[] ab, BExpressionReceiver receiver ) public void evaluate( boolean inverseDirection, byte[] ab, BExpressionReceiver receiver )
{ {
int inverseBitByteIndex = readVarLength ? 0 : 7;
_receiver = receiver; _receiver = receiver;
int abLen = ab.length; int abLen = ab.length;
boolean equalsCurrent = currentHashBucket >= 0 && abLen == currentByteArray.length; boolean equalsCurrent = currentHashBucket >= 0 && abLen == currentByteArray.length;
if ( equalsCurrent ) if ( equalsCurrent )
{ {
if ( (inverseDirection ? ab[0] ^ 1 : ab[0] ) == currentByteArray[0] ) for( int i=0; i<abLen; i++ )
{ {
for( int i=1; i<abLen; i++ ) byte b = ab[i];
{ if ( i == inverseBitByteIndex && inverseDirection ) b ^= 1;
if ( ab[i] != currentByteArray[i] ) { equalsCurrent = false; break; } if ( b != currentByteArray[i] ) { equalsCurrent = false; break; }
}
} }
else equalsCurrent = false;
} }
@ -303,7 +358,7 @@ public final class BExpressionContext
currentHashBucket = (crc & 0xfffffff) % hashSize; currentHashBucket = (crc & 0xfffffff) % hashSize;
currentByteArray = new byte[abLen]; currentByteArray = new byte[abLen];
System.arraycopy( ab, 0, currentByteArray, 0 , abLen ); System.arraycopy( ab, 0, currentByteArray, 0 , abLen );
if ( inverseDirection ) currentByteArray[0] ^= 1; if ( inverseDirection ) currentByteArray[inverseBitByteIndex] ^= 1;
} }
boolean hashBucketEquals = false; boolean hashBucketEquals = false;

View file

@ -46,6 +46,9 @@ public class OsmCutter extends MapCreatorBase
private BExpressionContext _expctxWay; private BExpressionContext _expctxWay;
private BExpressionContext _expctxNode; private BExpressionContext _expctxNode;
private BExpressionContext _expctxWayStat;
private BExpressionContext _expctxNodeStat;
public void process (File lookupFile, File outTileDir, File wayFile, File relFile, File mapFile) throws Exception public void process (File lookupFile, File outTileDir, File wayFile, File relFile, File mapFile) throws Exception
{ {
if ( !lookupFile.exists() ) if ( !lookupFile.exists() )
@ -55,9 +58,11 @@ public class OsmCutter extends MapCreatorBase
_expctxWay = new BExpressionContext("way"); _expctxWay = new BExpressionContext("way");
_expctxWay.readMetaData( lookupFile ); _expctxWay.readMetaData( lookupFile );
_expctxWayStat = new BExpressionContext("way");
_expctxNode = new BExpressionContext("node"); _expctxNode = new BExpressionContext("node");
_expctxNode.readMetaData( lookupFile ); _expctxNode.readMetaData( lookupFile );
_expctxNodeStat = new BExpressionContext("node");
this.outTileDir = outTileDir; this.outTileDir = outTileDir;
if ( !outTileDir.isDirectory() ) throw new RuntimeException( "out tile directory " + outTileDir + " does not exist" ); if ( !outTileDir.isDirectory() ) throw new RuntimeException( "out tile directory " + outTileDir + " does not exist" );
@ -77,11 +82,11 @@ public class OsmCutter extends MapCreatorBase
wayDos.close(); wayDos.close();
cyclewayDos.close(); cyclewayDos.close();
/* System.out.println( "-------- way-statistics -------- " ); System.out.println( "-------- way-statistics -------- " );
_expctxWay.dumpStatistics(); _expctxWayStat.dumpStatistics();
System.out.println( "-------- node-statistics -------- " ); System.out.println( "-------- node-statistics -------- " );
_expctxNode.dumpStatistics(); _expctxNodeStat.dumpStatistics();
*/
System.out.println( statsLine() ); System.out.println( statsLine() );
} }
@ -109,6 +114,7 @@ public class OsmCutter extends MapCreatorBase
{ {
String value = n.getTag( key ); String value = n.getTag( key );
_expctxNode.addLookupValue( key, value, lookupData ); _expctxNode.addLookupValue( key, value, lookupData );
_expctxNodeStat.addLookupValue( key, value, null );
} }
n.description = _expctxNode.encode(lookupData); n.description = _expctxNode.encode(lookupData);
} }
@ -155,6 +161,7 @@ public class OsmCutter extends MapCreatorBase
{ {
String value = w.getTag( key ); String value = w.getTag( key );
_expctxWay.addLookupValue( key, value, lookupData ); _expctxWay.addLookupValue( key, value, lookupData );
_expctxWayStat.addLookupValue( key, value, null );
} }
w.description = _expctxWay.encode(lookupData); w.description = _expctxWay.encode(lookupData);
} }

View file

@ -12,7 +12,7 @@ import java.io.IOException;
public class OsmNodeP implements Comparable<OsmNodeP> public class OsmNodeP implements Comparable<OsmNodeP>
{ {
public static final int EXTERNAL_BITMASK = 0x80; public static final int EXTERNAL_BITMASK = 0x80;
public static final int FIRSTFORWAY_BITMASK = 0x40; public static final int VARIABLEDESC_BITMASK = 0x40;
public static final int TRANSFERNODE_BITMASK = 0x20; public static final int TRANSFERNODE_BITMASK = 0x20;
public static final int WRITEDESC_BITMASK = 0x10; public static final int WRITEDESC_BITMASK = 0x10;
public static final int SKIPDETAILS_BITMASK = 0x08; public static final int SKIPDETAILS_BITMASK = 0x08;
@ -144,14 +144,14 @@ public class OsmNodeP implements Comparable<OsmNodeP>
if ( targetLonIdx == lonIdx && targetLatIdx == latIdx ) if ( targetLonIdx == lonIdx && targetLatIdx == latIdx )
{ {
// reduced position for internal target // reduced position for internal target
os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit ); os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit | VARIABLEDESC_BITMASK );
os2.writeShort( (short)(target.ilon - lonIdx*62500 - 31250) ); os2.writeShort( (short)(target.ilon - lonIdx*62500 - 31250) );
os2.writeShort( (short)(target.ilat - latIdx*62500 - 31250) ); os2.writeShort( (short)(target.ilat - latIdx*62500 - 31250) );
} }
else else
{ {
// full position for external target // full position for external target
os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit | EXTERNAL_BITMASK ); os2.writeByte( tranferbit | writedescbit | nodedescbit | skipDetailBit | EXTERNAL_BITMASK | VARIABLEDESC_BITMASK );
os2.writeInt( target.ilon ); os2.writeInt( target.ilon );
os2.writeInt( target.ilat ); os2.writeInt( target.ilat );
} }

View file

@ -23,6 +23,7 @@ public class RelationMerger extends MapCreatorBase
private CompactLongSet routesetall; private CompactLongSet routesetall;
private BExpressionContext expctxReport; private BExpressionContext expctxReport;
private BExpressionContext expctxCheck; private BExpressionContext expctxCheck;
private BExpressionContext expctxStat;
private DataOutputStream wayOutStream; private DataOutputStream wayOutStream;
@ -47,6 +48,7 @@ public class RelationMerger extends MapCreatorBase
expctxCheck = new BExpressionContext("way"); expctxCheck = new BExpressionContext("way");
expctxCheck.readMetaData( lookupFile ); expctxCheck.readMetaData( lookupFile );
expctxCheck.parseFile( checkProfile, "global" ); expctxCheck.parseFile( checkProfile, "global" );
expctxStat = new BExpressionContext("way");
// *** read the relation file into sets for each processed tag // *** read the relation file into sets for each processed tag
routesets = new HashMap<String,CompactLongSet>(); routesets = new HashMap<String,CompactLongSet>();
@ -76,6 +78,7 @@ public class RelationMerger extends MapCreatorBase
{ {
long wid = readId( dis ); long wid = readId( dis );
if ( wid == -1 ) break; if ( wid == -1 ) break;
expctxStat.addLookupValue( tagname, "yes", null );
if ( routeset != null && !routeset.contains( wid ) ) if ( routeset != null && !routeset.contains( wid ) )
{ {
routeset.add( wid ); routeset.add( wid );
@ -92,14 +95,17 @@ public class RelationMerger extends MapCreatorBase
{ {
CompactLongSet routeset = new FrozenLongSet( routesets.get( tagname ) ); CompactLongSet routeset = new FrozenLongSet( routesets.get( tagname ) );
routesets.put( tagname, routeset ); routesets.put( tagname, routeset );
System.out.println( "marked " + routeset.size() + " ways for tag: " + tagname ); System.out.println( "marked " + routeset.size() + " routes for tag: " + tagname );
} }
// *** finally process the way-file // *** finally process the way-file
wayOutStream = createOutStream( wayFileOut ); wayOutStream = createOutStream( wayFileOut );
new WayIterator( this, true ).processFile( wayFileIn ); new WayIterator( this, true ).processFile( wayFileIn );
wayOutStream.close(); wayOutStream.close();
}
System.out.println( "-------- route-statistics -------- " );
expctxStat.dumpStatistics();
}
@Override @Override
public void nextWay( WayData data ) throws Exception public void nextWay( WayData data ) throws Exception

View file

@ -20,16 +20,16 @@ public class WayCutter extends MapCreatorBase
public static void main(String[] args) throws Exception public static void main(String[] args) throws Exception
{ {
System.out.println("*** WayCutter: Soft-Cut way-data into tiles"); System.out.println("*** WayCutter: Soft-Cut way-data into tiles");
if (args.length != 4) if (args.length != 3)
{ {
System.out.println("usage: java WayCutter <node-tiles-in> <way-file-in> <way-tiles-out> <relation-file>" ); System.out.println("usage: java WayCutter <node-tiles-in> <way-file-in> <way-tiles-out>" );
return; return;
} }
new WayCutter().process( new File( args[0] ), new File( args[1] ), new File( args[2] ), new File( args[3] ) ); new WayCutter().process( new File( args[0] ), new File( args[1] ), new File( args[2] ) );
} }
public void process( File nodeTilesIn, File wayFileIn, File wayTilesOut, File relationFileIn ) throws Exception public void process( File nodeTilesIn, File wayFileIn, File wayTilesOut ) throws Exception
{ {
this.outTileDir = wayTilesOut; this.outTileDir = wayTilesOut;

View file

@ -8,6 +8,7 @@ import java.util.Collections;
import java.util.List; import java.util.List;
import btools.expressions.BExpressionContext; import btools.expressions.BExpressionContext;
import btools.util.ByteArrayUnifier;
import btools.util.CompactLongMap; import btools.util.CompactLongMap;
import btools.util.CompactLongSet; import btools.util.CompactLongSet;
import btools.util.Crc32; import btools.util.Crc32;
@ -43,6 +44,8 @@ public class WayLinker extends MapCreatorBase
private BExpressionContext expctxWay; private BExpressionContext expctxWay;
private ByteArrayUnifier abUnifier;
private int minLon; private int minLon;
private int minLat; private int minLat;
@ -80,6 +83,8 @@ public class WayLinker extends MapCreatorBase
creationTimeStamp = System.currentTimeMillis(); creationTimeStamp = System.currentTimeMillis();
abUnifier = new ByteArrayUnifier( 16384 );
// then process all segments // then process all segments
new WayIterator( this, true ).processDir( wayTilesIn, ".wt5" ); new WayIterator( this, true ).processDir( wayTilesIn, ".wt5" );
} }
@ -140,7 +145,7 @@ public class WayLinker extends MapCreatorBase
@Override @Override
public void nextWay( WayData way ) throws Exception public void nextWay( WayData way ) throws Exception
{ {
byte[] description = way.description; byte[] description = abUnifier.unify( way.description );
// filter according to profile // filter according to profile
expctxWay.evaluate( false, description, null ); expctxWay.evaluate( false, description, null );

View file

@ -43,7 +43,7 @@ public class MapcreatorTest
// run WayCutter // run WayCutter
File waytiles = new File( tmpdir, "waytiles" ); File waytiles = new File( tmpdir, "waytiles" );
waytiles.mkdir(); waytiles.mkdir();
new WayCutter().process( ftiles, wayFile2, waytiles, relFile ); new WayCutter().process( ftiles, wayFile2, waytiles );
// run WayCutter5 // run WayCutter5
File waytiles55 = new File( tmpdir, "waytiles55" ); File waytiles55 = new File( tmpdir, "waytiles55" );

View file

@ -22,7 +22,7 @@ final class MicroCache
// the object parsing position and length // the object parsing position and length
private byte[] ab; private byte[] ab;
private int aboffset; private int aboffset;
private int ablength; private int aboffsetEnd;
// cache control: a virgin cache can be // cache control: a virgin cache can be
// put to ghost state for later recovery // put to ghost state for later recovery
@ -150,7 +150,8 @@ final class MicroCache
if ( ( fapos[n] & 0x80000000 ) == 0 ) if ( ( fapos[n] & 0x80000000 ) == 0 )
{ {
aboffset = fapos[n]; aboffset = fapos[n];
ablength = ( n+1 < size ? fapos[n+1] & 0x7fffffff : ab.length ) - aboffset; int ablength = ( n+1 < size ? fapos[n+1] & 0x7fffffff : ab.length ) - aboffset;
aboffsetEnd = aboffset + ablength;
fapos[n] |= 0x80000000; // mark deleted fapos[n] |= 0x80000000; // mark deleted
delbytes+= ablength; delbytes+= ablength;
delcount++; delcount++;
@ -172,7 +173,7 @@ final class MicroCache
long id = node.getIdFromPos(); long id = node.getIdFromPos();
if ( getAndClear( id ) ) if ( getAndClear( id ) )
{ {
node.parseNodeBody( this, ablength, nodesMap, dc ); node.parseNodeBody( this, nodesMap, dc );
} }
if ( doCollect && delcount > size / 2 ) // garbage collection if ( doCollect && delcount > size / 2 ) // garbage collection
@ -206,7 +207,7 @@ final class MicroCache
int pos = fapos[i]; int pos = fapos[i];
if ( ( pos & 0x80000000 ) == 0 ) if ( ( pos & 0x80000000 ) == 0 )
{ {
ablength = ( i+1 < size ? fapos[i+1] & 0x7fffffff : ab.length ) - pos; int ablength = ( i+1 < size ? fapos[i+1] & 0x7fffffff : ab.length ) - pos;
System.arraycopy( ab, pos, nab, nab_off, ablength ); System.arraycopy( ab, pos, nab, nab_off, ablength );
nfaid[idx] = faid[i]; nfaid[idx] = faid[i];
nfapos[idx] = nab_off; nfapos[idx] = nab_off;
@ -297,4 +298,9 @@ final class MicroCache
System.arraycopy( ab, aboffset, ta, 0, ta.length ); System.arraycopy( ab, aboffset, ta, 0, ta.length );
aboffset += ta.length; aboffset += ta.length;
} }
public boolean hasMoreData()
{
return aboffset < aboffsetEnd;
}
} }

View file

@ -5,8 +5,11 @@
*/ */
package btools.mapaccess; package btools.mapaccess;
import java.util.*; import java.io.File;
import java.io.*; import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
public final class NodesCache public final class NodesCache
{ {

View file

@ -5,12 +5,14 @@
*/ */
package btools.mapaccess; package btools.mapaccess;
import btools.util.ByteArrayUnifier;
public class OsmNode implements OsmPos public class OsmNode implements OsmPos
{ {
public static final int EXTERNAL_BITMASK = 0x80; public static final int EXTERNAL_BITMASK = 0x80;
public static final int FIRSTFORWAY_BITMASK = 0x40; public static final int VARIABLEDESC_BITMASK = 0x40;
public static final int TRANSFERNODE_BITMASK = 0x20; public static final int TRANSFERNODE_BITMASK = 0x20;
public static final int WRITEDESC_BITMASK = 0x10; public static final int WRITEDESC_BITMASK = 0x10;
public static final int SKIPDETAILS_BITMASK = 0x08; public static final int SKIPDETAILS_BITMASK = 0x08;
@ -107,17 +109,18 @@ public class OsmNode implements OsmPos
} }
public void parseNodeBody( MicroCache is, int bodySize, OsmNodesMap hollowNodes, DistanceChecker dc ) public void parseNodeBody( MicroCache is, OsmNodesMap hollowNodes, DistanceChecker dc )
{ {
selev = is.readShort(); ByteArrayUnifier abUnifier = hollowNodes.getByteArrayUnifier();
bodySize -= 2;
selev = is.readShort();
OsmLink lastlink = null; OsmLink lastlink = null;
int lonIdx = ilon/62500; int lonIdx = ilon/62500;
int latIdx = ilat/62500; int latIdx = ilat/62500;
while( bodySize > 0 ) while( is.hasMoreData() )
{ {
OsmLink link = new OsmLink(); OsmLink link = new OsmLink();
OsmTransferNode firstTransferNode = null; OsmTransferNode firstTransferNode = null;
@ -128,32 +131,33 @@ public class OsmNode implements OsmPos
for(;;) for(;;)
{ {
int bitField = is.readByte(); int bitField = is.readByte();
bodySize -= 1;
if ( (bitField & EXTERNAL_BITMASK) != 0 ) if ( (bitField & EXTERNAL_BITMASK) != 0 )
{ {
// full position for external target // full position for external target
bodySize -= 8;
linklon = is.readInt(); linklon = is.readInt();
linklat = is.readInt(); linklat = is.readInt();
} }
else else
{ {
// reduced position for internal target // reduced position for internal target
bodySize -= 4;
linklon = is.readShort(); linklon = is.readShort();
linklat = is.readShort(); linklat = is.readShort();
linklon += lonIdx*62500 + 31250; linklon += lonIdx*62500 + 31250;
linklat += latIdx*62500 + 31250; linklat += latIdx*62500 + 31250;
} }
// read variable length or old 8 byte fixed, and ensure that 8 bytes is only fixed
boolean readFix8 = (bitField & VARIABLEDESC_BITMASK ) == 0; // old, fix length format
if ( (bitField & WRITEDESC_BITMASK ) != 0 ) if ( (bitField & WRITEDESC_BITMASK ) != 0 )
{ {
int dlen = is.readByte(); description = new byte[dlen]; is.readFully( description ); byte[] ab = new byte[readFix8 ? 8 : is.readByte()];
bodySize -= 1 + dlen; is.readFully( ab );
description = abUnifier.unify( ab );
} }
if ( (bitField & NODEDESC_BITMASK ) != 0 ) if ( (bitField & NODEDESC_BITMASK ) != 0 )
{ {
int dlen = is.readByte(); nodeDescription = new byte[dlen]; is.readFully( nodeDescription ); byte[] ab = new byte[readFix8 ? 8 : is.readByte()];
bodySize -= 1 + dlen; is.readFully( ab );
nodeDescription = abUnifier.unify( ab );
} }
if ( (bitField & SKIPDETAILS_BITMASK ) != 0 ) if ( (bitField & SKIPDETAILS_BITMASK ) != 0 )
{ {
@ -169,7 +173,6 @@ public class OsmNode implements OsmPos
trans.ilon = linklon; trans.ilon = linklon;
trans.ilat = linklat; trans.ilat = linklat;
trans.descriptionBitmap = description; trans.descriptionBitmap = description;
bodySize -= 2;
trans.selev = is.readShort(); trans.selev = is.readShort();
if ( lastTransferNode == null ) if ( lastTransferNode == null )
{ {

View file

@ -7,10 +7,19 @@ package btools.mapaccess;
import java.util.*; import java.util.*;
import btools.util.ByteArrayUnifier;
public final class OsmNodesMap public final class OsmNodesMap
{ {
private HashMap<Long,OsmNode> hmap = new HashMap<Long,OsmNode>(); private HashMap<Long,OsmNode> hmap = new HashMap<Long,OsmNode>();
private ByteArrayUnifier abUnifier = new ByteArrayUnifier( 16384 );
public ByteArrayUnifier getByteArrayUnifier()
{
return abUnifier;
}
private NodesList completedNodes = null; private NodesList completedNodes = null;
/** /**

View file

@ -0,0 +1,40 @@
package btools.util;
public final class ByteArrayUnifier
{
private byte[][] byteArrayCache;
private int size;
public ByteArrayUnifier( int size )
{
this.size = size;
byteArrayCache = new byte[size][];
}
/**
* Unify a byte array in order to reuse instances when possible.
* The byte arrays are assumed to be treated as immutable,
* allowing the reuse
* @param the byte array to unify
* @return the cached instance or the input instanced if not cached
*/
public byte[] unify( byte[] ab )
{
int n = ab.length;
int crc = Crc32.crc( ab, 0, n );
int idx = (crc & 0xfffffff) % size;
byte[] abc = byteArrayCache[idx];
if ( abc != null && abc.length == n )
{
int i = 0;
while( i < n )
{
if ( ab[i] != abc[i] ) break;
i++;
}
if ( i == n ) return abc;
}
byteArrayCache[idx] = ab;
return ab;
}
}

View file

@ -1,4 +1,5 @@
---lookupversion:2 ---lookupversion:10
---readvarlength
---context:way ---context:way

View file

@ -14,13 +14,15 @@ rm -rf /var/www/brouter/segments2_lastrun
mkdir tmp mkdir tmp
cd tmp cd tmp
mkdir nodetiles mkdir nodetiles
/java/bin/java -Xmx256m -Xms256m -Xmn32m -cp ../pbfparser.jar:../brouter.jar btools.mapcreator.OsmCutter ../lookups.dat nodetiles ways.dat cycleways.dat ../planet-latest.osm.pbf /java/bin/java -Xmx256m -Xms256m -Xmn32m -cp ../pbfparser.jar:../brouter.jar btools.mapcreator.OsmCutter ../lookups.dat nodetiles ways.dat relations.dat ../planet-latest.osm.pbf
/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.RelationMerger ways.dat ways2.dat relations.dat ../lookups.dat ../trekking.brf ../softaccess.brf
mkdir ftiles mkdir ftiles
/java/bin/java -Xmx512M -Xms512M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.NodeFilter nodetiles ways.dat ftiles /java/bin/java -Xmx512M -Xms512M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.NodeFilter nodetiles ways.dat ftiles
mkdir waytiles mkdir waytiles
/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter ftiles ways.dat waytiles cycleways.dat ../lookups.dat ../trekking.brf ../softaccess.brf /java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter ftiles ways.dat waytiles
mkdir waytiles55 mkdir waytiles55
/java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter5 ftiles waytiles waytiles55 bordernids.dat /java/bin/java -Xmx2600M -Xms2600M -Xmn32M -cp ../brouter.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.WayCutter5 ftiles waytiles waytiles55 bordernids.dat