diff --git a/brouter-core/src/main/java/btools/router/ProfileCache.java b/brouter-core/src/main/java/btools/router/ProfileCache.java index 1a71d4b..f03e6ae 100644 --- a/brouter-core/src/main/java/btools/router/ProfileCache.java +++ b/brouter-core/src/main/java/btools/router/ProfileCache.java @@ -64,7 +64,8 @@ public final class ProfileCache BExpressionContextGlobal expctxGlobal = new BExpressionContextGlobal( meta ); rc.expctxWay = new BExpressionContextWay( rc.memoryclass * 512, meta ); - rc.expctxNode = new BExpressionContextNode( rc.memoryclass * 128, meta ); + rc.expctxNode = new BExpressionContextNode( 0, meta ); + rc.expctxNode.setForeignContext( rc.expctxWay ); meta.readMetaData( new File( profileDir, "lookups.dat" ) ); diff --git a/brouter-expressions/src/main/java/btools/expressions/BExpression.java b/brouter-expressions/src/main/java/btools/expressions/BExpression.java index c66fc45..a029023 100644 --- a/brouter-expressions/src/main/java/btools/expressions/BExpression.java +++ b/brouter-expressions/src/main/java/btools/expressions/BExpression.java @@ -24,6 +24,7 @@ final class BExpression private static final int LOOKUP_EXP = 32; private static final int NUMBER_EXP = 33; private static final int VARIABLE_EXP = 34; + private static final int FOREIGN_VARIABLE_EXP = 35; private int typ; private BExpression op1; @@ -138,6 +139,8 @@ final class BExpression exp.typ = ASSIGN_EXP; String variable = ctx.parseToken(); if ( variable == null ) throw new IllegalArgumentException( "unexpected end of file" ); + if ( variable.indexOf( '=' ) >= 0 ) throw new IllegalArgumentException( "variable name cannot contain '=': " + variable ); + if ( variable.indexOf( ':' ) >= 0 ) throw new IllegalArgumentException( "cannot assign context-prefixed variable: " + variable ); exp.variableIdx = ctx.getVariableIdx( variable, true ); if ( exp.variableIdx < ctx.getMinWriteIdx() ) throw new IllegalArgumentException( "cannot assign to readonly variable " + variable ); } @@ -175,6 +178,13 @@ final class BExpression } } } + else if ( ( idx = operator.indexOf( ':' ) ) >= 0 ) + { + String context = operator.substring( 0, idx ); + String varname = operator.substring( idx+1 ); + exp.typ = FOREIGN_VARIABLE_EXP; + exp.variableIdx = ctx.getForeignVariableIdx( context, varname ); + } else if ( (idx = ctx.getVariableIdx( operator, false )) >= 0 ) { exp.typ = VARIABLE_EXP; @@ -257,6 +267,7 @@ final class BExpression case LOOKUP_EXP: return ctx.getLookupMatch( lookupNameIdx, lookupValueIdxArray ); case NUMBER_EXP: return numberValue; case VARIABLE_EXP: return ctx.getVariableValue( variableIdx ); + case FOREIGN_VARIABLE_EXP: return ctx.getForeignVariableValue( variableIdx ); case NOT_EXP: return op1.evaluate(ctx) == 0.f ? 1.f : 0.f; default: throw new IllegalArgumentException( "unknown op-code: " + typ ); } diff --git a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java index 9fc4526..3d265d5 100644 --- a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java +++ b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java @@ -70,6 +70,8 @@ public abstract class BExpressionContext implements IByteArrayUnifier private float[] currentVars; private int currentVarOffset; + private BExpressionContext foreignContext; + protected void setInverseVars() { currentVarOffset = nBuildInVars; @@ -108,8 +110,11 @@ public abstract class BExpressionContext implements IByteArrayUnifier if ( Boolean.getBoolean( "disableExpressionCache" ) ) hashSize = 1; // create the expression cache - cache = new LruMap( 4*hashSize, hashSize ); - resultVarCache = new LruMap( 4096, 4096 ); + if ( hashSize > 0 ) + { + cache = new LruMap( 4*hashSize, hashSize ); + resultVarCache = new LruMap( 4096, 4096 ); + } } /** @@ -347,6 +352,18 @@ public abstract class BExpressionContext implements IByteArrayUnifier { requests++; lookupDataValid = false; // this is an assertion for a nasty pifall + + if ( cache == null ) + { + decode( lookupData, inverseDirection, ab ); + if ( currentVars == null || currentVars.length != nBuildInVars ) + { + currentVars = new float[nBuildInVars]; + } + evaluateInto( currentVars, 0 ); + currentVarOffset = 0; + return; + } CacheNode cn; if ( lastCacheNode.ab == ab ) @@ -660,6 +677,50 @@ public abstract class BExpressionContext implements IByteArrayUnifier Integer num = lookupNumbers.get( name ); return num != null && lookupData[num.intValue()] == 2; } + + public int getOutputVariableIndex( String name ) + { + int idx = getVariableIdx( name, false ); + if ( idx < 0 ) + { + throw new IllegalArgumentException( "unknown variable: " + name ); + } + if ( idx < minWriteIdx ) + { + throw new IllegalArgumentException( "bad access to global variable: " + name ); + } + for( int i=0; i>>= 1; + bits--; + return value; + } + + public long decodeBits( int count ) + { + if ( count == 0 ) + { + return 0; + } + fillBuffer(); + long mask = -1L >>> ( 64 - count ); + long value = b & mask; + b >>>= count; + bits -= count; + return value; + } + + /** + * decode an integer in the range 0..max (inclusive). + */ + public long decodeBounded( long max ) + { + long value = 0; + long im = 1; // integer mask + fillBuffer(); + while (( value | im ) <= max) + { + if ( ( b & 1 ) != 0 ) + value |= im; + b >>>= 1; + bits--; + im <<= 1; + } + return value; + } + + /** + * decode a small number with a variable bit length + * (poor mans huffman tree) + * 1 -> 0 + * 01 -> 1 + following 1-bit word ( 1..2 ) + * 001 -> 3 + following 2-bit word ( 3..6 ) + * 0001 -> 7 + following 3-bit word ( 7..14 ) etc. + */ + public int decodeInt() + { + long range = 1; + int cnt = 1; + fillBuffer(); + while ((b & range) == 0) + { + range = (range << 1) | 1; + cnt++; + } + b >>>= cnt; + bits -= cnt; + return (int)((range >>> 1) + ( cnt > 1 ? decodeBits( cnt-1 ) : 0 )); + } + + /** + * double-log variant of decodeVarBits better suited for + * distributions with a big-number tail + */ + public long decodeLong() + { + int n = decodeInt(); + return (1L << n) + decodeBits( n ) - 1L; + } + + public long[] decodeSortedArray() + { + int size = decodeInt(); + long[] values = new long[size]; + if ( size == 0 ) + { + return values; + } + int offset = 0; + long value = 0; + int bits = decodeInt(); + int[] sizestack = new int[bits]; + int stackpointer = 0; + + for(;;) + { + while( size > 1 && bits > 0 ) + { + int size2 = (int)decodeBounded( size ); + sizestack[stackpointer++] = size2; + size -= size2; + value <<= 1; + bits--; + } + if ( size == 1 ) + { + values[offset++] = (value << bits) | decodeBits( bits ); + } + else + { + while (size-- > 0) + { + values[offset++] = value; + } + } + if ( stackpointer == 0 ) + { + return values; + } + while ( ( value & 1L ) == 1L ) + { + value >>= 1; + bits++; + } + value |= 1L; + size = sizestack[--stackpointer]; + } + } + + + private void fillBuffer() + { + while (bits <= 56) + { + if ( idx < idxMax ) + { + b |= (ab[++idx] & 0xffL) << bits; + } + bits += 8; + } + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapdecoder/CharDecoder.java b/brouter-map-creator/src/main/java/btools/mapdecoder/CharDecoder.java new file mode 100644 index 0000000..b2269ee --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapdecoder/CharDecoder.java @@ -0,0 +1,44 @@ +package btools.mapdecoder; + +/** + * Decoder for unicode characters, using simple (1st order) huffmann + */ +public final class CharDecoder extends HuffmannTreeDecoder +{ + private long[] alphabet; + private int range; + private char[] buffer = new char[64]; + + public CharDecoder( BitReadBuffer brb ) + { + super( brb ); + } + + @Override + protected Object decodeTree() + { + alphabet = brb.decodeSortedArray(); + range = alphabet.length - 1; +System.out.println( "decoded alphabet of length " + alphabet.length + " idx3 = " + alphabet[3] ); + return super.decodeTree(); + } + + protected Character decodeItem() + { + int idx = (int)brb.decodeBounded( range ); + long lc = alphabet[idx]; +System.out.println( "decoded item: c=" + ((char)lc) + " idx=" + idx ); + return Character.valueOf( (char)lc ); + } + + public String decodeString() + { + int n = brb.decodeInt(); + char[] b = n <= buffer.length ? buffer : new char[n]; + for( int i=0; i +{ + private Object tree; + protected BitReadBuffer brb; + + protected HuffmannTreeDecoder( BitReadBuffer brb ) + { + this.brb = brb; + tree = decodeTree(); + } + + public V decode() + { + Object node = tree; + while (node instanceof TreeNode) + { + TreeNode tn = (TreeNode) node; + node = brb.decodeBit() ? tn.child2 : tn.child1; + } + if ( node == null ) + { + return decodeItem(); // inline item + } + return (V) node; + } + + + protected Object decodeTree() + { + boolean isNode = brb.decodeBit(); + if ( isNode ) + { + TreeNode node = new TreeNode(); + node.child1 = decodeTree(); + node.child2 = decodeTree(); + return node; + } + boolean isInlinePrefix = brb.decodeBit(); + if ( isInlinePrefix ) + { + return null; + } + return decodeItem(); + } + + + private static final class TreeNode + { + public Object child1; + public Object child2; + } + + protected abstract V decodeItem(); +} diff --git a/brouter-map-creator/src/main/java/btools/mapdecoder/LocaleIndexMapping.java b/brouter-map-creator/src/main/java/btools/mapdecoder/LocaleIndexMapping.java new file mode 100644 index 0000000..24a3705 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapdecoder/LocaleIndexMapping.java @@ -0,0 +1,70 @@ +package btools.mapdecoder; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.zip.Inflater; + +/** + * Manage the mapping between locale and native node indexes + */ +public class LocaleIndexMapping +{ + private int[] refZoomDelta; + private int[] refNativeIndex; + private OsmTile[] tileForZoomDelta; + + public LocaleIndexMapping( OsmTile tile, BitReadBuffer brb ) throws Exception + { + // prepare the locale index array + int localeNodeCount = brb.decodeInt(); + refZoomDelta = new int[localeNodeCount]; + refNativeIndex = new int[localeNodeCount]; + + tileForZoomDelta = new OsmTile[tile.zoom + 1]; + for( OsmTile t = tile; t != null; t = t.parent ) + { + tileForZoomDelta[tile.zoom-t.zoom] = t; + } + + // decode the down-zoom refs + for( int zoomDelta=tile.zoom; zoomDelta > 0; zoomDelta-- ) + { + long[] localeIndexes = brb.decodeSortedArray(); + long[] nativeIndexes = brb.decodeSortedArray(); + + for( int i=0; inative mapping for zoomDelta=0 + int localeIdx = 0; + int nodecount = tile.nodePositions.length; + for( int i=0; i nodes; + + public static NodeTreeElement createNodeTree( long[] values, int offset, int subsize, long nextbit, long mask ) + { + if ( nextbit == 0 ) + { + return null; + } + + if ( subsize < 1 ) + { + return null; + } + + long data = mask & values[offset]; + mask |= nextbit; + + // count 0-bit-fraction + int i = offset; + int end = subsize + offset; + for ( ; i < end; i++ ) + { + if ( ( values[i] & mask ) != data ) + { + break; + } + } + int size1 = i - offset; + int size2 = subsize - size1; + +System.out.println( "createNodeTree: offset=" + offset + " subsize=" + subsize + " size1=" + size1 + " size2=" + size2 ); + + NodeTreeElement nte = new NodeTreeElement(); + nte.offset = offset; + nte.nnodes = subsize; + + nte.child0 = createNodeTree( values, offset, size1, nextbit >> 1, mask ); + nte.child1 = createNodeTree( values, i, size2, nextbit >> 1, mask ); + + return nte; + } + + public String toString() + { + return " child0=" + (child0 != null ) + " child1=" + (child1 != null ); + } + +} diff --git a/brouter-map-creator/src/main/java/btools/mapdecoder/OsmNode.java b/brouter-map-creator/src/main/java/btools/mapdecoder/OsmNode.java new file mode 100644 index 0000000..9860af9 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapdecoder/OsmNode.java @@ -0,0 +1,34 @@ +package btools.mapdecoder; + +public class OsmNode extends OsmObject +{ + public int ilon; + public int ilat; + + public boolean inBBox( int z, int x, int y ) + { + int shift = 28-z; + int x0 = x << shift; + int x1 = (x+1) << shift; + int y0 = y << shift; + int y1 = (y+1) << shift; + boolean outofbox = x1 < ilon || x0 >= ilon || y1 < ilat || y0 >= ilat; + return !outofbox; + } + + public static double gudermannian(double y) + { + return Math.atan(Math.sinh(y)) * (180. / Math.PI); + } + + public double getLon() + { + return (((double)ilon)/( 1L << 27 ) - 1.)*180.; + } + + public double getLat() + { + double y = (1. - ((double)ilat)/( 1L << 27 ))*Math.PI; + return gudermannian(y); + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapdecoder/OsmObject.java b/brouter-map-creator/src/main/java/btools/mapdecoder/OsmObject.java new file mode 100644 index 0000000..4636c5b --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapdecoder/OsmObject.java @@ -0,0 +1,12 @@ +package btools.mapdecoder; + +import java.util.Map; + +/** + * Base class of Nodes, Ways and Relations + */ +public class OsmObject +{ + public int id; + public Map tags; +} diff --git a/brouter-map-creator/src/main/java/btools/mapdecoder/OsmRelation.java b/brouter-map-creator/src/main/java/btools/mapdecoder/OsmRelation.java new file mode 100644 index 0000000..9dcecf9 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapdecoder/OsmRelation.java @@ -0,0 +1,48 @@ +package btools.mapdecoder; + +import java.util.List; + +public class OsmRelation extends OsmObject +{ + public List members; + + // bounding box + int minx; + int miny; + int maxx; + int maxy; + + public void calcBBox() + { + for( int i=0; i maxx ) maxx = w.maxx; + if ( w.miny < miny ) miny = w.miny; + if ( w.maxy > maxy ) maxy = w.maxy; + } + } + } + + public boolean inBBox( int z, int x, int y ) + { + int shift = 28-z; + int x0 = x << shift; + int x1 = (x+1) << shift; + int y0 = y << shift; + int y1 = (y+1) << shift; + boolean outofbox = x1 < minx || x0 >= maxx || y1 < miny || y0 >= maxy; + return !outofbox; + } + +} diff --git a/brouter-map-creator/src/main/java/btools/mapdecoder/OsmRelationMember.java b/brouter-map-creator/src/main/java/btools/mapdecoder/OsmRelationMember.java new file mode 100644 index 0000000..0639660 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapdecoder/OsmRelationMember.java @@ -0,0 +1,9 @@ +package btools.mapdecoder; + +import java.util.List; + +public class OsmRelationMember extends OsmObject +{ + public OsmWay way; + public String role; +} diff --git a/brouter-map-creator/src/main/java/btools/mapdecoder/OsmTile.java b/brouter-map-creator/src/main/java/btools/mapdecoder/OsmTile.java new file mode 100644 index 0000000..ae1a43c --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapdecoder/OsmTile.java @@ -0,0 +1,35 @@ +package btools.mapdecoder; + +import java.util.Collections; +import java.util.List; + +/** + * Container for waydata on the preprocessor level + * + * @author ab + */ +public class OsmTile +{ + public OsmTile parent; + public long sourceId; + + public int zoom; + public int x; + public int y; + + private static List emptyNodes = Collections.EMPTY_LIST; + private static List emptyWays = Collections.EMPTY_LIST; + private static List emptyRelations = Collections.EMPTY_LIST; + + public List nodes = emptyNodes; + public List ways = emptyWays; + public List relations = emptyRelations; + + public long[] nodePositions; + + + public String toString() + { + return "z=" + zoom+ " x=" + x + " y=" + y + " nodes=" + nodes.size() + " ways=" + ways.size() + " rels=" + relations.size(); + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapdecoder/OsmWay.java b/brouter-map-creator/src/main/java/btools/mapdecoder/OsmWay.java new file mode 100644 index 0000000..8e4a65a --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapdecoder/OsmWay.java @@ -0,0 +1,45 @@ +package btools.mapdecoder; + +import java.util.List; + +public class OsmWay extends OsmObject +{ + public List nodes; + + // bounding box + int minx; + int miny; + int maxx; + int maxy; + + public void calcBBox() + { + for( int i=0; i maxx ) maxx = n.ilon; + if ( n.ilat < miny ) miny = n.ilat; + if ( n.ilat > maxy ) maxy = n.ilat; + } + } + } + + public boolean inBBox( int z, int x, int y ) + { + int shift = 28-z; + int x0 = x << shift; + int x1 = (x+1) << shift; + int y0 = y << shift; + int y1 = (y+1) << shift; + boolean outofbox = x1 < minx || x0 >= maxx || y1 < miny || y0 >= maxy; + return !outofbox; + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapdecoder/TagSetDecoder.java b/brouter-map-creator/src/main/java/btools/mapdecoder/TagSetDecoder.java new file mode 100644 index 0000000..9145c8c --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapdecoder/TagSetDecoder.java @@ -0,0 +1,30 @@ +package btools.mapdecoder; + +/** + * Decoder for a set of tags + * + * Only tagsets detected at least twice + * have their own huffmann-codes, those + * detected only once are coded inline + */ +public final class TagSetDecoder extends HuffmannTreeDecoder +{ + public TagSetDecoder( BitReadBuffer brb ) + { + super( brb ); + } + + protected int[] decodeItem() + { + int tagcount = brb.decodeInt(); + int[] data = new int[tagcount]; + int lastIdx = -1; + for( int i=0; i stringList; + private ArrayList taglist; + private int roleIdx; + + private String decodeString( BitReadBuffer brb ) + { + boolean newIdx = brb.decodeBit(); + if ( newIdx ) + { + int slen = brb.decodeInt(); + try + { + String s = new String( textHeader, nextStringStart, slen, "UTF8" ); + nextStringStart += slen; + stringList.add( s ); + return s; + } + catch( UnsupportedEncodingException uee ) + { + throw new RuntimeException( uee ); + } + } + int idx = (int)brb.decodeBounded( stringList.size()-1 ); + return stringList.get( idx ); + } + + private class Tag extends HuffmannTreeDecoder + { + String name; + + Tag( BitReadBuffer brb, String tagName ) + { + super( brb ); + name = tagName; + } + + protected String decodeItem() + { + return decodeString( brb ); + } + } + + public TagValueDecoder( BitReadBuffer brb, byte[] textHeader ) + { + this.textHeader = textHeader; + stringList = new ArrayList(); + + int ntags = brb.decodeInt(); + taglist = new ArrayList(); + for( int i=0; i 0 ) + { + td.parent = new TileDecoder().process( tileDir, template, zoom-1, x >> 1, y >> 1 ); + } + + File file = new File( new File( tileDir, "" + zoom ), x + "_" + y + ".osb" ); + if ( !file.exists() ) + { + return td; + } + + DataInputStream dis = new DataInputStream( new FileInputStream( file ) ); + int textHeaderLen = dis.readInt(); + int textHeaderCompressedLen = dis.readInt(); + byte[] textHeaderCompressed = new byte[textHeaderCompressedLen]; + dis.readFully( textHeaderCompressed ); + byte[] textHeader = new byte[textHeaderLen]; + + Inflater decompresser = new Inflater(); + decompresser.setInput( textHeaderCompressed ); + int rawlen = decompresser.inflate( textHeader ); + + int bufferLen = dis.readInt(); + byte[] buffer = new byte[bufferLen]; + dis.readFully( buffer ); + BitReadBuffer brb = new BitReadBuffer( buffer ); + dis.close(); + + tagSetDecoder = new TagSetDecoder( brb ); + tagValueDecoder = new TagValueDecoder( brb, textHeader ); + + // decode the node positions + td.nodePositions = brb.decodeSortedArray(); + int nodecount = td.nodePositions.length; + td.nodes = new ArrayList(nodecount); + + int shift = 56-2*zoom; + long offset = (encodeMorton( x ) << shift) + (encodeMorton( y ) << (shift+1) ); + + for ( int nidx = 0; nidx < nodecount; nidx++ ) + { + OsmNode n = new OsmNode(); + long z = offset + td.nodePositions[nidx]; + n.id = nidx; + n.ilon = decodeMorton( z ); + n.ilat = decodeMorton( z >> 1 ); + td.nodes.add( n ); + } + + LocaleIndexMapping indexMapping = new LocaleIndexMapping( td, brb ); + + // decode tagged nodes + long[] taggedIndexes = brb.decodeSortedArray(); + int ntaggedNodes = taggedIndexes.length; + for( int tnidx=0; tnidx( nways ); + for( int widx=0; widx( nodeIndexes.length ); + for( int i=0; i( nrels ); + for( int ridx=0; ridx(nmembers); + for( int midx = 0; midx decodeTagValues() + { + HashMap map = new HashMap(); + int[] tagSet = tagSetDecoder.decode(); + for( int i=0; i> 1)) & 0x3333333333333333L; + x = (x | (x >> 2)) & 0x0F0F0F0F0F0F0F0FL; + x = (x | (x >> 4)) & 0x00FF00FF00FF00FFL; + x = (x | (x >> 8)) & 0x0000FFFF0000FFFFL; + return (int)(x | (x >> 16)); + } + public static long encodeMorton( int x ) + { + long z = x & 0xFFFFFFFFL; + z = (z | (z << 16)) & 0x0000FFFF0000FFFFL; + z = (z | (z << 8)) & 0x00FF00FF00FF00FFL; + z = (z | (z << 4)) & 0x0F0F0F0F0F0F0F0FL; + z = (z | (z << 2)) & 0x3333333333333333L; + return (z|(z << 1)) & 0x5555555555555555L; + } + +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/BitWriteBuffer.java b/brouter-map-creator/src/main/java/btools/mapsplitter/BitWriteBuffer.java new file mode 100644 index 0000000..6fe54d2 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/BitWriteBuffer.java @@ -0,0 +1,231 @@ +package btools.mapsplitter; + +import java.util.TreeMap; + + +public final class BitWriteBuffer +{ + private static TreeMap statsPerName; + private long lastbitpos = 0; + + private byte[] ab; + private int idxMax; + private int idx = -1; + private int bm = 0x100; // byte mask (write mode) + private int b; + + public BitWriteBuffer( byte[] ab ) + { + this.ab = ab; + idxMax = ab.length-1; + } + + /** + * encode a distance with a variable bit length + * (poor mans huffman tree) + * 1 -> 0 + * 01 -> 1 + following 1-bit word ( 1..2 ) + * 001 -> 3 + following 2-bit word ( 3..6 ) + * 0001 -> 7 + following 3-bit word ( 7..14 ) etc. + * + * @see #decodeVarBits + */ + public void encodeInt( int value ) + { + int range = 0; + while (value > range) + { + encodeBit( false ); + value -= range + 1; + range = 2 * range + 1; + } + + encodeBit( true ); + encodeBounded( range, value ); + } + + public void encodeLong( long n ) + { + int maxbit = 0; + long nn = n + 1L; + while( nn > 1L ) + { + maxbit++; + nn >>= 1; + } + encodeInt( maxbit ); + long range = 1 << maxbit; + encodeBounded( range-1L, n + 1L -range ); + } + + public void encodeBit( boolean value ) + { + if ( bm == 0x100 ) + { + bm = 1; + ab[++idx] = 0; + } + if ( value ) + ab[idx] |= bm; + bm <<= 1; + } + + + /** + * encode an integer in the range 0..max (inclusive). + * For max = 2^n-1, this just encodes n bits, but in general + * this is variable length encoding, with the shorter codes + * for the central value range + */ + public void encodeBounded( long max, long value ) + { + long im = 1L; // integer mask + while (im <= max) + { + if ( bm == 0x100 ) + { + bm = 1; + ab[++idx] = 0; + } + if ( ( value & im ) != 0 ) + { + ab[idx] |= bm; + max -= im; + } + bm <<= 1; + im <<= 1; + } + } + + /** + * @return the encoded length in bytes + */ + public int getEncodedLength() + { + return idx + 1; + } + + /** + * @return the encoded length in bits + */ + public long getWritingBitPosition() + { + long bitpos = idx << 3; + int m = bm; + while (m > 1) + { + bitpos++; + m >>= 1; + } + return bitpos; + } + + public void encodeSortedArray( long[] values ) + { + int size = values.length; + encodeInt( size ); + if ( size == 0 ) + { + return; + } + long maxValue = values[size-1]; + int nbits = 0; + while ( maxValue > 0 ) + { + nbits++; + maxValue >>= 1; + } + if ( nbits > 57 ) throw new IllegalArgumentException( "encodeSortedArray accepts 57-bit numbers at max" ); + encodeInt( nbits ); + encodeSortedArray( values, 0, size, ( 1L << nbits ) >> 1, 0L ); + } + + private void encodeSortedArray( long[] values, int offset, int subsize, long nextbit, long mask ) + { + if ( subsize == 1 ) // last-choice shortcut + { + long bit = 1L; + while ( bit <= nextbit ) + { + encodeBit( ( values[offset] & bit ) != 0 ); + bit <<= 1; + } + return; + } + if ( nextbit == 0 ) + { + return; + } + + long data = mask & values[offset]; + mask |= nextbit; + + // count 0-bit-fraction + int i = offset; + int end = subsize + offset; + for ( ; i < end; i++ ) + { + if ( ( values[i] & mask ) != data ) + { + break; + } + } + int size1 = i - offset; + int size2 = subsize - size1; + + encodeBounded( subsize, size2 ); + if ( size1 > 0 ) + { + encodeSortedArray( values, offset, size1, nextbit >> 1, mask ); + } + if ( size2 > 0 ) + { + encodeSortedArray( values, i, size2, nextbit >> 1, mask ); + } + } + + /** + * assign the de-/encoded bits since the last call assignBits to the given + * name. Used for encoding statistics + * + * @see #getBitReport + */ + public void assignBits( String name ) + { + long bitpos = getWritingBitPosition(); + if ( statsPerName == null ) + { + statsPerName = new TreeMap(); + } + long[] stats = statsPerName.get( name ); + if ( stats == null ) + { + stats = new long[2]; + statsPerName.put( name, stats ); + } + stats[0] += bitpos - lastbitpos; + stats[1] += 1; + lastbitpos = bitpos; + } + + /** + * Get a textual report on the bit-statistics + * + * @see #assignBits + */ + public static String getBitReport() + { + if ( statsPerName == null ) + { + return ""; + } + StringBuilder sb = new StringBuilder(); + for ( String name : statsPerName.keySet() ) + { + long[] stats = statsPerName.get( name ); + sb.append( name + " count=" + stats[1] + " bits=" + stats[0] + "\n" ); + } + statsPerName = null; + return sb.toString(); + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/CharEncoder.java b/brouter-map-creator/src/main/java/btools/mapsplitter/CharEncoder.java new file mode 100644 index 0000000..5b346b5 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/CharEncoder.java @@ -0,0 +1,71 @@ +package btools.mapsplitter; + +import java.util.TreeMap; + +/** + * Encoder for characters, using simple (1st order) huffmann + */ +public final class CharEncoder extends HuffmanTreeEncoder +{ + private long[] alphabet; + private int range; + + private TreeMap chars = new TreeMap(); + + public void encode( Character c ) + { + if ( pass == 1 ) + { + chars.put( c, null ); + } + super.encode( c ); + } + + public void encodeDictionary( BitWriteBuffer bwb ) + { + if ( pass == 1 ) // means 2... + { + int idx = 0; + alphabet = new long[chars.size()]; + range = chars.size()-1; + for ( Character c : chars.keySet() ) + { +System.out.println( "assigning index " + idx + " to char=" + c ); + alphabet[idx] = c; + chars.put( c, Integer.valueOf( idx++ ) ); + } + } + if ( alphabet != null ) + { + bwb.encodeSortedArray( alphabet ); + } + super.encodeDictionary( bwb ); + } + + protected void encodeItem( Character c ) + { + int idx = chars.get( c ).intValue(); +System.out.println( "encoding item: c=" + c + " idx=" + idx ); + bwb.encodeBounded( range, idx ); + } + + @Override + public boolean itemEquals( Character c1, Character c2 ) + { + if ( c1 == null ) + { + return c2 == null; + } + if ( c2 == null ) + { + return false; + } + return c1.charValue() == c2.charValue(); + } + + @Override + public int itemHashCode( Character c) + { + return c == 0 ? 0 : c.charValue(); + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/HuffmanTreeEncoder.java b/brouter-map-creator/src/main/java/btools/mapsplitter/HuffmanTreeEncoder.java new file mode 100644 index 0000000..35bf4bf --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/HuffmanTreeEncoder.java @@ -0,0 +1,140 @@ +package btools.mapsplitter; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.PriorityQueue; + +/** + * Encoder for a set of tags + * + * It detects identical sets and sorts them + * into a huffman-tree according to their frequencies + * + * Adapted for 3-pass encoding (counters -> statistics -> encoding ) + * but doesn't do anything at pass1 + */ +public abstract class HuffmanTreeEncoder +{ + private HashMap identityMap; + protected BitWriteBuffer bwb; + protected int pass; + private TreeNode freq1; + + public void encode( V data ) + { + if ( pass == 1 ) + { + return; + } + TreeNode probe = new TreeNode(); + probe.data = data; + TreeNode tn = identityMap.get( probe ); + if ( pass == 3 ) + { + if ( tn.frequency == 1 ) + { + bwb.encodeBounded( freq1.range - 1, freq1.code ); + encodeItem( data ); + } + else + { + bwb.encodeBounded( tn.range - 1, tn.code ); + } + } + else if ( pass == 2 ) + { + if ( tn == null ) + { + tn = probe; + identityMap.put( tn, tn ); + } + tn.frequency++; + } + } + + public void encodeDictionary( BitWriteBuffer bwb ) + { + this.bwb = bwb; + if ( ++pass == 3 ) + { + freq1 = new TreeNode(); + PriorityQueue queue = new PriorityQueue(2*identityMap.size(), new Comparator() + { + @Override + public int compare(TreeNode tn1, TreeNode tn2) + { + if ( tn1.frequency < tn2.frequency ) + return -1; + if ( tn1.frequency > tn2.frequency ) + return 1; + return 0; + } + } ); + queue.add( freq1 ); + while (queue.size() > 1) + { + TreeNode node = new TreeNode(); + node.child1 = queue.poll(); + node.child2 = queue.poll(); + node.frequency = node.child1.frequency + node.child2.frequency; + queue.add( node ); + } + TreeNode root = queue.poll(); + root.encode( 1, 0 ); + } + } + + public HuffmanTreeEncoder() + { + identityMap = new HashMap(); + } + + protected abstract void encodeItem( V data ); + + protected abstract boolean itemEquals( V i1, V i2 ); + + protected abstract int itemHashCode( V i); + + public final class TreeNode + { + public V data; + public int frequency; + public int code; + public int range; + public TreeNode child1; + public TreeNode child2; + + public void encode( int range, int code ) + { + this.range = range; + this.code = code; + boolean isNode = child1 != null; + bwb.encodeBit( isNode ); + if ( isNode ) + { + child1.encode( range << 1, code ); + child2.encode( range << 1, code + range ); + } + else + { + bwb.encodeBit( data == null ); + if ( data != null ) + { + encodeItem( data ); + } + } + } + + @Override + public boolean equals( Object o ) + { + return itemEquals( ((TreeNode)o).data, data ); + } + + @Override + public int hashCode() + { + return itemHashCode( data ); + } + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/MapCreatorBase.java b/brouter-map-creator/src/main/java/btools/mapsplitter/MapCreatorBase.java new file mode 100644 index 0000000..41b24f4 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/MapCreatorBase.java @@ -0,0 +1,173 @@ +/** + * common base class for the map-filters + * + * @author ab + */ +package btools.mapsplitter; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.HashMap; + +import btools.util.DiffCoderDataOutputStream; + +public abstract class MapCreatorBase implements WayListener, NodeListener, RelationListener +{ + private DiffCoderDataOutputStream[] tileOutStreams; + protected File outTileDir; + + protected HashMap tags; + + public void putTag( String key, String value ) + { + if ( tags == null ) tags = new HashMap(); + tags.put( key, value ); + } + + public String getTag( String key ) + { + return tags == null ? null : tags.get( key ); + } + + public HashMap getTagsOrNull() + { + return tags; + } + + public void setTags( HashMap tags ) + { + this.tags = tags; + } + + protected static long readId( DataInputStream is) throws IOException + { + int offset = is.readByte(); + if ( offset == 32 ) return -1; + long i = is.readInt(); + i = i << 5; + return i | offset; + } + + protected static void writeId( DataOutputStream o, long id ) throws IOException + { + if ( id == -1 ) + { + o.writeByte( 32 ); + return; + } + int offset = (int)( id & 0x1f ); + int i = (int)( id >> 5 ); + o.writeByte( offset ); + o.writeInt( i ); + } + + + protected static File[] sortBySizeAsc( File[] files ) + { + int n = files.length; + long[] sizes = new long[n]; + File[] sorted = new File[n]; + for( int i=0; i= ilon || y1 < ilat || y0 >= ilat; + return !outofbox; + } + + public static double gudermannianInv(double latitude) + { + double sign = latitude < 0. ? -1. : 1.; + double sin = Math.sin( latitude * (Math.PI / 180.) * sign); + return sign * (Math.log((1.0 + sin) / (1.0 - sin)) / 2.0); + } + + public static double gudermannian(double y) + { + return Math.atan(Math.sinh(y)) * (180. / Math.PI); + } + + + public double getLon() + { + return (((double)ilon)/( 1L << 27 ) - 1.)*180.; + } + + public double getLat() + { + double y = (1. - ((double)ilat)/( 1L << 27 ))*Math.PI; + return gudermannian(y); + } + + public void calcGeoId() + { + if ( zoom < 0 ) throw new IllegalArgumentException( "no zoom level yet" ); + + gid = 0L; + + for( long bm = 1L << (27-zoom); bm > 0; bm >>= 1 ) + { + gid <<= 2; + if ( ( ilon & bm ) != 0 ) gid |= 1; + if ( ( ilat & bm ) != 0 ) gid |= 2; + } + } + + public static void sortByGeoId( List nodes ) + { + Collections.sort( nodes, new Comparator() + { + @Override + public int compare(NodeData n1, NodeData n2) + { + long d = n1.gid - n2.gid; + + // for equal positions sort by nid + if ( d == 0L ) + { + d = n1.nid - n2.nid; + } + return d == 0 ? 0 : ( d < 0 ? -1 : 1 ); + } + } ); + + } + + public NodeData( DiffCoderDataInputStream dis ) throws Exception + { + zoom = dis.readInt(); + nativeIndex = dis.readInt(); + nid = dis.readDiffed( 0 ); + ilon = (int)dis.readDiffed( 1 ); + ilat = (int)dis.readDiffed( 2 ); + for (;;) + { + String key = dis.readUTF(); + if ( key.length() == 0 ) break; + String value = dis.readUTF(); + putTag( key, value ); + } + } + + public void writeTo( DiffCoderDataOutputStream dos ) throws Exception + { + dos.writeInt( zoom ); + dos.writeInt( nativeIndex ); + dos.writeDiffed( nid, 0 ); + dos.writeDiffed( ilon, 1 ); + dos.writeDiffed( ilat, 2 ); + if ( getTagsOrNull() != null ) + { + for( Map.Entry me : getTagsOrNull().entrySet() ) + { + if ( me.getKey().length() > 0 ) + { + dos.writeUTF( me.getKey() ); + dos.writeUTF( me.getValue() ); + } + } + } + dos.writeUTF( "" ); + } + + private int mercatorLon( long x, long z ) + { + return (int) ( ( 360000000L * x ) >> z ); + } + + private int mercatorLat( long y, long z ) + { + double n = Math.PI - ( 2.0 * Math.PI * y ) / ( 1L << z ); + double d = Math.toDegrees( Math.atan( Math.sinh( n ) ) ); + return (int) ( ( d + 90. ) * 1000000. + 0.5 ); + } + + @Override + public boolean equals( Object o ) + { + if ( o instanceof NodeData ) + { + NodeData n = (NodeData) o; + return n.nid == nid; + } + return false; + } + + @Override + public int hashCode() + { + return (int)((nid >> 32) ^ nid); + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/NodeIterator.java b/brouter-map-creator/src/main/java/btools/mapsplitter/NodeIterator.java new file mode 100644 index 0000000..4f11152 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/NodeIterator.java @@ -0,0 +1,65 @@ +package btools.mapsplitter; + +import java.io.BufferedInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; + +import btools.util.DiffCoderDataInputStream; + +/** + * Iterate over a singe nodefile or a directory + * of nodetiles and feed the nodes to the callback listener + * + * @author ab + */ +public class NodeIterator extends MapCreatorBase +{ + private NodeListener listener; + + public NodeIterator( NodeListener nodeListener ) + { + listener = nodeListener; + } + + public void processDir( File indir, String inSuffix ) throws Exception + { + if ( !indir.isDirectory() ) + { + throw new IllegalArgumentException( "not a directory: " + indir ); + } + + File[] af = sortBySizeAsc( indir.listFiles() ); + for( int i=0; i" ) ) + { + // read additional tags + for(;;) + { + String l2 = _br.readLine(); + if ( l2 == null ) return false; + + int i2; + if ( (i2 = l2.indexOf( "= 0 ) + { // property-tag + i2 += 8; + int ri2 = l2.indexOf( '"', i2 ); + String key = l2.substring( i2, ri2 ); + i2 = l2.indexOf( " v=\"", ri2 ); + if ( i2 >= 0 ) + { + i2 += 4; + int ri3 = l2.indexOf( '"', i2 ); + String value = l2.substring( i2, ri3 ); + + n.putTag( key, value ); + } + } + else if ( l2.indexOf( "" ) >= 0 ) + { // end-tag + break; + } + } + } + nListener.nextNode( n ); + return true; + } + + + private boolean checkWay( String line ) throws Exception + { + int idx0 = line.indexOf( "= 0 ) + { // node reference + i2 += 9; + int ri2 = l2.indexOf( '"', i2 ); + long nid = Long.parseLong( l2.substring( i2, ri2 ) ); + w.nodes.add( nid ); + } + else if ( (i2 = l2.indexOf( "= 0 ) + { // property-tag + i2 += 8; + int ri2 = l2.indexOf( '"', i2 ); + String key = l2.substring( i2, ri2 ); + i2 = l2.indexOf( " v=\"", ri2 ); + if ( i2 >= 0 ) + { + i2 += 4; + int ri3 = l2.indexOf( '"', i2 ); + String value = l2.substring( i2, ri3 ); + w.putTag( key, value ); + } + } + else if ( l2.indexOf( "" ) >= 0 ) + { // end-tag + break; + } + } + wListener.nextWay( w ); + return true; + } + + private boolean checkChangeset( String line ) throws Exception + { + int idx0 = line.indexOf( "" ) ) + { + int loopcheck = 0; + for(;;) + { + String l2 = _br.readLine(); + if ( l2.indexOf("") >= 0 || ++loopcheck > 10000 ) break; + } + } + return true; + } + + private boolean checkRelation( String line ) throws Exception + { + int idx0 = line.indexOf( " roles = new ArrayList(16); + RelationData r = new RelationData( rid, wayIds, roles ); + + // read the nodes + for(;;) + { + String l2 = _br.readLine(); + if ( l2 == null ) return false; + + int i2; + if ( (i2 = l2.indexOf( "= 0 ) // + { // node reference + i2 += 24; + int ri2 = l2.indexOf( '"', i2 ); + long wid = Long.parseLong( l2.substring( i2, ri2 ) ); + + int role1 = ri2 + 8; + int role2 = l2.indexOf( '"', role1 ); + String role = l2.substring( role1, role2 ); + + r.ways.add( wid ); + r.roles.add( role ); + } + else if ( (i2 = l2.indexOf( "= 0 ) + { // property-tag + i2 += 8; + int ri2 = l2.indexOf( '"', i2 ); + String key = l2.substring( i2, ri2 ); + i2 = l2.indexOf( " v=\"", ri2 ); + if ( i2 >= 0 ) + { + i2 += 4; + int ri3 = l2.indexOf( '"', i2 ); + String value = l2.substring( i2, ri3 ); + r.putTag( key, value ); + } + } + else if ( l2.indexOf( "" ) >= 0 ) + { // end-tag + break; + } + } + rListener.nextRelation( r ); + return true; + } + +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/OsmSplitter.java b/brouter-map-creator/src/main/java/btools/mapsplitter/OsmSplitter.java new file mode 100644 index 0000000..d30f115 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/OsmSplitter.java @@ -0,0 +1,109 @@ +/** + * This program + * - reads an *.osm from stdin + * - writes zoom 0 tiles + * + * @author ab + */ +package btools.mapsplitter; + +import java.io.BufferedOutputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; + +import btools.util.DiffCoderDataOutputStream; + +public class OsmSplitter extends MapCreatorBase +{ + private long recordCnt; + private long nodesParsed; + private long waysParsed; + private long relsParsed; + private long changesetsParsed; + + private DataOutputStream wayDos; + private DataOutputStream relDos; + private DiffCoderDataOutputStream nodeDos; + + public static void main(String[] args) throws Exception + { + System.out.println("*** OsmSplitter : transform an osm map to zoom 0 tiles"); + if (args.length != 2) + { + System.out.println("usage : java OsmSplitter "); + return; + } + + new OsmSplitter().process( + new File( args[0] ) + , new File( args[1] ) + ); + } + + public void process (File outTileDir, File mapFile ) throws Exception + { + if ( !outTileDir.isDirectory() ) throw new RuntimeException( "out tile directory " + outTileDir + " does not exist" ); + + File z0 = new File( outTileDir, "0" ); + z0.mkdirs(); + File ways = new File( z0, "0_0.wtl" ); + File nodes = new File( z0, "0_0.ntl" ); + File rels = new File( z0, "0_0.rtl" ); + + wayDos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( ways ) ) ); + relDos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( rels ) ) ); + nodeDos = new DiffCoderDataOutputStream( new BufferedOutputStream( new FileOutputStream( nodes ) ) ); + + // read the osm map into memory + long t0 = System.currentTimeMillis(); + new OsmParser2().readMap( mapFile, this, this, this ); + long t1 = System.currentTimeMillis(); + + System.out.println( "parsing time (ms) =" + (t1-t0) ); + + // close all files + wayDos.close(); + nodeDos.close(); + + System.out.println( statsLine() ); + } + + private void checkStats() + { + if ( (++recordCnt % 100000) == 0 ) System.out.println( statsLine() ); + } + + private String statsLine() + { + return "records read: " + recordCnt + " nodes=" + nodesParsed + " ways=" + waysParsed + " rels=" + relsParsed + " changesets=" + changesetsParsed; + } + + + @Override + public void nextNode( NodeData n ) throws Exception + { + nodesParsed++; + checkStats(); + n.writeTo( nodeDos ); + } + + + @Override + public void nextWay( WayData w ) throws Exception + { + waysParsed++; + checkStats(); + + w.writeTo( wayDos ); + } + + @Override + public void nextRelation( RelationData r ) throws Exception + { + relsParsed++; + checkStats(); + + r.writeTo( relDos ); + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/RelationData.java b/brouter-map-creator/src/main/java/btools/mapsplitter/RelationData.java new file mode 100644 index 0000000..c90ca02 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/RelationData.java @@ -0,0 +1,74 @@ +package btools.mapsplitter; + +import java.io.DataInputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import btools.util.LongList; + +/** + * Container for relation data on the preprocessor level + * + * @author ab + */ +public class RelationData extends MapCreatorBase +{ + public long rid; + public LongList ways; + public List roles; + + public RelationData( long id, LongList ways, List roles ) + { + rid = id; + this.ways = ways; + this.roles = roles; + } + + public RelationData( DataInputStream di ) throws Exception + { + ways = new LongList( 16 ); + roles = new ArrayList(); + rid = readId( di) ; + for (;;) + { + String key = di.readUTF(); + if ( key.length() == 0 ) break; + String value = di.readUTF(); + putTag( key, value ); + } + for (;;) + { + long wid = readId( di ); + if ( wid == -1 ) break; + ways.add( wid ); + roles.add( di.readUTF() ); + } + } + + public void writeTo( java.io.DataOutputStream dos ) throws Exception + { + writeId( dos, rid ); + if ( getTagsOrNull() != null ) + { + for( Map.Entry me : getTagsOrNull().entrySet() ) + { + if ( me.getKey().length() > 0 ) + { + dos.writeUTF( me.getKey() ); + dos.writeUTF( me.getValue() ); + } + } + } + dos.writeUTF( "" ); + + int size = ways.size(); + for( int i=0; i < size; i++ ) + { + writeId( dos, ways.get( i ) ); + dos.writeUTF( roles.get(i) ); + } + writeId( dos, -1 ); // stopbyte + } + +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/RelationIterator.java b/brouter-map-creator/src/main/java/btools/mapsplitter/RelationIterator.java new file mode 100644 index 0000000..c9b1b07 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/RelationIterator.java @@ -0,0 +1,44 @@ +package btools.mapsplitter; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; + +/** + * Iterate over a relation file + * + * @author ab + */ +public class RelationIterator extends MapCreatorBase +{ + private RelationListener listener; + + public RelationIterator( RelationListener relationListener ) + { + listener = relationListener; + } + + public void processFile(File relationfile) throws Exception + { + System.out.println( "*** RelationIterator reading: " + relationfile ); + + listener.relationFileStart( relationfile ); + + DataInputStream di = new DataInputStream( new BufferedInputStream ( new FileInputStream( relationfile ) ) ); + try + { + for(;;) + { + RelationData r = new RelationData( di ); + listener.nextRelation( r ); + } + } + catch( EOFException eof ) + { + di.close(); + } + listener.relationFileEnd( relationfile ); + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/RelationListener.java b/brouter-map-creator/src/main/java/btools/mapsplitter/RelationListener.java new file mode 100644 index 0000000..19d9c77 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/RelationListener.java @@ -0,0 +1,18 @@ +package btools.mapsplitter; + +import java.io.File; + + +/** + * Callbacklistener for Relations + * + * @author ab + */ +public interface RelationListener +{ + void relationFileStart( File relfile ) throws Exception; + + void nextRelation( RelationData data ) throws Exception; + + void relationFileEnd( File relfile ) throws Exception; +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/TagSetEncoder.java b/brouter-map-creator/src/main/java/btools/mapsplitter/TagSetEncoder.java new file mode 100644 index 0000000..c0123d5 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/TagSetEncoder.java @@ -0,0 +1,194 @@ +package btools.mapsplitter; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.PriorityQueue; + +/** + * Encoder for a set of tags + * + * It detects identical sets and sorts them + * into a huffman-tree according to their frequencies + * + * Adapted for 3-pass encoding (counters -> statistics -> encoding ) + * but doesn't do anything at pass1 + */ +public final class TagSetEncoder +{ + private HashMap identityMap; + private BitWriteBuffer bwb; + private int pass; + private TagSet freq1; + + public void encodeTagSet( int[] data ) + { + if ( pass == 1 ) + { + return; + } + TagSet tvsProbe = new TagSet(); + tvsProbe.data = data; + TagSet tvs = identityMap.get( tvsProbe ); + if ( pass == 3 ) + { + if ( tvs.frequency == 1 ) + { + bwb.encodeBounded( freq1.range - 1, freq1.code ); + encodeTagSequence( bwb, data ); + } + else + { + bwb.encodeBounded( tvs.range - 1, tvs.code ); + } + } + else if ( pass == 2 ) + { + if ( tvs == null ) + { + tvs = tvsProbe; + identityMap.put( tvs, tvs ); + } + tvs.frequency++; + } + } + + public void encodeDictionary( BitWriteBuffer bwb ) + { + if ( ++pass == 3 ) + { + freq1 = new TagSet(); + PriorityQueue queue = new PriorityQueue(2*identityMap.size(), new TagSet.FrequencyComparator()); + for( TagSet ts : identityMap.values() ) + { + if ( ts.frequency > 1 ) + { + queue.add( ts ); + } + else + { + freq1.frequency++; + } + } + queue.add( freq1 ); + while (queue.size() > 1) + { + TagSet node = new TagSet(); + node.child1 = queue.poll(); + node.child2 = queue.poll(); + node.frequency = node.child1.frequency + node.child2.frequency; + queue.add( node ); + } + TagSet root = queue.poll(); + root.encode( bwb, 1, 0 ); + } + this.bwb = bwb; + } + + public TagSetEncoder() + { + identityMap = new HashMap(); + } + + private static void encodeTagSequence( BitWriteBuffer bwb, int[] data ) + { + int tagcount = data.length; + bwb.encodeInt( tagcount ); + int lastIdx = -1; + for( int i=0; i + { + + @Override + public int compare(TagSet tvs1, TagSet tvs2) { + if ( tvs1.frequency < tvs2.frequency ) + return -1; + if ( tvs1.frequency > tvs2.frequency ) + return 1; + return 0; + } + } + + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/TagValueEncoder.java b/brouter-map-creator/src/main/java/btools/mapsplitter/TagValueEncoder.java new file mode 100644 index 0000000..521c45c --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/TagValueEncoder.java @@ -0,0 +1,303 @@ +package btools.mapsplitter; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.PriorityQueue; + +/** + * Encoder for the tag-value statistics + * + * @author ab + */ +public class TagValueEncoder +{ + HashMap tags = new HashMap(); + ArrayList groups = new ArrayList(); + + ByteArrayOutputStream baos ; + DataOutputStream dos; + + ArrayList stringList; + HashMap stringMap; + ArrayList taglist; + + private int setId = 0; + private int nextIdx = 0; + + private int pass; + + private static String[][] taggroups = new String[][] { + { "highway", "name", "maxspeed", "lanes", "service", "tracktype", "surface" } + , { "access", "foot", "bicycle", "motorcar", "motor_vehicle", "motorcycle", "vehicle" } + , { "building", "addr:street", "addr:housenumber", "addr:city", "addr:postcode", "addr:housename" } + }; + + private void encodeString( BitWriteBuffer bc, String s ) + { + Integer ii = stringMap.get( s ); + bc.encodeBit( ii == null ); + if ( ii == null ) + { + try + { + byte[] textBytes = s.getBytes( "UTF8" ); + bc.encodeInt( textBytes.length ); + dos.write( textBytes ); + } + catch( Exception e ) + { + throw new RuntimeException( e ); + } + ii = Integer.valueOf( stringList.size() ); + stringList.add( s ); + stringMap.put( s, ii ); + return; + } + bc.encodeBounded( stringList.size()-1, ii.intValue() ); + } + + private class TagGroup implements Comparable + { + String[] names; + int count; + + int lastSetId = 0; + + void incCount() + { + if ( setId != lastSetId ) count++; + lastSetId = setId; + } + + TagGroup( String[] names ) + { + this.names = names; + for( String name : names ) + { + tags.put( name, new Tag( name, this ) ); + } + groups.add( this ); + } + + void indexTags() + { + for( String name : names ) + { + Tag t = tags.get( name ); + if ( t.count > 0 ) t.idx = nextIdx++; + } + } + + @Override + public int compareTo( TagGroup g ) + { + return g.count - count; + } + } + + public TagValueEncoder() + { + for( String[] names : taggroups ) + { + new TagGroup( names ); + } + } + + public class Tag implements Comparable + { + Tag( String name, TagGroup group ) + { + this.name = name; + this.group = group; + } + String name; + int count; + int idx; + + private Object tree; + + HashMap values = new HashMap(); + + List valueList; + TagGroup group; + + void addValue( String value ) + { + Value v = values.get( value ); + if ( v == null ) + { + v = new Value( value ); + values.put( value, v ); + } + v.frequency++; + count++; + } + + public void encodeDictionary( BitWriteBuffer bc ) throws IOException + { + encodeString( bc, name ); + + PriorityQueue queue = new PriorityQueue( values.size() ); + queue.addAll( values.values() ); + while (queue.size() > 1) + { + queue.add( new Value( queue.poll(), queue.poll() ) ); + } + queue.poll().encodeTree( bc, 1, 0 ); + } + + @Override + public int compareTo( Tag t ) + { + return idx - t.idx; + } + } + + private class Value implements Comparable + { + Value( String value ) + { + this.value = value; + } + Value( Value c1, Value c2 ) + { + child1 = c1; + child2 = c2; + frequency = c1.frequency + c2.frequency; + } + String value; + int code; + int range; + Value child1; + Value child2; + int frequency; + + void encodeTree( BitWriteBuffer bc, int range, int code ) throws IOException + { + this.range = range; + this.code = code; + boolean isNode = child1 != null; + bc.encodeBit( isNode ); + if ( isNode ) + { + child1.encodeTree( bc, range << 1, code ); + child2.encodeTree( bc, range << 1, code + range ); + return; + } + bc.encodeBit( false ); // no inline item here + encodeString( bc, value ); + } + + void encode( BitWriteBuffer bc ) + { + bc.encodeBounded( range - 1, code ); + } + + @Override + public int compareTo( Value v ) + { + return frequency - v.frequency; + } + } + + public byte[] encodeDictionary( BitWriteBuffer bc ) throws IOException + { + if ( ++pass == 1 ) + { + return null; + } + else if ( pass == 2 ) + { + nextIdx = 0; + Collections.sort( groups ); + for( TagGroup g : groups ) + { + g.indexTags(); + } + + taglist = new ArrayList(); + for( Tag t : tags.values() ) + { + if ( t.count > 0 ) + { + taglist.add( t ); + } + } + Collections.sort( taglist ); + return null; + } + + stringList = new ArrayList(); + stringMap = new HashMap(); + + baos = new ByteArrayOutputStream(); + dos = new DataOutputStream( baos ); + + bc.encodeInt( taglist.size() ); + for( Tag t : taglist ) + { + t.encodeDictionary( bc ); + } + + dos.close(); + byte[] textData = baos.toByteArray(); + dos = null; + baos = null; + return textData; + } + + public int getTagIndex( String name ) + { + return tags.get( name ).idx; + } + + public List sortTagNames( Collection col ) + { + ArrayList taglist = new ArrayList( col.size() ); + for( String name : col ) + { + taglist.add( tags.get( name ) ); + } + Collections.sort( taglist ); + ArrayList res = new ArrayList( taglist.size() ); + for( Tag t : taglist ) + { + res.add( t.name ); + } + return res; + } + + public void startTagSet() + { + if ( pass == 1 ) + { + setId++; + } + } + + public void encodeValue( BitWriteBuffer bc, String name, String value ) + { + if ( pass == 1 ) + { + Tag t = tags.get( name ); + if ( t == null ) + { + String[] names = new String[1]; + names[0] = name; + new TagGroup( names ); + t = tags.get( name ); + } + t.addValue( value ); + } + else // pass 2+3 + { + tags.get( name ).values.get( value ).encode( bc ); + } + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/TileData.java b/brouter-map-creator/src/main/java/btools/mapsplitter/TileData.java new file mode 100644 index 0000000..d76b8b6 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/TileData.java @@ -0,0 +1,21 @@ +package btools.mapsplitter; + +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +import btools.util.DiffCoderDataInputStream; +import btools.util.DiffCoderDataOutputStream; + +/** + * Container a tile during encoding + */ +public class TileData extends MapCreatorBase +{ + public int zoom; + public int x; + public int y; + public List nodeList; + public TileData parent; +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/TileEncoder.java b/brouter-map-creator/src/main/java/btools/mapsplitter/TileEncoder.java new file mode 100644 index 0000000..e31cd57 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/TileEncoder.java @@ -0,0 +1,555 @@ +package btools.mapsplitter; + +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.Deflater; + +/** + * TileEncoder encodes a given node/way file pair + * + * @author ab + */ +public class TileEncoder extends MapCreatorBase +{ + private Map nodeMap; + private Map wayMap; + private List used = new ArrayList(); + + private NodeData templateNode = new NodeData( 0, 0, 0 ); + private WayData templateWay = new WayData( 0, null ); + + private TileData tile; + + private BitWriteBuffer bwb; + + private byte[] buffer; + + private List wayList; + + private List relationList; + + // statistics only + private int nTagSets; + + private int nTaggedNodes; + private long totalNodes; + private long totalTaggedNodes; + private long totalWays; + private long totalTextBytes; + private long totalTiles; + + private int pass; + private boolean dostats; + private TagValueEncoder tagValueEncoder; + private TagSetEncoder tagSetEnoder; + + public static void main( String[] args ) throws Exception + { + System.out.println( "*** TileEncoder: encodes a given node/way file pair" ); + if ( args.length != 1 ) + { + System.out.println( "usage: java TileEncoder " ); + return; + } + new TileEncoder().process( new File( args[0] ) ); + } + + public void process( File nodeFile) throws Exception + { + TileData t0 = new TileData(); // zoom 0 dummy + process( nodeFile, t0 ); + + System.out.println( "**** total statistics ****" ); + System.out.println( "tiles=" + totalTiles + " nodes=" + totalNodes + " taggedNodes=" + totalTaggedNodes + " ways=" + totalWays + " textBytes= " + totalTextBytes ); + System.out.println( bwb.getBitReport() ); + } + + public void process( File nodeFile, TileData tile ) throws Exception + { + this.tile = tile; + + if ( !nodeFile.exists() ) + { + return; + } + + System.out.println( "******* processing: " + nodeFile ); + + new NodeIterator( this ).processFile( nodeFile ); + + // process childs + + int zoomStep = 2; + int xyStep = 1 << zoomStep; + + int nextZoom = tile.zoom + zoomStep; + int x0 = tile.x << zoomStep; + int y0 = tile.y << zoomStep; + + File childDir = new File( nodeFile.getParentFile().getParentFile(), "" + nextZoom ); + + for( int dx = 0; dx < xyStep; dx++ ) + { + for( int dy = 0; dy < xyStep; dy++ ) + { + TileData nextTile = new TileData(); + nextTile.zoom = nextZoom; + nextTile.x = x0 + dx; + nextTile.y = y0 + dy; + nextTile.parent = tile; + File nextFile = new File( childDir, nextTile.x + "_" + nextTile.y + ".ntl" ); + process( nextFile, nextTile ); + } + } + } + + @Override + public void nodeFileStart( File nodeFile ) throws Exception + { + tile.nodeList = new ArrayList(); + nodeMap = new HashMap(); + wayMap = new HashMap(); + } + + @Override + public void nextNode( NodeData n ) throws Exception + { + // if no level yet, it's ours + if ( n.zoom == -1 || n.zoom == tile.zoom ) + { + n.zoom = tile.zoom; + n.used = true; + tile.nodeList.add( n ); + } + n.localeIndex = nodeMap.size(); + nodeMap.put( n,n ); + n.calcGeoId(); + } + + @Override + public void nodeFileEnd( File nodeFile ) throws Exception + { + NodeData.sortByGeoId( tile.nodeList ); + int idx = 0; + for( NodeData n : tile.nodeList ) + { + n.nativeIndex = idx++; + } + + // read corresponding way-file into wayList + wayList = new ArrayList(); + String name = nodeFile.getName(); + String wayfilename = name.substring( 0, name.length()-3 ) + "wtl"; + File wayfile = new File( nodeFile.getParent(), wayfilename ); + if ( wayfile.exists() ) + { + new WayIterator( this ).processFile( wayfile ); + } + + // read corresponding relation-file + relationList = new ArrayList(); + String relfilename = name.substring( 0, name.length()-3 ) + "rtl"; + File relfile = new File( nodeFile.getParent(), relfilename ); + if ( relfile.exists() ) + { + new RelationIterator( this ).processFile( relfile ); + } + + + int nnodes = tile.nodeList.size(); + + tagValueEncoder = new TagValueEncoder(); + tagSetEnoder = new TagSetEncoder(); + + long[] nodePositions = new long[nnodes]; + for( int i=0; i(); + for( NodeData n : nodeMap.values() ) + { + if ( n.used ) + { + used.add( n ); + } + } + NodeData.sortByGeoId( used ); + int idx = 0; + for( NodeData n : used ) + { + n.localeIndex = idx++; + } + } + + private void writeDownzoomRefs( BitWriteBuffer bwb ) + { + // total locale nodes + bwb.encodeInt( used.size() ); + + for( int zoom=0; zoom locale=" + n.localeIndex + " native=" + n.nativeIndex ); + localeIndexes[idx] = n.localeIndex; + nativeIndexes[idx] = n.nativeIndex; + idx++; + } + } + bwb.encodeSortedArray( localeIndexes ); + if ( dostats ) bwb.assignBits( "localindexes" ); + bwb.encodeSortedArray( nativeIndexes ); + if ( dostats ) bwb.assignBits( "nativeindexes" ); + } + } + + private int getLocaleIndexForNid( long nid ) + { + templateNode.nid = nid; + NodeData n = nodeMap.get( templateNode ); + if ( n == null ) throw new IllegalArgumentException( "nid=" + nid + " not found" ); + n.used = true; + return n.localeIndex; + } + + private void encodeWay( BitWriteBuffer bwb, WayData way ) throws Exception + { + int nnodes = way.nodes.size(); + boolean closedPoly = way.nodes.get(0) == way.nodes.get(nnodes-1); + if ( closedPoly ) + { + nnodes--; + } + if ( nnodes < 2 ) + { + return; + } + + writeTags( way.getTagsOrNull() ); + + bwb.encodeBit( closedPoly ); + bwb.encodeInt( nnodes-2 ); + + if ( dostats ) bwb.assignBits( "way-node-count" ); + + // determine the tile-index for each node + int lastIdx = 0; + for (int i=0; i 0 ) + { + ArrayList goodWays = new ArrayList(); + for( WayData w : wayList ) + { + if ( w.startNodeIdx >= 0 ) + { + goodWays.add( w ); + } + } + WayData.sortByStartNode( goodWays ); + wayList = goodWays; + } + + // encode start-node-indexes + int waycount = wayList.size(); + long[] startIndexes = new long[waycount]; + int i = 0; + for( WayData w : wayList ) + { + w.nativeIndex = i; + startIndexes[i++] = w.startNodeIdx; + } + bwb.encodeSortedArray( startIndexes ); + if ( dostats ) bwb.assignBits( "way-start-idx" ); + } + for( WayData way : wayList ) + { + encodeWay( bwb, way ); + } + } + + private void writeRelations( BitWriteBuffer bwb ) throws Exception + { + bwb.encodeInt( relationList.size() ); + if ( dostats ) bwb.assignBits( "relation-count" ); + for( RelationData rel : relationList ) + { + encodeRelation( bwb, rel ); + } + } + + private void encodeRelation( BitWriteBuffer bwb, RelationData rel ) throws Exception + { + writeTags( rel.getTagsOrNull() ); + + int size = rel.ways.size(); + if ( dostats ) bwb.assignBits( "way-node-count" ); + + // count valid members + int validMembers = 0; + for( int i=0; i < size; i++ ) + { + long wid = rel.ways.get( i ); + String role = rel.roles.get(i); + templateWay.wid = wid; + WayData w = wayMap.get( templateWay ); + if ( w == null ) continue; + validMembers++; + } + bwb.encodeInt( validMembers ); + + for( int i=0; i < size; i++ ) + { + long wid = rel.ways.get( i ); + String role = rel.roles.get(i); + templateWay.wid = wid; + WayData w = wayMap.get( templateWay ); + if ( w == null ) continue; + + int zoomDelta = tile.zoom - w.zoom; + + bwb.encodeInt( zoomDelta ); + bwb.encodeInt( w.nativeIndex ); + tagValueEncoder.encodeValue( bwb, "role", role ); + } + } + + private void writeTaggedNodes() throws Exception + { + // count tagged nodes + int cnt = 0; + for( int idx=0; idx tags ) throws Exception + { + List names; + + if ( tags == null ) + { + tags = new HashMap(); + } + + if ( pass > 1 ) + { + // create tagset as sorted int-array + names = tagValueEncoder.sortTagNames( tags.keySet() ); + int ntags = names.size(); + int[] tagset = new int[ ntags ]; + for( int i=0; i( tags.keySet() ); // unsorted is o.k. in pass 1 + } + + // then encode the values + tagValueEncoder.startTagSet(); + for( String name : names ) + { + String value = tags.get( name ); + tagValueEncoder.encodeValue( bwb, name, value ); + if ( dostats ) bwb.assignBits( "value-index" ); + } + } + +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/TileSplitter.java b/brouter-map-creator/src/main/java/btools/mapsplitter/TileSplitter.java new file mode 100644 index 0000000..78b9ff2 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/TileSplitter.java @@ -0,0 +1,492 @@ +package btools.mapsplitter; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import btools.util.DenseLongMap; +import btools.util.TinyDenseLongMap; + +/** + * TileSplitter splits a tile into pieces + */ +public class TileSplitter extends MapCreatorBase +{ + private NodeData templateNode = new NodeData( 0, 0, 0 ); + + private DenseLongMap nodeIndexMap; + private DenseLongMap bigWayMemberMap; + + private DenseLongMap wayIndexMap; + private DenseLongMap bigRelMemberMap; + + private DenseLongMap relIndexMap; + + private Map nodeMap; + + private List thisLevelNodes; + private Map thisLevelNodesIndexes; + + private List thisLevelWays; + private Map thisLevelWaysIndexes; + + private int level; + private int baseLon; + private int baseLat; + + private int nodeCount = 0; + private String typeSuffix; + private boolean inPassLoop; + private int pass; // 1 == build tileIndexMap, 2 == collect this-level-nodes, 3 == output nodes + private File inTileDir; + + public static void main(String[] args) throws Exception + { + System.out.println("*** TileSplitter: cut tiles into 16 pieces"); + if (args.length != 1) + { + System.out.println("usage: java TileSplitter " ); + + return; + } + new TileSplitter().process( new File( args[0] ) ); + } + + public void process( File tileDir) throws Exception + { + for( int level = 0; level < 12; level += 2 ) + { + process( tileDir, level ); + } + } + + public void process( File tileDir, int level ) throws Exception + { + System.out.println("processing level: " + level ); + + inTileDir = new File( tileDir, "" + (level) ); + outTileDir = new File( tileDir, "" + (level+2) ); + outTileDir.mkdirs(); + this.level = level; + + // *** initialize 3-pass processing of nodes, ways and relations + inPassLoop = false; + new NodeIterator( this ).processDir( inTileDir, ".ntl" ); + } + + @Override + public void nodeFileStart( File nodeFile ) throws Exception + { + if ( !inPassLoop ) + { + inPassLoop = true; + pass = 1; + new NodeIterator( this ).processFile( nodeFile ); + pass = 2; + new NodeIterator( this ).processFile( nodeFile ); + pass = 3; + new NodeIterator( this ).processFile( nodeFile ); + pass = 4; + inPassLoop = false; + } + +System.out.println( "nodeFileStart pass=" + pass ); + + if ( pass == 1 ) + { + getBaseTileFromName( nodeFile.getName() ); + nodeIndexMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap() : new TinyDenseLongMap(); + } + else if ( pass == 2 ) + { + } + else if ( pass == 3 ) + { + nodeMap = new HashMap(); + thisLevelNodes = new ArrayList(); + } + else // nodePass = 4 + { + NodeData.sortByGeoId( thisLevelNodes ); + + thisLevelNodesIndexes = new HashMap(); + int idx = 0; + for( NodeData n : thisLevelNodes ) + { + thisLevelNodesIndexes.put( Long.valueOf( n.nid ), Integer.valueOf( idx++ ) ); + } + thisLevelNodes = null; + } + typeSuffix = "ntl"; + } + + + private void getBaseTileFromName( String name ) + { +System.out.println( "getBaseTileFromName: " + name ); + int idx1 = name.indexOf( '_' ); + int idx2 = name.indexOf( '.' ); + baseLon = Integer.parseInt( name.substring( 0, idx1 ) ); + baseLat = Integer.parseInt( name.substring( idx1+1, idx2 ) ); + } + + @Override + public void nextNode( NodeData n ) throws Exception + { + int tidx = getTileIndex( n ); + + if ( pass == 1 ) + { + nodeCount++; + nodeIndexMap.put( n.nid, tidx ); + } + else if ( pass == 2 ) + { + } + else + { + boolean usedHere = bigWayMemberMap.getInt( n.nid ) == 0; + + if ( usedHere ) // if used on this level... + { + // if no level yet, this is it + if ( n.zoom == -1 ) + { + n.zoom = level; + } + } + + if ( pass == 3 ) + { + if ( n.zoom != -1 ) + { + n.calcGeoId(); + nodeMap.put( n,n ); + if ( n.zoom == level ) + { + thisLevelNodes.add( n ); + } + } + } + else // pass == 4 + { + // add the index + if ( n.zoom == level ) + { + n.nativeIndex = thisLevelNodesIndexes.get( Long.valueOf( n.nid ) ); + } + + if ( usedHere ) + { + n.writeTo( getOutStreamForTile( 16 ) ); + } + n.writeTo( getOutStreamForTile( tidx ) ); // write to subtile + } + } + } + + @Override + public void nodeFileEnd( File nodeFile ) throws Exception + { +System.out.println( "nodeFileEnd pass=" + pass ); + + closeTileOutStreams(); + File parentNodes = new File( outTileDir, getNameForTile( 16 ) ); + + // read corresponding way-file + if ( pass == 2 ) + { + bigWayMemberMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap() : new TinyDenseLongMap(); + } + String name = nodeFile.getName(); + String wayfilename = name.substring( 0, name.length()-3 ) + "wtl"; + File wayfile = new File( inTileDir, wayfilename ); + if ( wayfile.exists() ) + { + new WayIterator( this ).processFile( wayfile ); + } + + // read corresponding relation-file + if ( pass == 1 ) + { + bigRelMemberMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap() : new TinyDenseLongMap(); + } + String relfilename = name.substring( 0, name.length()-3 ) + "rtl"; + File relfile = new File( inTileDir, relfilename ); + if ( relfile.exists() ) + { + new RelationIterator( this ).processFile( relfile ); + } + + if ( pass == 4 ) + { + nodeFile.delete(); + if ( parentNodes.exists() ) + { + parentNodes.renameTo( nodeFile ); + } + else if ( nodeCount > 0 ) + { + nodeFile.createNewFile(); // create even empty to signal existence of childs + } + } + } + + + @Override + public void wayFileStart( File wayFile ) throws Exception + { +System.out.println( "wayFileStart pass=" + pass ); + + if ( pass == 1 ) + { + wayIndexMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap() : new TinyDenseLongMap(); + } + else if ( pass == 3 ) + { + thisLevelWays = new ArrayList(); + } + else if ( pass == 4 ) + { + WayData.sortByStartNode( thisLevelWays ); + + thisLevelWaysIndexes = new HashMap(); + int idx = 0; + for( WayData w : thisLevelWays ) + { + thisLevelWaysIndexes.put( Long.valueOf( w.wid ), Integer.valueOf( idx++ ) ); + } + thisLevelWays = null; + } + typeSuffix = "wtl"; + } + + + @Override + public void nextWay( WayData w ) throws Exception + { + int widx = getTileIndex( w ); + if ( widx == -1 ) + { +System.out.println( "************ invalid way: " + w.wid ); + return; + } + + if ( pass == 1 ) + { + wayIndexMap.put( w.wid, widx ); + } + else // pass >= 2 + { + boolean usedHere = bigRelMemberMap.getInt( w.wid ) == 0; + if ( usedHere || widx == 16 ) + { + // if no level yet, this is it + if ( w.zoom == -1 ) + { + w.zoom = level; + } + + if ( pass == 2 ) + { + int nnodes = w.nodes.size(); + for (int i=0; i> ( 26 - level ) ) & 3; + int idxLat = ( n.ilat >> ( 26 - level ) ) & 3; + return 4 * idxLon + idxLat; + } + + private int getTileIndex( WayData w ) + { + int nnodes = w.nodes.size(); + + int wayTileIndex = 16; + + // determine the tile-index for each node + for (int i=0; i> 2); + int idxLat = baseLat * 4 + (tileIndex & 3); + return idxLon + "_" + idxLat + "." + typeSuffix; + } + +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/WayData.java b/brouter-map-creator/src/main/java/btools/mapsplitter/WayData.java new file mode 100644 index 0000000..5e9e4d6 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/WayData.java @@ -0,0 +1,159 @@ +package btools.mapsplitter; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +import btools.util.LongList; + +/** + * Container for waydata on the preprocessor level + * + * @author ab + */ +public class WayData extends MapCreatorBase +{ + public long wid; + public LongList nodes; + + public int startNodeIdx; + + private int minx; + private int miny; + private int maxx; + private int maxy; + + public int zoom = -1; // the zoom level this node is on + public int nativeIndex; // the index along all NATIVE ways of it's tile + + public void calcBBox( List nodeList ) + { + int nn = nodes.size(); + for( int i=0; i maxx ) maxx = n.ilon; + if ( n.ilat < miny ) miny = n.ilat; + if ( n.ilat > maxy ) maxy = n.ilat; + } + } + } + + public boolean inBBox( int z, int x, int y ) + { + int shift = 28-z; + int x0 = x << shift; + int x1 = (x+1) << shift; + int y0 = y << shift; + int y1 = (y+1) << shift; + boolean outofbox = x1 < minx || x0 >= maxx || y1 < miny || y0 >= maxy; + return !outofbox; + } + + public WayData( long id ) + { + wid = id; + nodes = new LongList( 16 ); + } + + public WayData( long id, LongList nodes ) + { + wid = id; + this.nodes = nodes; + } + + public WayData( DataInputStream di ) throws Exception + { + zoom = di.readInt(); + nativeIndex = di.readInt(); + nodes = new LongList( 16 ); + wid = readId( di) ; + for (;;) + { + String key = di.readUTF(); + if ( key.length() == 0 ) break; + String value = di.readUTF(); + putTag( key, value ); + } + for (;;) + { + long nid = readId( di ); + if ( nid == -1 ) break; + nodes.add( nid ); + } + } + + public void writeTo( DataOutputStream dos ) throws Exception + { + dos.writeInt( zoom ); + dos.writeInt( nativeIndex ); + writeId( dos, wid ); + if ( getTagsOrNull() != null ) + { + for( Map.Entry me : getTagsOrNull().entrySet() ) + { + if ( me.getKey().length() > 0 ) + { + dos.writeUTF( me.getKey() ); + dos.writeUTF( me.getValue() ); + } + } + } + dos.writeUTF( "" ); + + int size = nodes.size(); + for( int i=0; i < size; i++ ) + { + writeId( dos, nodes.get( i ) ); + } + writeId( dos, -1 ); // stopbyte + } + + public static void sortByStartNode( List ways ) + { + Collections.sort( ways, new Comparator() + { + @Override + public int compare(WayData w1, WayData w2) + { + long d = w1.startNodeIdx - w2.startNodeIdx; + + // for equal start indexes sort by wid + if ( d == 0L ) + { + d = w1.wid - w2.wid; + } + return d == 0 ? 0 : ( d < 0 ? -1 : 1 ); + } + } ); + + } + + @Override + public boolean equals( Object o ) + { + if ( o instanceof WayData ) + { + WayData w = (WayData) o; + return w.wid == wid; + } + return false; + } + + @Override + public int hashCode() + { + return (int)((wid >> 32) ^ wid); + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/WayIterator.java b/brouter-map-creator/src/main/java/btools/mapsplitter/WayIterator.java new file mode 100644 index 0000000..05c5f60 --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/WayIterator.java @@ -0,0 +1,45 @@ +package btools.mapsplitter; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; + +/** + * Iterate over a singe wayfile or a directory + * of waytiles and feed the ways to the callback listener + * + * @author ab + */ +public class WayIterator extends MapCreatorBase +{ + private WayListener listener; + + public WayIterator( WayListener wayListener ) + { + listener = wayListener; + } + + public void processFile(File wayfile) throws Exception + { + System.out.println( "*** WayIterator reading: " + wayfile ); + + listener.wayFileStart( wayfile ); + + DataInputStream di = new DataInputStream( new BufferedInputStream ( new FileInputStream( wayfile ) ) ); + try + { + for(;;) + { + WayData w = new WayData( di ); + listener.nextWay( w ); + } + } + catch( EOFException eof ) + { + di.close(); + } + listener.wayFileEnd( wayfile ); + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapsplitter/WayListener.java b/brouter-map-creator/src/main/java/btools/mapsplitter/WayListener.java new file mode 100644 index 0000000..440359b --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapsplitter/WayListener.java @@ -0,0 +1,17 @@ +package btools.mapsplitter; + +import java.io.File; + +/** + * Callbacklistener for WayIterator + * + * @author ab + */ +public interface WayListener +{ + void wayFileStart( File wayfile ) throws Exception; + + void nextWay( WayData data ) throws Exception; + + void wayFileEnd( File wayfile ) throws Exception; +} diff --git a/brouter-map-creator/src/test/java/btools/mapsplitter/BitCodingTest.java b/brouter-map-creator/src/test/java/btools/mapsplitter/BitCodingTest.java new file mode 100644 index 0000000..e672271 --- /dev/null +++ b/brouter-map-creator/src/test/java/btools/mapsplitter/BitCodingTest.java @@ -0,0 +1,141 @@ +package btools.mapsplitter; + +import java.util.Arrays; +import java.util.Random; + +import org.junit.Assert; +import org.junit.Test; + +import btools.mapdecoder.BitReadBuffer; +import btools.mapdecoder.CharDecoder; + +public class BitCodingTest +{ + // @Test + public void charEncodeDecodeTest() + { + byte[] ab = new byte[4000]; + BitWriteBuffer bwb = new BitWriteBuffer( ab ); + CharEncoder ce = new CharEncoder(); + + for( int pass=1; pass<=3; pass++ ) + { + ce.encodeDictionary( bwb ); + for ( char c = 'a'; c <= 'z'; c++ ) + { + ce.encode( Character.valueOf( c ) ); + } + } + + BitReadBuffer brb = new BitReadBuffer( ab ); + CharDecoder cd = new CharDecoder( brb ); + for ( char c = 'a'; c <= 'z'; c++ ) + { + Character c1 = cd.decode(); + Assert.assertTrue( "char mismatch c=" + c + "c1=" + c1, c == c1.charValue() ); + } + } + + @Test + public void varBitsEncodeDecodeTest() + { + byte[] ab = new byte[4000]; + BitWriteBuffer bwb = new BitWriteBuffer( ab ); + for ( int i = 0; i < 1000; i++ ) + { + bwb.encodeInt( i ); + bwb.encodeLong( i ); + } + BitReadBuffer brb = new BitReadBuffer( ab ); + + for ( int i = 0; i < 1000; i++ ) + { + int value = brb.decodeInt(); + Assert.assertTrue( "int value mismatch i=" + i + "v=" + value, value == i ); + long lvalue = brb.decodeLong(); + Assert.assertTrue( "long value mismatch i=" + i + "v=" + lvalue, value == i ); + } + } + + @Test + public void boundedEncodeDecodeTest() + { + byte[] ab = new byte[581969]; + BitWriteBuffer bwb = new BitWriteBuffer( ab ); + for ( int max = 1; max < 1000; max++ ) + { + for ( int val = 0; val <= max; val++ ) + { + bwb.encodeBounded( max, val ); + } + } + + BitReadBuffer brb = new BitReadBuffer( ab ); + + for ( int max = 1; max < 1000; max++ ) + { + for ( int val = 0; val <= max; val++ ) + { + long valDecoded = brb.decodeBounded( max ); + if ( valDecoded != val ) + { + Assert.fail( "mismatch at max=" + max + " " + valDecoded + "<>" + val ); + } + } + } + } + + @Test + public void sortedLongArrayEncodeDecodeTest() + { + Random rand = new Random(1234); + int size = 20; + long[] values = new long[size]; + for ( int i = 0; i < size; i++ ) + { + values[i] = rand.nextInt() & 0x0fffffff; + } + values[5] = 175384; // force collision + values[8] = 175384; + + values[15] = 275384; // force neighbours + values[18] = 275385; + + encodeDecodeArray( "Test1", values ); + + values = new long[1]; + values[0] = 0x134567890123456L; + encodeDecodeArray( "Test2", values ); + + values = new long[0]; + encodeDecodeArray( "Test3", values ); + + values = new long[100000]; + for ( int i = 0; i < values.length; i++ ) + { + values[i] = (((long)rand.nextInt())&0xffffffffL) << rand.nextInt(26); // 32 + 25 bits + } + encodeDecodeArray( "Test4", values ); + } + + private void encodeDecodeArray( String testName, long[] values ) + { + Arrays.sort( values ); + + byte[] ab = new byte[3000000]; + BitWriteBuffer bwb = new BitWriteBuffer( ab ); + + bwb.encodeSortedArray( values ); + + long[] decodedValues = new BitReadBuffer( ab ).decodeSortedArray(); + + for ( int i = 0; i < values.length; i++ ) + { + if ( values[i] != decodedValues[i] ) + { + Assert.fail( "mismatch at " + testName + " i=" + i + " " + values[i] + "<>" + decodedValues[i] ); + } + } + } + +} diff --git a/brouter-map-creator/src/test/java/btools/mapsplitter/MapsplitterTest.java b/brouter-map-creator/src/test/java/btools/mapsplitter/MapsplitterTest.java new file mode 100644 index 0000000..8dbf0db --- /dev/null +++ b/brouter-map-creator/src/test/java/btools/mapsplitter/MapsplitterTest.java @@ -0,0 +1,37 @@ +package btools.mapsplitter; + +import java.util.Random; +import java.util.HashMap; + +import org.junit.Assert; +import org.junit.Test; +import java.net.URL; +import java.io.File; + +import btools.mapdecoder.TileDecoder; +import btools.mapdecoder.OsmTile; + +public class MapsplitterTest +{ + @Test + public void mapsplitterTest() throws Exception + { + URL mapurl = this.getClass().getResource( "/dreieich.osm.gz" ); + Assert.assertTrue( "test-osm-map dreieich.osm not found", mapurl != null ); + File mapfile = new File(mapurl.getFile()); + File workingDir = mapfile.getParentFile(); + File tmpdir = new File( workingDir, "tmp2" ); + tmpdir.mkdir(); + + // run OsmSplitter + File tiles = new File( tmpdir, "tiles" ); + tiles.mkdir(); + new OsmSplitter().process( tiles, mapfile ); + + // run TileSplitter to split up to level 12 + new TileSplitter().process( tiles ); + + new TileEncoder().process( new File( tiles, "0/0_0.ntl" ) ); + new TileDecoder().process( tiles, null, 12, 2147, 1389 ); + } +} diff --git a/misc/pbfparser/BPbfBlobDecoder2.java b/misc/pbfparser/BPbfBlobDecoder2.java new file mode 100644 index 0000000..67b2be2 --- /dev/null +++ b/misc/pbfparser/BPbfBlobDecoder2.java @@ -0,0 +1,275 @@ +package btools.mapsplitter; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.openstreetmap.osmosis.osmbinary.Fileformat; +import org.openstreetmap.osmosis.osmbinary.Osmformat; +import btools.util.LongList; + +import java.io.IOException; +import java.util.*; +import java.util.logging.Level; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; + +import btools.mapcreator.BPbfFieldDecoder; + +/** + * Converts PBF block data into decoded entities ready to be passed into an Osmosis pipeline. This + * class is designed to be passed into a pool of worker threads to allow multi-threaded decoding. + *

+ * @author Brett Henderson + */ +public class BPbfBlobDecoder2 +{ + private String blobType; + private byte[] rawBlob; + + private OsmParser2 parser; + + /** + * Creates a new instance. + *

+ * @param blobType The type of blob. + * @param rawBlob The raw data of the blob. + * @param listener The listener for receiving decoding results. + */ + public BPbfBlobDecoder2( String blobType, byte[] rawBlob, OsmParser2 parser ) + { + this.blobType = blobType; + this.rawBlob = rawBlob; + this.parser = parser; + } + + public void process() throws Exception + { + if ("OSMHeader".equals(blobType)) + { + processOsmHeader(readBlobContent()); + + } else if ("OSMData".equals(blobType)) + { + processOsmPrimitives(readBlobContent()); + + } else + { + System.out.println("Skipping unrecognised blob type " + blobType); + } + } + + private byte[] readBlobContent() throws IOException + { + Fileformat.Blob blob = Fileformat.Blob.parseFrom(rawBlob); + byte[] blobData; + + if (blob.hasRaw()) + { + blobData = blob.getRaw().toByteArray(); + } else if (blob.hasZlibData()) + { + Inflater inflater = new Inflater(); + inflater.setInput(blob.getZlibData().toByteArray()); + blobData = new byte[blob.getRawSize()]; + try + { + inflater.inflate(blobData); + } catch (DataFormatException e) + { + throw new RuntimeException("Unable to decompress PBF blob.", e); + } + if (!inflater.finished()) + { + throw new RuntimeException("PBF blob contains incomplete compressed data."); + } + } else + { + throw new RuntimeException("PBF blob uses unsupported compression, only raw or zlib may be used."); + } + + return blobData; + } + + private void processOsmHeader( byte[] data ) throws InvalidProtocolBufferException + { + Osmformat.HeaderBlock header = Osmformat.HeaderBlock.parseFrom(data); + + // Build the list of active and unsupported features in the file. + List supportedFeatures = Arrays.asList("OsmSchema-V0.6", "DenseNodes"); + List activeFeatures = new ArrayList(); + List unsupportedFeatures = new ArrayList(); + for (String feature : header.getRequiredFeaturesList()) + { + if (supportedFeatures.contains(feature)) + { + activeFeatures.add(feature); + } else + { + unsupportedFeatures.add(feature); + } + } + + // We can't continue if there are any unsupported features. We wait + // until now so that we can display all unsupported features instead of + // just the first one we encounter. + if (unsupportedFeatures.size() > 0) + { + throw new RuntimeException("PBF file contains unsupported features " + unsupportedFeatures); + } + + } + + private Map buildTags( List keys, List values, BPbfFieldDecoder fieldDecoder ) + { + + Iterator keyIterator = keys.iterator(); + Iterator valueIterator = values.iterator(); + if (keyIterator.hasNext()) + { + Map tags = new HashMap(); + while (keyIterator.hasNext()) + { + String key = fieldDecoder.decodeString(keyIterator.next()); + String value = fieldDecoder.decodeString(valueIterator.next()); + tags.put(key, value); + } + return tags; + } + return null; + } + + private void processNodes( List nodes, BPbfFieldDecoder fieldDecoder ) + { + for (Osmformat.Node node : nodes) + { + Map tags = buildTags(node.getKeysList(), node.getValsList(), fieldDecoder); + + parser.addNode( node.getId(), tags, fieldDecoder.decodeLatitude(node + .getLat()), fieldDecoder.decodeLatitude(node.getLon())); + } + } + + private void processNodes( Osmformat.DenseNodes nodes, BPbfFieldDecoder fieldDecoder ) + { + List idList = nodes.getIdList(); + List latList = nodes.getLatList(); + List lonList = nodes.getLonList(); + + Iterator keysValuesIterator = nodes.getKeysValsList().iterator(); + + long nodeId = 0; + long latitude = 0; + long longitude = 0; + + for (int i = 0; i < idList.size(); i++) + { + // Delta decode node fields. + nodeId += idList.get(i); + latitude += latList.get(i); + longitude += lonList.get(i); + + // Build the tags. The key and value string indexes are sequential + // in the same PBF array. Each set of tags is delimited by an index + // with a value of 0. + Map tags = null; + while (keysValuesIterator.hasNext()) + { + int keyIndex = keysValuesIterator.next(); + if (keyIndex == 0) + { + break; + } + int valueIndex = keysValuesIterator.next(); + + if (tags == null) + { + tags = new HashMap(); + } + + tags.put(fieldDecoder.decodeString(keyIndex), fieldDecoder.decodeString(valueIndex)); + } + + parser.addNode( nodeId, tags, ((double) latitude) / 10000000, ((double) longitude) / 10000000); + } + } + + private void processWays( List ways, BPbfFieldDecoder fieldDecoder ) + { + for (Osmformat.Way way : ways) + { + Map tags = buildTags(way.getKeysList(), way.getValsList(), fieldDecoder); + + // Build up the list of way nodes for the way. The node ids are + // delta encoded meaning that each id is stored as a delta against + // the previous one. + long nodeId = 0; + LongList wayNodes = new LongList( 16 ); + for (long nodeIdOffset : way.getRefsList()) + { + nodeId += nodeIdOffset; + wayNodes.add(nodeId); + } + + parser.addWay( way.getId(), tags, wayNodes ); + } + } + + private ArrayList roles; + + private LongList buildRelationMembers( + List memberIds, List memberRoles, List memberTypes, + BPbfFieldDecoder fieldDecoder ) + { + LongList wayIds = new LongList( 16 ); + roles = new ArrayList(16); + + Iterator memberIdIterator = memberIds.iterator(); + Iterator memberRoleIterator = memberRoles.iterator(); + Iterator memberTypeIterator = memberTypes.iterator(); + + // Build up the list of relation members for the way. The member ids are + // delta encoded meaning that each id is stored as a delta against + // the previous one. + long refId = 0; + while (memberIdIterator.hasNext()) + { + Osmformat.Relation.MemberType memberType = memberTypeIterator.next(); + refId += memberIdIterator.next(); + + String role = fieldDecoder.decodeString( memberRoleIterator.next() ); + + if ( memberType == Osmformat.Relation.MemberType.WAY ) // currently just waymembers + { + wayIds.add( refId ); + roles.add( role ); + } + } + return wayIds; + } + + private void processRelations( List relations, BPbfFieldDecoder fieldDecoder ) + { + for (Osmformat.Relation relation : relations) + { + Map tags = buildTags(relation.getKeysList(), relation.getValsList(), fieldDecoder); + + LongList wayIds = buildRelationMembers( relation.getMemidsList(), relation.getRolesSidList(), + relation.getTypesList(), fieldDecoder); + + parser.addRelation( relation.getId(), tags, wayIds, roles ); + } + } + + private void processOsmPrimitives( byte[] data ) throws InvalidProtocolBufferException + { + Osmformat.PrimitiveBlock block = Osmformat.PrimitiveBlock.parseFrom(data); + BPbfFieldDecoder fieldDecoder = new BPbfFieldDecoder(block); + + for (Osmformat.PrimitiveGroup primitiveGroup : block.getPrimitivegroupList()) + { + processNodes(primitiveGroup.getDense(), fieldDecoder); + processNodes(primitiveGroup.getNodesList(), fieldDecoder); + processWays(primitiveGroup.getWaysList(), fieldDecoder); + processRelations(primitiveGroup.getRelationsList(), fieldDecoder); + } + } + +} diff --git a/misc/pbfparser/OsmParser2.java b/misc/pbfparser/OsmParser2.java new file mode 100644 index 0000000..21cdffa --- /dev/null +++ b/misc/pbfparser/OsmParser2.java @@ -0,0 +1,111 @@ +package btools.mapsplitter; + +import java.io.*; +import java.util.*; +import java.util.zip.*; +import btools.util.*; + +import org.openstreetmap.osmosis.osmbinary.Fileformat; + +/** + * Parser for OSM data + * + * @author ab + */ +public class OsmParser2 extends MapCreatorBase +{ + private BufferedReader _br; + + private NodeListener nListener; + private WayListener wListener; + private RelationListener rListener; + + public void readMap( File mapFile, + NodeListener nListener, + WayListener wListener, + RelationListener rListener ) throws Exception + { + + this.nListener = nListener; + this.wListener = wListener; + this.rListener = rListener; + + System.out.println( "*** PBF Parsing (2): " + mapFile ); + + // once more for testing + int rawBlobCount = 0; + DataInputStream dis = new DataInputStream( new BufferedInputStream ( new FileInputStream( mapFile ) ) ); + for(;;) + { + int headerLength; + try + { + headerLength = dis.readInt(); + } + catch (EOFException e) + { + break; + } + + byte[] headerBuffer = new byte[headerLength]; + dis.readFully(headerBuffer); + Fileformat.BlobHeader blobHeader = Fileformat.BlobHeader.parseFrom(headerBuffer); + + byte[] blobData = new byte[blobHeader.getDatasize()]; + dis.readFully(blobData); + + new BPbfBlobDecoder2( blobHeader.getType(), blobData, this ).process(); + + rawBlobCount++; + } + dis.close(); + System.out.println( "read raw blobs: " + rawBlobCount ); + } + + + + public void addNode( long nid, Map tags, double lat, double lon ) + { + NodeData n = new NodeData( nid, lon, lat ); + n.setTags( (HashMap)tags ); + try + { + nListener.nextNode( n ); + } + catch( Exception e ) + { + throw new RuntimeException( "error writing node: " + e ); + } + } + + public void addWay( long wid, Map tags, LongList nodes ) + { + WayData w = new WayData( wid, nodes ); + w.setTags( (HashMap)tags ); + + try + { + wListener.nextWay( w ); + } + catch( Exception e ) + { + throw new RuntimeException( "error writing way: " + e ); + } + } + + public void addRelation( long rid, Map tags, LongList wayIds, List roles ) + { + RelationData r = new RelationData( rid, wayIds, roles ); + r.setTags( (HashMap)tags ); + + try + { + rListener.nextRelation( r ); + } + catch( Exception e ) + { + throw new RuntimeException( "error writing relation: " + e ); + } + } + +}