diff --git a/brouter-codec/src/main/java/btools/codec/TagValueCoder.java b/brouter-codec/src/main/java/btools/codec/TagValueCoder.java index 77c6bac..72b61c7 100644 --- a/brouter-codec/src/main/java/btools/codec/TagValueCoder.java +++ b/brouter-codec/src/main/java/btools/codec/TagValueCoder.java @@ -21,6 +21,7 @@ public final class TagValueCoder private Object tree; private BitCoderContext bc; private int pass; + private int nextTagValueSetId; public void encodeTagValueSet( byte[] data ) { @@ -28,7 +29,7 @@ public final class TagValueCoder { return; } - TagValueSet tvsProbe = new TagValueSet(); + TagValueSet tvsProbe = new TagValueSet(nextTagValueSetId); tvsProbe.data = data; TagValueSet tvs = identityMap.get( tvsProbe ); if ( pass == 3 ) @@ -40,6 +41,7 @@ public final class TagValueCoder if ( tvs == null ) { tvs = tvsProbe; + nextTagValueSetId++; identityMap.put( tvs, tvs ); } tvs.frequency++; @@ -64,14 +66,14 @@ public final class TagValueCoder { if ( identityMap.size() == 0 ) { - TagValueSet dummy = new TagValueSet(); + TagValueSet dummy = new TagValueSet(nextTagValueSetId++); identityMap.put( dummy, dummy ); } PriorityQueue queue = new PriorityQueue(2*identityMap.size(), new TagValueSet.FrequencyComparator()); queue.addAll(identityMap.values()); while (queue.size() > 1) { - TagValueSet node = new TagValueSet(); + TagValueSet node = new TagValueSet(nextTagValueSetId++); node.child1 = queue.poll(); node.child2 = queue.poll(); node.frequency = node.child1.frequency + node.child2.frequency; @@ -178,6 +180,12 @@ public final class TagValueCoder public int range; public TagValueSet child1; public TagValueSet child2; + private int id; // serial number to make the comparator well defined in case of equal frequencies + + public TagValueSet( int id ) + { + this.id = id; + } public void encode( BitCoderContext bc, int range, int code ) { @@ -267,23 +275,15 @@ public final class TagValueCoder if ( tvs1.frequency > tvs2.frequency ) return 1; - // to avoid ordering instability, decide on the data - // if frequency is equal - int l1 = tvs1.data == null ? 0 : tvs1.data.length; - int l2 = tvs2.data == null ? 0 : tvs2.data.length; - - if ( l1 < l2 ) + // to avoid ordering instability, decide on the id if frequency is equal + if ( tvs1.id < tvs2.id ) return -1; - if ( l1 > l2 ) + if ( tvs1.id > tvs2.id ) return 1; - for( int i=0; i b2 ) - return 1; + throw new RuntimeException( "identity corruption!" ); } return 0; } diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java index ccf3367..1f796f7 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java @@ -25,6 +25,12 @@ final public class Rd5DiffTool implements ProgressListener { public static void main( String[] args ) throws Exception { + if ( args.length == 2 ) + { + reEncode( new File( args[0] ),new File( args[1] ) ); + return; + } + if ( args[1].endsWith( ".rd5diff" ) ) { if ( args[0].endsWith( ".rd5diff" ) ) @@ -625,4 +631,80 @@ final public class Rd5DiffTool implements ProgressListener } } } + + + public static void reEncode( File f1, File outFile ) throws Exception + { + byte[] abBuf1 = new byte[10 * 1024 * 1024]; + + DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) ); + DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) ); + + // copy header to outfile + long[] fileIndex1 = readFileIndex( dis1, dos ); + + long t0 = System.currentTimeMillis(); + + try + { + DataBuffers dataBuffers = new DataBuffers(); + for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ ) + { + boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx ); + + int[] posIdx1 = hasData1 ? readPosIndex( dis1, dos ) : null; + + for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ ) + { + byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false ); + + if ( ab1 == null ) continue; + + MicroCache mc1 = createMicroCache( ab1, dataBuffers ); + + int len = mc1.encodeMicroCache( abBuf1 ); + + dos.write( abBuf1, 0, len ); + dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 ); + } + } + // write any remaining data to the output file + for(;;) + { + int len = dis1.read( abBuf1 ); + if (len < 0) + { + break; + } + dos.write( abBuf1, 0, len ); + } + long t1 = System.currentTimeMillis(); + System.out.println( "re-encoding took " + (t1-t0) + "ms" ); + } + finally + { + if ( dis1 != null ) + { + try + { + dis1.close(); + } + catch (Exception ee) + { + } + } + if ( dos != null ) + { + try + { + dos.close(); + } + catch (Exception ee) + { + } + } + } + } + + }