rd5 diff progress

brouter-codec/src/main/java/btools/codec/TagValueCoder.java

@@ -21,6 +21,7 @@ public final class TagValueCoder
   private Object tree;
   private BitCoderContext bc;
   private int pass;
+  private int nextTagValueSetId;
 
   public void encodeTagValueSet( byte[] data )
   {
@@ -28,7 +29,7 @@ public final class TagValueCoder
     {
       return;
     }
-    TagValueSet tvsProbe = new TagValueSet();
+    TagValueSet tvsProbe = new TagValueSet(nextTagValueSetId);
     tvsProbe.data = data;
     TagValueSet tvs = identityMap.get( tvsProbe );
     if ( pass == 3 )
@@ -40,6 +41,7 @@ public final class TagValueCoder
       if ( tvs == null )
       {
         tvs = tvsProbe;
+        nextTagValueSetId++;
         identityMap.put( tvs, tvs );
       }
       tvs.frequency++;
@@ -64,14 +66,14 @@ public final class TagValueCoder
     {
       if ( identityMap.size() == 0 )
       {
-        TagValueSet dummy = new TagValueSet();
+        TagValueSet dummy = new TagValueSet(nextTagValueSetId++);
         identityMap.put( dummy, dummy );
       }
       PriorityQueue<TagValueSet> queue = new PriorityQueue<TagValueSet>(2*identityMap.size(), new TagValueSet.FrequencyComparator());
       queue.addAll(identityMap.values());
       while (queue.size() > 1)
       {
-        TagValueSet node = new TagValueSet();
+        TagValueSet node = new TagValueSet(nextTagValueSetId++);
         node.child1 = queue.poll();
         node.child2 = queue.poll();
         node.frequency = node.child1.frequency + node.child2.frequency;
@@ -178,6 +180,12 @@ public final class TagValueCoder
     public int range;
     public TagValueSet child1;
     public TagValueSet child2;
+    private int id; // serial number to make the comparator well defined in case of equal frequencies
+
+    public TagValueSet( int id )
+    {
+      this.id = id;
+    }
 
     public void encode( BitCoderContext bc, int range, int code )
     {
@@ -267,23 +275,15 @@ public final class TagValueCoder
         if ( tvs1.frequency > tvs2.frequency )
           return 1;
 
-        // to avoid ordering instability, decide on the data
+        // to avoid ordering instability, decide on the id if frequency is equal
-        // if frequency is equal
+        if ( tvs1.id < tvs2.id )
-        int l1 = tvs1.data == null ? 0 : tvs1.data.length;
-        int l2 = tvs2.data == null ? 0 : tvs2.data.length;
-
-        if ( l1 < l2 )
           return -1;
-        if ( l1 > l2 )
+        if ( tvs1.id > tvs2.id )
           return 1;
-        for( int i=0; i<l1; i++ )
+
+        if ( tvs1 != tvs2 )
         {
-          byte b1 = tvs1.data[i];
+          throw new RuntimeException( "identity corruption!" );
-          byte b2 = tvs2.data[i];
-          if ( b1 < b2 )
-            return -1;
-          if ( b1 > b2 )
-            return 1;
         }
         return 0;
       }

brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java

@@ -25,6 +25,12 @@ final public class Rd5DiffTool implements ProgressListener
 {
   public static void main( String[] args ) throws Exception
   {
+    if ( args.length == 2 )
+    {
+      reEncode( new File( args[0] ),new File( args[1] ) );
+      return;
+    }
+
     if ( args[1].endsWith( ".rd5diff" ) )
     {
       if ( args[0].endsWith( ".rd5diff" ) )
@@ -625,4 +631,80 @@ final public class Rd5DiffTool implements ProgressListener
       }
     }
   }
+
+
+  public static void reEncode( File f1, File outFile ) throws Exception
+  {
+    byte[] abBuf1 = new byte[10 * 1024 * 1024];
+
+    DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
+    DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
+
+    // copy header to outfile
+    long[] fileIndex1 = readFileIndex( dis1, dos );
+
+    long t0 = System.currentTimeMillis();
+
+    try
+    {
+      DataBuffers dataBuffers = new DataBuffers();
+      for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ )
+      {
+         boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx );
+
+         int[] posIdx1 = hasData1 ? readPosIndex( dis1, dos ) : null;
+
+         for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ )
+         {
+           byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false );
+
+           if ( ab1 == null ) continue;
+
+           MicroCache mc1 = createMicroCache( ab1, dataBuffers );
+
+           int len = mc1.encodeMicroCache( abBuf1 );
+
+           dos.write( abBuf1, 0, len );
+           dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 );
+        }
+      }
+      // write any remaining data to the output file
+      for(;;)
+      {
+        int len = dis1.read( abBuf1 );
+        if (len < 0)
+        {
+          break;
+        }
+        dos.write( abBuf1, 0, len );
+      }
+      long t1 = System.currentTimeMillis();
+      System.out.println( "re-encoding took " + (t1-t0) + "ms" );
+    }
+    finally
+    {
+      if ( dis1 != null )
+      {
+        try
+        {
+          dis1.close();
+        }
+        catch (Exception ee)
+        {
+        }
+      }
+      if ( dos != null )
+      {
+        try
+        {
+          dos.close();
+        }
+        catch (Exception ee)
+        {
+        }
+      }
+    }
+  }
+
+
 }