rd5 diff progress

brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java

@@ -0,0 +1,119 @@
+/**
+ * Proof of concept for delta rd5's
+ *
+ * @author ab
+ */
+package btools.mapaccess;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
+
+final public class Rd5DiffManager
+{
+  public static void main( String[] args ) throws Exception
+  {
+    calcDiffs( new File( args[0] ),new File( args[1] ) );
+  }
+
+  /**
+   * Compute diffs for all RD5 files
+   */
+  public static void calcDiffs( File oldDir, File newDir ) throws Exception
+  {
+    File oldDiffDir = new File( oldDir, "diff" );
+    File newDiffDir = new File( newDir, "diff" );
+
+    File[] filesNew = newDir.listFiles();
+    
+    for( File fn : filesNew )
+    {
+      String name = fn.getName();
+      if ( !name.endsWith( ".rd5" ) )
+      {
+        continue;
+      }
+      if ( fn.length() < 1024*1024 )
+      {
+        continue; // exclude very small files from diffing
+      }
+      String basename = name.substring( 0, name.length() - 4 );
+      File fo = new File( oldDir, name );
+      if ( !fo.isFile() )
+      {
+        continue;
+      }
+      
+      // calculate MD5 of old file
+      String md5 = getMD5( fo );
+      
+      System.out.println( "name=" + name + " md5=" + md5 );
+      
+      File specificNewDiffs = new File( newDiffDir, basename );
+      specificNewDiffs.mkdirs();
+      
+      String diffFileName = md5 + ".rd5diff";
+      
+      File diffFile = new File( specificNewDiffs, diffFileName );
+      
+      // calc the new diff
+      Rd5DiffTool.diff2files( fo, fn, diffFile  );
+      
+      // ... and add that to old diff files
+      File specificOldDiffs = new File( oldDiffDir, basename );
+      if ( specificOldDiffs.isDirectory() )
+      {
+        File[] oldDiffs = specificOldDiffs.listFiles();
+        for( File od : oldDiffs )
+        {
+          if ( !od.getName().endsWith( ".rd5diff" ) )
+          {
+            continue;
+          }
+          if ( System.currentTimeMillis() - od.lastModified() > 31*86400000L )
+          {
+            continue; // limit diff history to 31 days
+          }
+          
+          File updatedDiff = new File( specificNewDiffs, od.getName() );
+          
+          Rd5DiffTool.addDeltas( od, diffFile, updatedDiff  );
+          updatedDiff.setLastModified( od.lastModified() );
+        }
+      }
+    }
+  }
+
+  public static String getMD5( File f ) throws Exception
+  {
+    MessageDigest md = MessageDigest.getInstance("MD5");
+    BufferedInputStream bis = new BufferedInputStream( new FileInputStream( f ) );
+    DigestInputStream dis = new DigestInputStream(bis, md);
+    byte[] buf = new byte[8192];
+    for(;;)
+    {
+      int len = dis.read( buf );
+      if ( len <= 0 )
+      {
+        break;
+      }
+    }
+    dis.close();
+    byte[] bytes = md.digest();
+
+    StringBuilder sb = new StringBuilder();
+    for (int j = 0; j < bytes.length; j++)
+    {
+      int v = bytes[j] & 0xff;
+      sb.append( hexChar( v >>> 4 ) ).append( hexChar( v & 0xf ) );
+    }
+    return sb.toString();
+  }
+  
+  private static char hexChar( int v )
+  {
+    return (char) ( v > 9 ? 'a' + (v-10) : '0' + v );
+  }
+}

brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java

@@ -138,6 +138,8 @@ final public class Rd5DiffTool
     long[] fileIndex1 = readFileIndex( dis1, null );
     long[] fileIndex2 = readFileIndex( dis2, dos );
 
+    long t0 = System.currentTimeMillis();
+
     try
     {
       DataBuffers dataBuffers = new DataBuffers();
@@ -181,15 +183,15 @@ final public class Rd5DiffTool
              dos.writeInt( len );
              dos.write( abBuf1, 0, len );
 
-                 // do some consistemcy checks on the encoding
+             bytesDiff += len;
+             nodesDiff += mc.getSize();
+             diffedTiles++;
+
+/*                 // do some consistemcy checks on the encoding
 
                  byte[] bytes = new byte[len];
                  System.arraycopy( abBuf1, 0, bytes, 0, len );
                  
-                 bytesDiff += len;
-                 nodesDiff += mc.getSize();
-                 diffedTiles++;
-                 
                  // cross-check the encoding: decode again
                  MicroCache mcCheck = new MicroCache2( new StatCoderContext( bytes ), new DataBuffers( null ), 0, 0, 32, null, null );
                  
@@ -220,6 +222,7 @@ final public class Rd5DiffTool
                      throw new IllegalArgumentException( "decoded data mismatch at i=" + i );
                    }
                  }
+             */
           }
         }
       }
@@ -233,7 +236,8 @@ final public class Rd5DiffTool
         }
         dos.write( abBuf1, 0, len );
       }
-      System.out.println( "nodesDiff=" + nodesDiff + " bytesDiff=" + bytesDiff + " diffedTiles=" + diffedTiles );
+      long t1 = System.currentTimeMillis();
+      System.out.println( "nodesDiff=" + nodesDiff + " bytesDiff=" + bytesDiff + " diffedTiles=" + diffedTiles + " took " + (t1-t0) + "ms" );
     }
     finally
     {
@@ -416,6 +420,8 @@ final public class Rd5DiffTool
     long[] fileIndex1 = readFileIndex( dis1, null );
     long[] fileIndex2 = readFileIndex( dis2, dos );
 
+    long t0 = System.currentTimeMillis();
+
     try
     {
       DataBuffers dataBuffers = new DataBuffers();
@@ -475,6 +481,8 @@ final public class Rd5DiffTool
         }
         dos.write( abBuf1, 0, len );
       }
+      long t1 = System.currentTimeMillis();
+      System.out.println( "adding diffs took " + (t1-t0) + "ms" );
     }
     finally
     {