pre-process speed: fast binary also for 90m srtm

brouter-map-creator/src/main/java/btools/mapcreator/RasterCoder.java

@@ -47,10 +47,10 @@ public class RasterCoder
 
     _decodeRaster(raster, is);
     
-    raster.usingWeights = true;
+    raster.usingWeights = raster.ncols > 6001;
 
     long t1 = System.currentTimeMillis();
-    System.out.println("finished decoding in " + (t1 - t0) + " ms");
+    System.out.println("finished decoding in " + (t1 - t0) + " ms ncols=" + raster.ncols + " nrows=" + raster.nrows );
     return raster;
   }
 
@@ -102,7 +102,7 @@ public class RasterCoder
         
         // remap nodata
         int v30 = code == -1 ? Short.MIN_VALUE : ( code < 0 ? code + 1 : code ); 
-        if ( v30 > -32766 )
+        if ( raster.usingWeights && v30 > -32766 )
         {
           v30 *= 2;
         } 

brouter-map-creator/src/main/java/btools/mapcreator/SrtmData.java

@@ -11,11 +11,14 @@ package btools.mapcreator;
  */
 
 import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.OutputStream;
 import java.util.StringTokenizer;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
@@ -126,4 +129,67 @@ public class SrtmData
     }
     br.close();
   }
+
+  public static void main( String[] args ) throws Exception
+  {
+    String fromDir = args[0];
+    String toDir = args[1];
+    
+    File[] files = new File( fromDir ).listFiles();
+    for( File f : files )
+    {
+      if ( !f.getName().endsWith( ".zip" ) )
+      {
+        continue;
+      }
+      System.out.println( "*** reading: " + f );
+      long t0 = System.currentTimeMillis();
+      SrtmRaster raster = new SrtmData( f ).getRaster();
+      long t1 = System.currentTimeMillis();
+      String name = f.getName();
+      
+      long zipTime = t1-t0;
+      
+      File fbef = new File( new File( toDir ), name.substring( 0, name.length()-3 ) + "bef" );
+      System.out.println( "recoding: " + f + " to " + fbef );
+      OutputStream osbef = new BufferedOutputStream( new FileOutputStream( fbef ) );
+      new RasterCoder().encodeRaster( raster, osbef );
+      osbef.close();
+
+      System.out.println( "*** re-reading: " + fbef );
+      
+      long t2 = System.currentTimeMillis();
+      InputStream isc = new BufferedInputStream( new FileInputStream( fbef ) );
+      SrtmRaster raster2 = new RasterCoder().decodeRaster( isc );
+      isc.close();
+      long t3 = System.currentTimeMillis();
+
+      long befTime = t3-t2;
+
+      System.out.println( "*** zip-time: " + zipTime + "*** bef-time: " + befTime );
+      
+      String s1 = raster.toString();
+      String s2 = raster2.toString();
+      
+      if ( !s1.equals( s2 ) )
+      {
+        throw new IllegalArgumentException( "missmatch: " + s1 + "<--->" + s2 );
+      }
+      
+      int cols = raster.ncols;
+      int rows = raster.nrows;
+      for( int c = 0; c < cols; c++ )
+      {
+        for( int r = 0; r < rows; r++ )
+        {
+          int idx = r * cols + c;         
+          
+          if ( raster.eval_array[idx] != raster2.eval_array[idx] )
+          {
+            throw new IllegalArgumentException( "missmatch: at " + c + "," + r + ": " + raster.eval_array[idx] + "<--->" + raster2.eval_array[idx] );
+          }
+        }
+      }
+    }
+  }
 }

brouter-map-creator/src/main/java/btools/mapcreator/SrtmRaster.java

@@ -285,5 +285,10 @@ public class SrtmRaster
     }
     return shiftWeights;
   }
-
+  
+  @Override
+  public String toString()
+  {
+    return ncols + "," + nrows + "," + halfcol + "," + xllcorner + "," + yllcorner + "," + cellsize + "," + noDataValue + "," + usingWeights;
+  }
 }

brouter-util/src/main/java/btools/util/BitCoderContext.java

@@ -9,8 +9,8 @@ public class BitCoderContext
   private int bits; // bits left in buffer
   private int b; // buffer word
 
-  private static final int[] vl_values = new int[4096];
-  private static final int[] vl_length = new int[4096];
+  public static final int[] vl_values = new int[4096];
+  public static final int[] vl_length = new int[4096];
 
   private static final int[] vc_values = new int[4096];
   private static final int[] vc_length = new int[4096];

brouter-util/src/main/java/btools/util/MixCoderDataInputStream.java

@@ -16,8 +16,11 @@ public final class MixCoderDataInputStream extends DataInputStream
   private int repCount;
   private int diffshift;
 
-  private int bm = 0x100;
-  private int b;
+  private int bits; // bits left in buffer
+  private int b; // buffer word
+
+  private static final int[] vl_values = BitCoderContext.vl_values;
+  private static final int[] vl_length = BitCoderContext.vl_length;
 
   public MixCoderDataInputStream( InputStream is )
   {
@@ -40,46 +43,99 @@ public final class MixCoderDataInputStream extends DataInputStream
 
   public final boolean decodeBit() throws IOException
   {
-    if ( bm == 0x100 )
-    {
-      bm = 1;
-      b = readByte();
-    }
-    boolean value = ( ( b & bm ) != 0 );
-    bm <<= 1;
+    fillBuffer();
+    boolean value = ( ( b & 1 ) != 0 );
+    b >>>= 1;
+    bits--;
     return value;
   }
 
-  public final int decodeVarBits() throws IOException
+  public final int decodeVarBits2() throws IOException
   {
     int range = 0;
-    int value = 0;
     while (!decodeBit())
     {
-      value += range + 1;
       range = 2 * range + 1;
     }
-    return value + decodeBounded( range );
+    return range + decodeBounded( range );
   }
 
-
+  /**
+   * decode an integer in the range 0..max (inclusive).
+   * @see #encodeBounded
+   */
   public final int decodeBounded( int max ) throws IOException
   {
     int value = 0;
     int im = 1; // integer mask
     while (( value | im ) <= max)
     {
-      if ( bm == 0x100 )
+      if ( decodeBit() )
       {
-        bm = 1;
-        b = readByte();
-      }
-      if ( ( b & bm ) != 0 )
         value |= im;
-      bm <<= 1;
+      }
       im <<= 1;
     }
     return value;
   }
 
+
+  /**
+   * @see #encodeVarBits
+   */
+
+  public final int decodeVarBits() throws IOException
+  {
+    fillBuffer();
+    int b12 = b & 0xfff;
+    int len = vl_length[b12];
+    if ( len <= 12 )
+    {
+      b >>>= len;
+      bits -= len;
+      return vl_values[b12]; // full value lookup
+    }
+    if ( len <= 23 ) // // only length lookup
+    {
+      int len2 = len >> 1;
+      b >>>= (len2+1);
+      int mask = 0xffffffff >>> ( 32 - len2 );
+      mask += b & mask;
+      b >>>= len2;
+      bits -= len;
+      return mask;
+    }
+    if ( (b & 0xffffff) != 0 )
+    {
+      // here we just know len in [25..47]
+      // ( fillBuffer guarantees only 24 bits! )
+      b >>>= 12;
+      int len3 = 1 + (vl_length[b & 0xfff]>>1);
+      b >>>= len3;
+      int len2 = 11 + len3;
+      bits -= len2+1;
+      fillBuffer();
+      int mask = 0xffffffff >>> ( 32 - len2 );
+      mask += b & mask;
+      b >>>= len2;
+      bits -= len2;
+      return mask;
+    }
+    return decodeVarBits2(); // no chance, use the slow one
+  }
+
+  private void fillBuffer() throws IOException
+  {
+    while (bits < 24)
+    {
+      int nextByte = read();
+      
+      if ( nextByte != -1 )
+      {
+        b |= (nextByte & 0xff) << bits;
+      }
+      bits += 8;
+    }
+  }
+
 }