From 98b3f62efb22599d2a9945425beb9d958fea8516 Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Sat, 28 Sep 2019 12:59:29 +0200 Subject: [PATCH] pre-process speed: fast binary also for 90m srtm --- .../java/btools/mapcreator/RasterCoder.java | 6 +- .../main/java/btools/mapcreator/SrtmData.java | 66 +++++++++++++ .../java/btools/mapcreator/SrtmRaster.java | 7 +- .../java/btools/util/BitCoderContext.java | 4 +- .../btools/util/MixCoderDataInputStream.java | 96 +++++++++++++++---- 5 files changed, 153 insertions(+), 26 deletions(-) diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/RasterCoder.java b/brouter-map-creator/src/main/java/btools/mapcreator/RasterCoder.java index b7d3495..7a8473a 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/RasterCoder.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/RasterCoder.java @@ -47,10 +47,10 @@ public class RasterCoder _decodeRaster(raster, is); - raster.usingWeights = true; + raster.usingWeights = raster.ncols > 6001; long t1 = System.currentTimeMillis(); - System.out.println("finished decoding in " + (t1 - t0) + " ms"); + System.out.println("finished decoding in " + (t1 - t0) + " ms ncols=" + raster.ncols + " nrows=" + raster.nrows ); return raster; } @@ -102,7 +102,7 @@ public class RasterCoder // remap nodata int v30 = code == -1 ? Short.MIN_VALUE : ( code < 0 ? code + 1 : code ); - if ( v30 > -32766 ) + if ( raster.usingWeights && v30 > -32766 ) { v30 *= 2; } diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/SrtmData.java b/brouter-map-creator/src/main/java/btools/mapcreator/SrtmData.java index 026db04..039fadb 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/SrtmData.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/SrtmData.java @@ -11,11 +11,14 @@ package btools.mapcreator; */ import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.util.StringTokenizer; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -126,4 +129,67 @@ public class SrtmData } br.close(); } + + public static void main( String[] args ) throws Exception + { + String fromDir = args[0]; + String toDir = args[1]; + + File[] files = new File( fromDir ).listFiles(); + for( File f : files ) + { + if ( !f.getName().endsWith( ".zip" ) ) + { + continue; + } + System.out.println( "*** reading: " + f ); + long t0 = System.currentTimeMillis(); + SrtmRaster raster = new SrtmData( f ).getRaster(); + long t1 = System.currentTimeMillis(); + String name = f.getName(); + + long zipTime = t1-t0; + + File fbef = new File( new File( toDir ), name.substring( 0, name.length()-3 ) + "bef" ); + System.out.println( "recoding: " + f + " to " + fbef ); + OutputStream osbef = new BufferedOutputStream( new FileOutputStream( fbef ) ); + new RasterCoder().encodeRaster( raster, osbef ); + osbef.close(); + + System.out.println( "*** re-reading: " + fbef ); + + long t2 = System.currentTimeMillis(); + InputStream isc = new BufferedInputStream( new FileInputStream( fbef ) ); + SrtmRaster raster2 = new RasterCoder().decodeRaster( isc ); + isc.close(); + long t3 = System.currentTimeMillis(); + + long befTime = t3-t2; + + System.out.println( "*** zip-time: " + zipTime + "*** bef-time: " + befTime ); + + String s1 = raster.toString(); + String s2 = raster2.toString(); + + if ( !s1.equals( s2 ) ) + { + throw new IllegalArgumentException( "missmatch: " + s1 + "<--->" + s2 ); + } + + int cols = raster.ncols; + int rows = raster.nrows; + for( int c = 0; c < cols; c++ ) + { + for( int r = 0; r < rows; r++ ) + { + int idx = r * cols + c; + + if ( raster.eval_array[idx] != raster2.eval_array[idx] ) + { + throw new IllegalArgumentException( "missmatch: at " + c + "," + r + ": " + raster.eval_array[idx] + "<--->" + raster2.eval_array[idx] ); + } + } + } + } + } } diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/SrtmRaster.java b/brouter-map-creator/src/main/java/btools/mapcreator/SrtmRaster.java index 1fb0d6b..60ef81f 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/SrtmRaster.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/SrtmRaster.java @@ -285,5 +285,10 @@ public class SrtmRaster } return shiftWeights; } - + + @Override + public String toString() + { + return ncols + "," + nrows + "," + halfcol + "," + xllcorner + "," + yllcorner + "," + cellsize + "," + noDataValue + "," + usingWeights; + } } diff --git a/brouter-util/src/main/java/btools/util/BitCoderContext.java b/brouter-util/src/main/java/btools/util/BitCoderContext.java index 680f014..2ded5d5 100644 --- a/brouter-util/src/main/java/btools/util/BitCoderContext.java +++ b/brouter-util/src/main/java/btools/util/BitCoderContext.java @@ -9,8 +9,8 @@ public class BitCoderContext private int bits; // bits left in buffer private int b; // buffer word - private static final int[] vl_values = new int[4096]; - private static final int[] vl_length = new int[4096]; + public static final int[] vl_values = new int[4096]; + public static final int[] vl_length = new int[4096]; private static final int[] vc_values = new int[4096]; private static final int[] vc_length = new int[4096]; diff --git a/brouter-util/src/main/java/btools/util/MixCoderDataInputStream.java b/brouter-util/src/main/java/btools/util/MixCoderDataInputStream.java index 8e86904..ea2f705 100644 --- a/brouter-util/src/main/java/btools/util/MixCoderDataInputStream.java +++ b/brouter-util/src/main/java/btools/util/MixCoderDataInputStream.java @@ -16,8 +16,11 @@ public final class MixCoderDataInputStream extends DataInputStream private int repCount; private int diffshift; - private int bm = 0x100; - private int b; + private int bits; // bits left in buffer + private int b; // buffer word + + private static final int[] vl_values = BitCoderContext.vl_values; + private static final int[] vl_length = BitCoderContext.vl_length; public MixCoderDataInputStream( InputStream is ) { @@ -40,46 +43,99 @@ public final class MixCoderDataInputStream extends DataInputStream public final boolean decodeBit() throws IOException { - if ( bm == 0x100 ) - { - bm = 1; - b = readByte(); - } - boolean value = ( ( b & bm ) != 0 ); - bm <<= 1; + fillBuffer(); + boolean value = ( ( b & 1 ) != 0 ); + b >>>= 1; + bits--; return value; } - public final int decodeVarBits() throws IOException + public final int decodeVarBits2() throws IOException { int range = 0; - int value = 0; while (!decodeBit()) { - value += range + 1; range = 2 * range + 1; } - return value + decodeBounded( range ); + return range + decodeBounded( range ); } - + /** + * decode an integer in the range 0..max (inclusive). + * @see #encodeBounded + */ public final int decodeBounded( int max ) throws IOException { int value = 0; int im = 1; // integer mask while (( value | im ) <= max) { - if ( bm == 0x100 ) + if ( decodeBit() ) { - bm = 1; - b = readByte(); - } - if ( ( b & bm ) != 0 ) value |= im; - bm <<= 1; + } im <<= 1; } return value; } + + /** + * @see #encodeVarBits + */ + + public final int decodeVarBits() throws IOException + { + fillBuffer(); + int b12 = b & 0xfff; + int len = vl_length[b12]; + if ( len <= 12 ) + { + b >>>= len; + bits -= len; + return vl_values[b12]; // full value lookup + } + if ( len <= 23 ) // // only length lookup + { + int len2 = len >> 1; + b >>>= (len2+1); + int mask = 0xffffffff >>> ( 32 - len2 ); + mask += b & mask; + b >>>= len2; + bits -= len; + return mask; + } + if ( (b & 0xffffff) != 0 ) + { + // here we just know len in [25..47] + // ( fillBuffer guarantees only 24 bits! ) + b >>>= 12; + int len3 = 1 + (vl_length[b & 0xfff]>>1); + b >>>= len3; + int len2 = 11 + len3; + bits -= len2+1; + fillBuffer(); + int mask = 0xffffffff >>> ( 32 - len2 ); + mask += b & mask; + b >>>= len2; + bits -= len2; + return mask; + } + return decodeVarBits2(); // no chance, use the slow one + } + + private void fillBuffer() throws IOException + { + while (bits < 24) + { + int nextByte = read(); + + if ( nextByte != -1 ) + { + b |= (nextByte & 0xff) << bits; + } + bits += 8; + } + } + }