From 91c463302eccc47b9d2ada1104abaae948bd90aa Mon Sep 17 00:00:00 2001 From: Arndt Date: Sun, 5 Apr 2015 09:24:46 +0200 Subject: [PATCH] srtm30 utils --- .../btools/util/MixCoderDataInputStream.java | 57 ++++++++++++ .../btools/util/MixCoderDataOutputStream.java | 79 ++++++++++++++++ .../java/btools/util/ReducedMedianFilter.java | 92 +++++++++++++++++++ .../test/java/btools/util/MixCoderTest.java | 63 +++++++++++++ .../btools/util/ReducedMedianFilterTest.java | 47 ++++++++++ 5 files changed, 338 insertions(+) create mode 100644 brouter-util/src/main/java/btools/util/MixCoderDataInputStream.java create mode 100644 brouter-util/src/main/java/btools/util/MixCoderDataOutputStream.java create mode 100644 brouter-util/src/main/java/btools/util/ReducedMedianFilter.java create mode 100644 brouter-util/src/test/java/btools/util/MixCoderTest.java create mode 100644 brouter-util/src/test/java/btools/util/ReducedMedianFilterTest.java diff --git a/brouter-util/src/main/java/btools/util/MixCoderDataInputStream.java b/brouter-util/src/main/java/btools/util/MixCoderDataInputStream.java new file mode 100644 index 0000000..a7d911a --- /dev/null +++ b/brouter-util/src/main/java/btools/util/MixCoderDataInputStream.java @@ -0,0 +1,57 @@ +/** + * DataInputStream for decoding fast-compact encoded number sequences + * + * @author ab + */ +package btools.util; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; + + +public final class MixCoderDataInputStream extends DataInputStream +{ + private long lastValue; + private long repCount; + + public MixCoderDataInputStream( InputStream is ) + { + super( is ); + } + + public long readSigned() throws IOException + { + long v = readUnsigned(); + return ( v & 1 ) == 0 ? v >> 1 : -(v >> 1 ); + } + + public long readUnsigned() throws IOException + { + long v = 0; + int shift = 0; + for(;;) + { + long i7 = readByte() & 0xff; + v |= (( i7 & 0x7f ) << shift); + if ( ( i7 & 0x80 ) == 0 ) break; + shift += 7; + } + return v; + } + + public long readMixed() throws IOException + { + if ( repCount == 0 ) + { + long b = readByte() & 0xff; + long repCode = b >> 6; + long diffcode = b & 0x3f; + repCount = repCode == 0 ? readUnsigned() : repCode; + lastValue += diffcode == 0 ? readSigned() : diffcode - 32; + } + repCount--; + return lastValue; + } + +} diff --git a/brouter-util/src/main/java/btools/util/MixCoderDataOutputStream.java b/brouter-util/src/main/java/btools/util/MixCoderDataOutputStream.java new file mode 100644 index 0000000..b4a5469 --- /dev/null +++ b/brouter-util/src/main/java/btools/util/MixCoderDataOutputStream.java @@ -0,0 +1,79 @@ +/** + * DataOutputStream for fast-compact encoding of number sequences + * + * @author ab + */ +package btools.util; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; + + +public final class MixCoderDataOutputStream extends DataOutputStream +{ + private long lastValue; + private long lastLastValue; + private long repCount; + private boolean doFlush; + + public MixCoderDataOutputStream( OutputStream os ) + { + super( os ); + } + + public void writeSigned( long v ) throws IOException + { + writeUnsigned( v < 0 ? ( (-v) << 1 ) | 1 : v << 1 ); + } + + public void writeUnsigned( long v ) throws IOException + { + if ( v < 0 ) throw new IllegalArgumentException( "writeUnsigned: " + v ); + do + { + long i7 = v & 0x7f; + v >>= 7; + if ( v != 0 ) i7 |= 0x80; + writeByte( (byte)( i7 & 0xff ) ); + } + while( v != 0 ); + } + + public void writeMixed( long v ) throws IOException + { + if ( v != lastValue && repCount > 0 ) + { + long d = lastValue - lastLastValue; + lastLastValue = lastValue; + + // if diff fits within 6 bits and rep-count < 4, write a single byte + int repCode = repCount < 4 ? (int)repCount : 0; + int diffcode = (int)(d > -32 && d < 32 ? d+32 : 0); + + writeByte( (byte)( diffcode | repCode << 6 ) ); + if ( repCode == 0) + { + writeUnsigned( repCount ); + } + if ( diffcode == 0) + { + writeSigned( d ); + } + repCount = 0; + } + lastValue = v; + repCount++; + } + + @Override + public void flush() throws IOException + { + // todo: does this keep stream consistency after flush ? + long v = lastValue; + writeMixed( v+1 ); + lastValue = v; + repCount = 0; + } + +} diff --git a/brouter-util/src/main/java/btools/util/ReducedMedianFilter.java b/brouter-util/src/main/java/btools/util/ReducedMedianFilter.java new file mode 100644 index 0000000..1b7868a --- /dev/null +++ b/brouter-util/src/main/java/btools/util/ReducedMedianFilter.java @@ -0,0 +1,92 @@ +package btools.util; + +/** + * a median filter with additional edge reduction + */ +public final class ReducedMedianFilter +{ + private int nsamples; + private double[] weights; + private int[] values; + + public ReducedMedianFilter( int size ) + { + weights = new double[size]; + values = new int[size]; + } + + public void reset() + { + nsamples = 0; + } + + public void addSample( double weight, int value ) + { + if ( weight > 0. ) + { + weights[nsamples] = weight; + values[nsamples] = value; + nsamples++; + } + } + + public double calcEdgeReducedMedian( double fraction ) + { + removeEdgeWeight( (1. - fraction)/2., true ); + removeEdgeWeight( (1. - fraction)/2., false ); + + double totalWeight = 0.; + double totalValue = 0.; + for( int i=0; i 0. ) + { + // first pass to find minmax value + double totalWeight = 0.; + int minmax = 0; + for( int i=0; i 0. ) + { + int v = values[i]; + if ( totalWeight == 0. || ( high ? v > minmax : v < minmax ) ) + { + minmax = v; + } + totalWeight += w; + } + } + + if ( totalWeight < excessWeight ) throw new IllegalArgumentException( "ups, not enough weight to remove" ); + + // second pass to remove + for( int i=0; i 0. ) + { + if ( excessWeight > weights[i] ) + { + excessWeight -= weights[i]; + weights[i] = 0.; + } + else + { + weights[i] -= excessWeight; + excessWeight = 0.; + } + } + } + } + } +} diff --git a/brouter-util/src/test/java/btools/util/MixCoderTest.java b/brouter-util/src/test/java/btools/util/MixCoderTest.java new file mode 100644 index 0000000..34706fa --- /dev/null +++ b/brouter-util/src/test/java/btools/util/MixCoderTest.java @@ -0,0 +1,63 @@ +package btools.util; + +import java.util.Random; +import java.io.*; + +import org.junit.Assert; +import org.junit.Test; + +public class MixCoderTest +{ + @Test + public void mixEncodeDecodeTest() throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + MixCoderDataOutputStream mco = new MixCoderDataOutputStream( baos ); + MixCoderDataInputStream mci = null; + + for(;;) + { + Random rnd = new Random( 1234 ); + for( int i=0; i<1500; i++ ) + { + checkEncodeDecode( rnd.nextInt( 3800 ), mco, mci ); + } + for( int i=0; i<1500; i++ ) + { + checkEncodeDecode( rnd.nextInt( 35 ), mco, mci ); + } + for( int i=0; i<1500; i++ ) + { + checkEncodeDecode( 0, mco, mci ); + } + for( int i=0; i<1500; i++ ) + { + checkEncodeDecode( 1000, mco, mci ); + } + + if ( mco != null ) + { + mco.close(); + mco = null; + mci = new MixCoderDataInputStream( new ByteArrayInputStream( baos.toByteArray() ) ); + } + else break; + } + } + + private void checkEncodeDecode( long v, MixCoderDataOutputStream mco, MixCoderDataInputStream mci ) throws IOException + { + if ( mco != null ) + { + mco.writeMixed( v ); + } + if ( mci != null ) + { + long vv = mci.readMixed(); + if ( vv != v ) + { + Assert.assertTrue( "value mismatch: v=" + v + " vv=" + vv, false ); + } + } + } +} diff --git a/brouter-util/src/test/java/btools/util/ReducedMedianFilterTest.java b/brouter-util/src/test/java/btools/util/ReducedMedianFilterTest.java new file mode 100644 index 0000000..c8ca7e9 --- /dev/null +++ b/brouter-util/src/test/java/btools/util/ReducedMedianFilterTest.java @@ -0,0 +1,47 @@ +package btools.util; + +import java.util.Random; +import java.io.*; + +import org.junit.Assert; +import org.junit.Test; + +public class ReducedMedianFilterTest +{ + @Test + public void reducedMedianFilterTest() throws IOException + { + ReducedMedianFilter f = new ReducedMedianFilter( 10 ); + f.reset(); + f.addSample( .2, 10 ); + f.addSample( .2, 10 ); + f.addSample( .2, 10 ); + f.addSample( .2, 15 ); + f.addSample( .2, 20 ); + + double m = f.calcEdgeReducedMedian( 0.5 ); + Assert.assertTrue( "median1 mismatch m=" + m + " expected 11.5", doubleEquals( m, 11.5 ) ); + + f.reset(); + f.addSample( .2, 10 ); + f.addSample( .2, 10 ); + f.addSample( .2, 10 ); + f.addSample( .2, 10 ); + f.addSample( .2, 20 ); + + m = f.calcEdgeReducedMedian( 1. ); + Assert.assertTrue( "median1 mismatch m=" + m + " expected 12", doubleEquals( m, 12. ) ); + + f.reset(); + f.addSample( .5, -10 ); + f.addSample( .5, 10 ); + m = f.calcEdgeReducedMedian( 0.5 ); + Assert.assertTrue( "median2 mismatch m=" + m + " expected 0", doubleEquals( m, 0. ) ); + } + + private boolean doubleEquals( double d1, double d2 ) + { + double d = d1 - d2; + return d < 1e-9 && d > -1e-9; + } +}