From 0a6ead24a176c29b09192bfc66c5a8197464147a Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Mon, 22 Jul 2019 19:25:13 +0200 Subject: [PATCH] corrected lazy crc logic --- .../main/java/btools/codec/MicroCache2.java | 4 +-- .../java/btools/mapcreator/WayLinker.java | 2 +- .../java/btools/mapaccess/DirectWeaver.java | 4 +-- .../main/java/btools/mapaccess/OsmFile.java | 33 +++++++++++-------- .../java/btools/util/BitCoderContext.java | 4 +-- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/brouter-codec/src/main/java/btools/codec/MicroCache2.java b/brouter-codec/src/main/java/btools/codec/MicroCache2.java index c291f01..b4178fe 100644 --- a/brouter-codec/src/main/java/btools/codec/MicroCache2.java +++ b/brouter-codec/src/main/java/btools/codec/MicroCache2.java @@ -34,15 +34,13 @@ public final class MicroCache2 extends MicroCache return b; } - public MicroCache2( DataBuffers dataBuffers, int lonIdx, int latIdx, int divisor, TagValueValidator wayValidator, WaypointMatcher waypointMatcher ) throws Exception + public MicroCache2( StatCoderContext bc, DataBuffers dataBuffers, int lonIdx, int latIdx, int divisor, TagValueValidator wayValidator, WaypointMatcher waypointMatcher ) throws Exception { super( null ); cellsize = 1000000 / divisor; lonBase = lonIdx*cellsize; latBase = latIdx*cellsize; - StatCoderContext bc = new StatCoderContext( dataBuffers.iobuffer ); - TagValueCoder wayTagCoder = new TagValueCoder( bc, dataBuffers, wayValidator ); TagValueCoder nodeTagCoder = new TagValueCoder( bc, dataBuffers, null ); NoisyDiffCoder nodeIdxDiff = new NoisyDiffCoder( bc ); diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java b/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java index 6c4bd23..7f7157a 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java @@ -433,7 +433,7 @@ public class WayLinker extends MapCreatorBase System.arraycopy( abBuf1, 0, subBytes, 0, len ); // cross-check the encoding: re-instantiate the cache - MicroCache mc2 = new MicroCache2( new DataBuffers( subBytes ), lonIdxDiv, latIdxDiv, divisor, null, null ); + MicroCache mc2 = new MicroCache2( new StatCoderContext( subBytes ), new DataBuffers( null ), lonIdxDiv, latIdxDiv, divisor, null, null ); // ..and check if still the same String diffMessage = mc.compareWith( mc2 ); if ( diffMessage != null ) diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/DirectWeaver.java b/brouter-mapaccess/src/main/java/btools/mapaccess/DirectWeaver.java index 7331af3..ed25fd8 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/DirectWeaver.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/DirectWeaver.java @@ -20,14 +20,12 @@ public final class DirectWeaver extends ByteDataWriter private int size = 0; - public DirectWeaver( DataBuffers dataBuffers, int lonIdx, int latIdx, int divisor, TagValueValidator wayValidator, WaypointMatcher waypointMatcher, OsmNodesMap hollowNodes ) throws Exception + public DirectWeaver( StatCoderContext bc, DataBuffers dataBuffers, int lonIdx, int latIdx, int divisor, TagValueValidator wayValidator, WaypointMatcher waypointMatcher, OsmNodesMap hollowNodes ) throws Exception { super( null ); int cellsize = 1000000 / divisor; id64Base = ((long)(lonIdx*cellsize))<<32 | (latIdx*cellsize); - StatCoderContext bc = new StatCoderContext( dataBuffers.iobuffer ); - TagValueCoder wayTagCoder = new TagValueCoder( bc, dataBuffers, wayValidator ); TagValueCoder nodeTagCoder = new TagValueCoder( bc, dataBuffers, null ); NoisyDiffCoder nodeIdxDiff = new NoisyDiffCoder( bc ); diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java index 4645d1f..bf03490 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java @@ -11,6 +11,7 @@ import java.io.RandomAccessFile; import btools.codec.DataBuffers; import btools.codec.MicroCache; import btools.codec.MicroCache2; +import btools.codec.StatCoderContext; import btools.codec.TagValueValidator; import btools.codec.WaypointMatcher; import btools.util.ByteDataReader; @@ -142,9 +143,10 @@ final class OsmFile { ab = new byte[asize]; asize = getDataInputForSubIdx( subIdx, ab ); - dataBuffers = new DataBuffers( ab ); } + StatCoderContext bc = new StatCoderContext( ab ); + try { if ( !reallyDecode ) @@ -153,25 +155,28 @@ final class OsmFile } if ( hollowNodes == null ) { - return new MicroCache2( dataBuffers, lonIdx, latIdx, divisor, wayValidator, waypointMatcher ); + return new MicroCache2( bc, dataBuffers, lonIdx, latIdx, divisor, wayValidator, waypointMatcher ); } - new DirectWeaver( dataBuffers, lonIdx, latIdx, divisor, wayValidator, waypointMatcher, hollowNodes ); + new DirectWeaver( bc, dataBuffers, lonIdx, latIdx, divisor, wayValidator, waypointMatcher, hollowNodes ); return MicroCache.emptyNonVirgin; } - catch( Throwable t ) + finally { - // checksum test now only in case of trouble - int crcData = Crc32.crc( ab, 0, asize - 4 ); - int crcFooter = new ByteDataReader( ab, asize - 4 ).readInt(); - if ( crcData == crcFooter ) + // crc check only if the buffer has not been fully read + int readBytes = (bc.getReadingBitPosition()+7)>>3; + if ( readBytes != asize-4 ) { - throw new IOException( "old, unsupported data-format" ); + int crcData = Crc32.crc( ab, 0, asize - 4 ); + int crcFooter = new ByteDataReader( ab, asize - 4 ).readInt(); + if ( crcData == crcFooter ) + { + throw new IOException( "old, unsupported data-format" ); + } + else if ( ( crcData ^ 2 ) != crcFooter ) + { + throw new IOException( "checkum error" ); + } } - else if ( ( crcData ^ 2 ) == crcFooter ) - { - throw new IOException( "checkum error" ); - } - throw t instanceof Exception ? (Exception)t : new Exception( t.toString(), t ); } } diff --git a/brouter-util/src/main/java/btools/util/BitCoderContext.java b/brouter-util/src/main/java/btools/util/BitCoderContext.java index 9154702..680f014 100644 --- a/brouter-util/src/main/java/btools/util/BitCoderContext.java +++ b/brouter-util/src/main/java/btools/util/BitCoderContext.java @@ -287,9 +287,9 @@ public class BitCoderContext { while (bits < 24) { - if ( idx < idxMax ) + if ( idx++ < idxMax ) { - b |= (ab[++idx] & 0xff) << bits; + b |= (ab[idx] & 0xff) << bits; } bits += 8; }