diff --git a/brouter-core/pom.xml b/brouter-core/pom.xml index e75292b..6be1aaa 100644 --- a/brouter-core/pom.xml +++ b/brouter-core/pom.xml @@ -27,5 +27,9 @@ brouter-expressions ${project.version} + + junit + junit + diff --git a/brouter-expressions/pom.xml b/brouter-expressions/pom.xml index a58eda5..22da797 100644 --- a/brouter-expressions/pom.xml +++ b/brouter-expressions/pom.xml @@ -10,4 +10,12 @@ brouter-expressions jar + + + + org.btools + brouter-util + ${project.version} + + diff --git a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java index 7b2f0c3..e01d9b1 100644 --- a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java +++ b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java @@ -17,6 +17,8 @@ import java.util.Map; import java.util.StringTokenizer; import java.util.TreeMap; +import btools.util.Crc32; + public final class BExpressionContext { private static final String CONTEXT_TAG = "---context:"; @@ -228,40 +230,6 @@ public final class BExpressionContext } } - private static int[] crctable = { - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, - 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, - 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, - 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, - 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, - 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, - 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, - 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, - 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, - 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, - 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, - 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, - 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, - 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, - 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, - 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, - 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, - 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, - 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, - 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, - 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, - 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, - 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, - 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, - 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, - 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, - 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, - 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, - 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, - 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, - 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, - }; public void evaluate( long bitmap, BExpressionReceiver receiver ) @@ -271,13 +239,7 @@ public final class BExpressionContext if ( currentBitmap != bitmap || currentHashBucket < 0 ) { // calc hash bucket from crc - int crc = 0xFFFFFFFF; - long bm = bitmap; - for( int j=0; j<8; j++ ) - { - crc = (crc >>> 8) ^ crctable[(crc ^ (int)bm) & 0xff]; - bm >>= 8; - } + int crc = Crc32.crc( bitmap ); int hashSize = _arrayBitmap.length; currentHashBucket = (crc & 0xfffffff) % hashSize; currentBitmap = bitmap; diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java b/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java index 0dabc7d..a7aaeeb 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java @@ -31,6 +31,8 @@ public class WayLinker extends MapCreatorBase private CompactLongSet borderSet; private short lookupVersion; + private long creationTimeStamp; + private BExpressionContext expctxWay; private int minLon; @@ -67,6 +69,8 @@ public class WayLinker extends MapCreatorBase expctxWay.readMetaData( lookupFile ); lookupVersion = expctxWay.lookupVersion; expctxWay.parseFile( profileFile, "global" ); + + creationTimeStamp = System.currentTimeMillis(); // then process all segments new WayIterator( this, true ).processDir( wayTilesIn, ".wt5" ); @@ -88,7 +92,7 @@ public class WayLinker extends MapCreatorBase // read this tile's nodes readingBorder = false; - new NodeIterator( this, true ).processFile( nodeFile ); + new NodeIterator( this, false ).processFile( nodeFile ); // freeze the nodes-map FrozenLongMap nodesMapFrozen = new FrozenLongMap( nodesMap ); @@ -203,8 +207,10 @@ public class WayLinker extends MapCreatorBase File outfile = fileFromTemplate( wayfile, dataTilesOut, dataTilesSuffix ); DataOutputStream os = createOutStream( outfile ); - // write 5*5 index dummy long[] fileIndex = new long[25]; + int[] fileHeaderCrcs = new int[25]; + + // write 5*5 index dummy for( int i55=0; i55<25; i55++) { os.writeLong( 0 ); @@ -243,7 +249,7 @@ public class WayLinker extends MapCreatorBase ByteArrayOutputStream bos = new ByteArrayOutputStream( ); DataOutputStream dos = new DataOutputStream( bos ); - dos.writeInt( subList.size() ); + dos.writeInt( subList.size() + 1 ); // reserve 1 dummy node for crc for( int ni=0; ni brouter-mapaccess jar + + + + org.btools + brouter-util + ${project.version} + + diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/MicroCache.java b/brouter-mapaccess/src/main/java/btools/mapaccess/MicroCache.java index eb6a752..efbbcd9 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/MicroCache.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/MicroCache.java @@ -8,6 +8,8 @@ package btools.mapaccess; import java.util.*; import java.io.*; +import btools.util.Crc32; + final class MicroCache { private long[] faid; @@ -32,10 +34,10 @@ final class MicroCache int subIdx = (latIdx80-80*latDegree)*80 + (lonIdx80-80*lonDegree); - try { ab = iobuffer; int asize = segfile.getDataInputForSubIdx(subIdx, ab); + if ( asize == 0 ) { return; @@ -48,8 +50,30 @@ final class MicroCache aboffset = 0; size = readInt(); + // get net size + int nbytes = 0; + for(int i = 0; i fileCache; - private HashMap indexCache; + private HashMap fileCache; private byte[] iobuffer; private OsmFile[][] fileRows = new OsmFile[180][]; @@ -37,14 +36,12 @@ public final class NodesCache if ( oldCache != null ) { fileCache = oldCache.fileCache; - indexCache = oldCache.indexCache; iobuffer = oldCache.iobuffer; oom_carsubset_hint = oldCache.oom_carsubset_hint; } else { - fileCache = new HashMap(4); - indexCache = new HashMap(4); + fileCache = new HashMap(4); iobuffer = new byte[65636]; } } @@ -144,6 +141,7 @@ public final class NodesCache currentFileName = filenameBase + ".rd5/cd5"; + PhysicalFile ra = null; if ( !fileCache.containsKey( filenameBase ) ) { File f = null; @@ -158,39 +156,17 @@ public final class NodesCache if ( fullFile.exists() ) f = fullFile; if ( carMode && f != null ) oom_carsubset_hint = true; } - RandomAccessFile ra = f != null ? new RandomAccessFile( f, "r" ) : null; - fileCache.put( filenameBase, ra ); - if ( ra != null ) + if ( f != null ) { - long[] fileIndex = new long[25]; - ra.readFully( iobuffer, 0, 200 ); - ByteDataReader dis = new ByteDataReader( iobuffer ); - for( int i=0; i<25; i++ ) - { - long lv = dis.readLong(); - short readVersion = (short)(lv >> 48); - if ( readVersion != lookupVersion ) - { - throw new IllegalArgumentException( "lookup version mismatch (old rd5?) lookups.dat=" - + lookupVersion + " " + f. getAbsolutePath() + "=" + readVersion ); - } - fileIndex[i] = lv & 0xffffffffffffL; - } - indexCache.put( filenameBase, fileIndex ); + currentFileName = f.getName(); + ra = new PhysicalFile( f, iobuffer, lookupVersion ); } + fileCache.put( filenameBase, ra ); } - RandomAccessFile ra = fileCache.get( filenameBase ); - long startPos = 0L; - if ( ra != null ) - { - long[] index = indexCache.get( filenameBase ); - startPos = tileIndex > 0 ? index[ tileIndex-1 ] : 200L; - if ( startPos == index[ tileIndex] ) ra = null; - } - OsmFile osmf = new OsmFile( ra, startPos, iobuffer ); + ra = fileCache.get( filenameBase ); + OsmFile osmf = new OsmFile( ra, tileIndex, iobuffer ); osmf.lonDegree = lonDegree; osmf.latDegree = latDegree; - osmf.filename = currentFileName; return osmf; } @@ -208,11 +184,11 @@ public final class NodesCache public void close() { - for( RandomAccessFile f: fileCache.values() ) + for( PhysicalFile f: fileCache.values() ) { try { - f.close(); + f.ra.close(); } catch( IOException ioe ) { diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java index 4327e09..0f54b51 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java @@ -7,6 +7,7 @@ package btools.mapaccess; import java.io.IOException; import java.io.RandomAccessFile; +import btools.util.Crc32; final class OsmFile { @@ -21,16 +22,31 @@ final class OsmFile public String filename; - public OsmFile( RandomAccessFile rafile, long startPos, byte[] iobuffer ) throws Exception + public OsmFile( PhysicalFile rafile, int tileIndex, byte[] iobuffer ) throws Exception { - fileOffset = startPos; if ( rafile != null ) { - is = rafile; + filename = rafile.fileName; + + long[] index = rafile.fileIndex; + fileOffset = tileIndex > 0 ? index[ tileIndex-1 ] : 200L; + if ( fileOffset == index[ tileIndex] ) return; // empty + + is = rafile.ra; posIdx = new int[6400]; microCaches = new MicroCache[6400]; is.seek( fileOffset ); is.readFully( iobuffer, 0, 25600 ); + + if ( rafile.fileHeaderCrcs != null ) + { + int headerCrc = Crc32.crc( iobuffer, 0, 25600 ); + if ( rafile.fileHeaderCrcs[tileIndex] != headerCrc ) + { + throw new IOException( "sub index checksum error" ); + } + } + ByteDataReader dis = new ByteDataReader( iobuffer ); for( int i=0; i<6400; i++ ) { @@ -54,7 +70,7 @@ final class OsmFile is.seek( fileOffset + startPos ); if ( size <= iobuffer.length ) { - is.readFully( iobuffer ); + is.readFully( iobuffer, 0, size ); } } return size; diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/PhysicalFile.java b/brouter-mapaccess/src/main/java/btools/mapaccess/PhysicalFile.java new file mode 100644 index 0000000..3b424c0 --- /dev/null +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/PhysicalFile.java @@ -0,0 +1,69 @@ +/** + * cache for a single square + * + * @author ab + */ +package btools.mapaccess; + +import java.io.*; +import btools.util.Crc32; + +final class PhysicalFile +{ + RandomAccessFile ra = null; + long[] fileIndex = new long[25]; + int[] fileHeaderCrcs; + + private int fileIndexCrc; + public long creationTime; + + String fileName; + + public PhysicalFile( File f, byte[] iobuffer, int lookupVersion ) throws Exception + { + fileName = f.getName(); + + ra = new RandomAccessFile( f, "r" ); + ra.readFully( iobuffer, 0, 200 ); + fileIndexCrc = Crc32.crc( iobuffer, 0, 200 ); + ByteDataReader dis = new ByteDataReader( iobuffer ); + for( int i=0; i<25; i++ ) + { + long lv = dis.readLong(); + short readVersion = (short)(lv >> 48); + if ( readVersion != lookupVersion ) + { + throw new IllegalArgumentException( "lookup version mismatch (old rd5?) lookups.dat=" + + lookupVersion + " " + f. getAbsolutePath() + "=" + readVersion ); + } + fileIndex[i] = lv & 0xffffffffffffL; + } + + // read some extra info from the end of the file, if present + long len = ra.length(); + + long pos = fileIndex[24]; + int extraLen = 8 + 26*4; + + if ( len == pos ) return; // old format o.k. + + if ( len < pos+extraLen ) // > is o.k. for future extensions! + { + throw new IOException( "file of size " + len + " + too short, should be " + (pos+extraLen) ); + } + + ra.seek( pos ); + ra.readFully( iobuffer, 0, extraLen ); + dis = new ByteDataReader( iobuffer ); + creationTime = dis.readLong(); + if ( dis.readInt() != fileIndexCrc ) + { + throw new IOException( "top index checksum error" ); + } + fileHeaderCrcs = new int[25]; + for( int i=0; i<25; i++ ) + { + fileHeaderCrcs[i] = dis.readInt(); + } + } +} diff --git a/brouter-server/pom.xml b/brouter-server/pom.xml index ede752c..62b408c 100644 --- a/brouter-server/pom.xml +++ b/brouter-server/pom.xml @@ -51,5 +51,9 @@ brouter-map-creator ${project.version} + + junit + junit + diff --git a/brouter-server/src/test/java/btools/server/RouterTest.java b/brouter-server/src/test/java/btools/server/RouterTest.java new file mode 100644 index 0000000..12f4ad9 --- /dev/null +++ b/brouter-server/src/test/java/btools/server/RouterTest.java @@ -0,0 +1,56 @@ +package btools.server; + +import java.util.*; + +import org.junit.Assert; +import org.junit.Test; +import java.net.URL; +import java.io.File; + +import btools.router.*; +import btools.mapaccess.*; + +public class RouterTest +{ + @Test + public void routerTest() throws Exception + { + URL resulturl = this.getClass().getResource( "/testtrack0.gpx" ); + Assert.assertTrue( "reference result not found: ", resulturl != null ); + File resultfile = new File(resulturl.getFile()); + File workingDir = resultfile.getParentFile(); + + String wd = workingDir.getAbsolutePath(); + + List wplist = new ArrayList(); + OsmNodeNamed n; + n = new OsmNodeNamed(); + n.name = "from"; + n.ilon = 180000000 + 8720897; + n.ilat = 90000000 + 50002515; + wplist.add( n ); + + n = new OsmNodeNamed(); + n.name = "to"; + n.ilon = 180000000 + 8723658; + n.ilat = 90000000 + 49997510; + wplist.add( n ); + + RoutingContext rctx = new RoutingContext(); + rctx.localFunction = wd + "/../../../misc/profiles2/trekking.brf"; + // c.setAlternativeIdx( 1 ); + + RoutingEngine re = new RoutingEngine( + wd + "/testtrack", + wd + "/testlog", + wd + "/../../../brouter-map-creator/target/test-classes/tmp/segments", wplist, rctx ); + re.doRun( 0 ); + + // error message from router? + Assert.assertTrue( "routing failed: " + re.getErrorMessage(), re.getErrorMessage() == null ); + + // if the track didn't change, we expect the first alternative also + File a1 = new File( workingDir, "testtrack1.gpx" ); + Assert.assertTrue( "result content missmatch", a1.exists() ); + } +} diff --git a/brouter-server/src/test/resources/testtrack0.gpx b/brouter-server/src/test/resources/testtrack0.gpx new file mode 100644 index 0000000..bc3a2da --- /dev/null +++ b/brouter-server/src/test/resources/testtrack0.gpx @@ -0,0 +1,44 @@ + + + + + brouter_trekking_0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/brouter-util/src/main/java/btools/util/Crc32.java b/brouter-util/src/main/java/btools/util/Crc32.java new file mode 100644 index 0000000..9bff391 --- /dev/null +++ b/brouter-util/src/main/java/btools/util/Crc32.java @@ -0,0 +1,67 @@ +package btools.util; + +/** + * checksum stuff + * + * @author ab + */ +public class Crc32 +{ + public static int crc( long bitmap ) + { + int crc = 0xFFFFFFFF; + long bm = bitmap; + for( int j=0; j<8; j++ ) + { + crc = (crc >>> 8) ^ crctable[(crc ^ (int)bm) & 0xff]; + bm >>= 8; + } + return crc; + } + + public static int crc( byte[] ab, int offset, int len ) + { + int crc = 0xFFFFFFFF; + for( int j=offset; j>> 8) ^ crctable[(crc ^ ab[j]) & 0xff]; + } + return crc; + } + + private static int[] crctable = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, + }; + +} diff --git a/brouter-util/src/test/java/btools/util/CompactLongMap.java b/brouter-util/src/test/java/btools/util/CompactLongMap.java deleted file mode 100644 index 314772b..0000000 --- a/brouter-util/src/test/java/btools/util/CompactLongMap.java +++ /dev/null @@ -1,328 +0,0 @@ -package btools.util; - -import java.util.ArrayList; - -/** - * Memory efficient Map to map a long-key to an object-value - * - * Implementation is such that basically the 12 bytes - * per entry is allocated that's needed to store - * a long- and an object-value. - * This class does not implement the Map interface - * because it's not complete (remove() is not implemented, - * CompactLongMap can only grow.) - * - * @author ab - */ -public class CompactLongMap -{ - private long[][] al; - private int[] pa; - private int size = 0; - private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays - - private V value_in; - protected V value_out; - - protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE - private static boolean earlyDuplicateCheck; - - public CompactLongMap() - { - // pointer array - pa = new int[MAXLISTS]; - - // allocate key lists - al = new long[MAXLISTS][]; - al[0] = new long[1]; // make the first array (the transient buffer) - - // same for the values - vla = new Object[MAXLISTS][]; - vla[0] = new Object[1]; - - earlyDuplicateCheck = Boolean.getBoolean( "earlyDuplicateCheck" ); - } - - - /* - * - * The Map extension: - * next 5 protected methods are needed to implement value-support - * overwrite them all to support value structures other than the - * long-values implemented here as a sample. - * - * Furthermore, put() and get() method need to be implemented - * to access the values. - * - * Note that this map does not behave exactly like java.util.Map - * - put(..) with already existing key throws exception - * - get(..) with non-existing key thros exception - * - * If you have keys that cannot easily be mapped on long's, use - * a hash-function to do the mapping. But note that, in comparison - * to java.util.HashMap, in that case the keys itself are not saved, - * only the hash-values, so you need to be sure that random duplicate - * hashs are either excluded by the structure of your data or that - * you can handle the possible IllegalArgumentException - * - */ - - private Object[][] vla; // value list array - - - public boolean put( long id, V value ) - { - try - { - value_in = value; - if ( contains( id, true ) ) - { - return true; - } - vla[0][0] = value; - _add( id ); - return false; - } - finally - { - value_in = null; - value_out = null; - } - } - - /** - * Same as put( id, value ) but duplicate check - * is skipped for performance. Be aware that you - * can get a duplicate exception later on if the - * map is restructured! - * with System parameter earlyDuplicateCheck=true you - * can enforce the early duplicate check for debugging - * - * @param id the key to insert - * @param value the value to insert object - * @exception IllegalArgumentException for duplicates if enabled - */ - public void fastPut( long id, V value ) - { - if ( earlyDuplicateCheck && contains( id ) ) - { - throw new IllegalArgumentException( "duplicate key found in early check: " + id ); - } - vla[0][0] = value; - _add( id ); - } - - /** - * Get the value for the given id - * @param id the key to query - * @return the object, or null if id not known - */ - public V get( long id ) - { - try - { - if ( contains( id, false ) ) - { - return value_out; - } - return null; - } - finally - { - value_out = null; - } - } - - - /** - * @return the number of entries in this map - */ - public int size() - { - return size; - } - - - private boolean _add( long id ) - { - if ( size == Integer.MAX_VALUE ) - { - throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" ); - } - - // put the new entry in the first array - al[0][0] = id; - - // determine the first empty array - int bp = size++; // treat size as bitpattern - int idx = 1; - int n = 1; - - pa[0] = 1; - pa[1] = 1; - - while ( (bp&1) == 1 ) - { - bp >>= 1; - pa[idx++] = n; - n <<= 1; - } - - // create it if not existant - if ( al[idx] == null ) - { - al[idx] = new long[n]; - vla[idx] = new Object[n]; - } - - // now merge the contents of arrays 0...idx-1 into idx - while ( n > 0 ) - { - long maxId = 0; - int maxIdx = -1; - - for ( int i=0; i 0 ) - { - long currentId = al[i][p-1]; - if ( maxIdx < 0 || currentId > maxId ) - { - maxIdx = i; - maxId = currentId; - } - } - } - - // current maximum found, copy to target array - if ( n < al[idx].length && maxId == al[idx][n] ) - { - throw new IllegalArgumentException( "duplicate key found in late check: " + maxId ); - } - --n; - al[idx][n] = maxId; - vla[idx][n] = vla[maxIdx][pa[maxIdx]-1]; - - --pa[maxIdx]; - } - - // de-allocate empty arrays of a certain size (fix at 64kByte) - while ( idx-- > _maxKeepExponent ) - { - al[idx] = null; - vla[idx] = null; - } - - return false; - } - - /** - * @return true if "id" is contained in this set. - */ - public boolean contains( long id ) - { - try - { - return contains( id, false ); - } - finally - { - value_out = null; - } - } - - protected boolean contains( long id, boolean doPut ) - { - // determine the first empty array - int bp = size; // treat size as bitpattern - int idx = 1; - - while ( bp != 0 ) - { - if ( (bp&1) == 1 ) - { - // array at idx is valid, check - if ( contains( idx, id, doPut ) ) - { - return true; - } - } - idx++; - bp >>= 1; - } - return false; - } - - - // does sorted array "a" contain "id" ? - private boolean contains( int idx, long id, boolean doPut ) - { - long[] a = al[idx]; - int offset = a.length; - int n = 0; - - while ( (offset >>= 1) > 0 ) - { - int nn = n + offset; - if ( a[nn] <= id ) - { - n = nn; - } - } - if ( a[n] == id ) - { - value_out = (V)vla[idx][n]; - if ( doPut ) vla[idx][n] = value_in; - return true; - } - return false; - } - - protected void moveToFrozenArrays( long[] faid, ArrayList flv ) - { - for( int i=1; i>= 1; - } - faid[ti] = minId; - flv.add( (V)vla[minIdx][pa[minIdx]] ); - pa[minIdx]++; - - if ( ti > 0 && faid[ti-1] == minId ) - { - throw new IllegalArgumentException( "duplicate key found in late check: " + minId ); - } - } - - // free the non-frozen arrays - al = null; - vla = null; - } - -} diff --git a/brouter-util/src/test/java/btools/util/CompactLongSet.java b/brouter-util/src/test/java/btools/util/CompactLongSet.java deleted file mode 100644 index 08dace4..0000000 --- a/brouter-util/src/test/java/btools/util/CompactLongSet.java +++ /dev/null @@ -1,220 +0,0 @@ -package btools.util; - -/** - * Memory efficient Set for long-keys - * - * @author ab - */ -public class CompactLongSet -{ - private long[][] al; - private int[] pa; - private int size = 0; - private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays - - protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE - private static boolean earlyDuplicateCheck; - - public CompactLongSet() - { - // pointer array - pa = new int[MAXLISTS]; - - // allocate key lists - al = new long[MAXLISTS][]; - al[0] = new long[1]; // make the first array (the transient buffer) - - earlyDuplicateCheck = Boolean.getBoolean( "earlyDuplicateCheck" ); - } - - - /** - * @return the number of entries in this set - */ - public int size() - { - return size; - } - - /** - * add a long value to this set if not yet in. - * @param id the value to add to this set. - * @return true if "id" already contained in this set. - */ - public boolean add( long id ) - { - if ( contains( id ) ) - { - return true; - } - _add( id ); - return false; - } - - public void fastAdd( long id ) - { - _add( id ); - } - - private void _add( long id ) - { - if ( size == Integer.MAX_VALUE ) - { - throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" ); - } - - // put the new entry in the first array - al[0][0] = id; - - // determine the first empty array - int bp = size++; // treat size as bitpattern - int idx = 1; - int n = 1; - - pa[0] = 1; - pa[1] = 1; - - while ( (bp&1) == 1 ) - { - bp >>= 1; - pa[idx++] = n; - n <<= 1; - } - - // create it if not existant - if ( al[idx] == null ) - { - al[idx] = new long[n]; - } - - // now merge the contents of arrays 0...idx-1 into idx - while ( n > 0 ) - { - long maxId = 0; - int maxIdx = -1; - - for ( int i=0; i 0 ) - { - long currentId = al[i][p-1]; - if ( maxIdx < 0 || currentId > maxId ) - { - maxIdx = i; - maxId = currentId; - } - } - } - - // current maximum found, copy to target array - if ( n < al[idx].length && maxId == al[idx][n] ) - { - throw new IllegalArgumentException( "duplicate key found in late check: " + maxId ); - } - --n; - al[idx][n] = maxId; - - --pa[maxIdx]; - } - - // de-allocate empty arrays of a certain size (fix at 64kByte) - while ( idx-- > _maxKeepExponent ) - { - al[idx] = null; - } - } - - /** - * @return true if "id" is contained in this set. - */ - public boolean contains( long id ) - { - // determine the first empty array - int bp = size; // treat size as bitpattern - int idx = 1; - - while ( bp != 0 ) - { - if ( (bp&1) == 1 ) - { - // array at idx is valid, check - if ( contains( idx, id ) ) - { - return true; - } - } - idx++; - bp >>= 1; - } - return false; - } - - - // does sorted array "a" contain "id" ? - private boolean contains( int idx, long id ) - { - long[] a = al[idx]; - int offset = a.length; - int n = 0; - - while ( (offset >>= 1) > 0 ) - { - int nn = n + offset; - if ( a[nn] <= id ) - { - n = nn; - } - } - if ( a[n] == id ) - { - return true; - } - return false; - } - - protected void moveToFrozenArray( long[] faid ) - { - for( int i=1; i>= 1; - } - faid[ti] = minId; - pa[minIdx]++; - - if ( ti > 0 && faid[ti-1] == minId ) - { - throw new IllegalArgumentException( "duplicate key found in late check: " + minId ); - } - } - - // free the non-frozen array - al = null; - } - -} diff --git a/brouter-util/src/test/java/btools/util/CompactMapTest.java b/brouter-util/src/test/java/btools/util/CompactMapTest.java new file mode 100644 index 0000000..4149690 --- /dev/null +++ b/brouter-util/src/test/java/btools/util/CompactMapTest.java @@ -0,0 +1,68 @@ +package btools.util; + +import java.util.Random; +import java.util.HashMap; + +import org.junit.Assert; +import org.junit.Test; + +public class CompactMapTest +{ + @Test + public void hashMapComparisonTest() + { + hashMapComparison( 0, 1 ); + hashMapComparison( 1, 1 ); + hashMapComparison( 2, 2 ); + hashMapComparison( 3, 3 ); + hashMapComparison( 4, 4 ); + hashMapComparison( 5, 5 ); + hashMapComparison( 7, 10 ); + hashMapComparison( 8, 10 ); + hashMapComparison( 10000, 20000 ); + } + + private void hashMapComparison( int mapsize, int trycount ) + { + Random rand = new Random( 12345 ); + HashMap hmap = new HashMap(); + CompactLongMap cmap_slow = new CompactLongMap(); + CompactLongMap cmap_fast = new CompactLongMap(); + + for( int i=0; i( cmap_slow ); + cmap_fast = new FrozenLongMap( cmap_fast ); + } + long k = mapsize < 10 ? i : rand.nextInt( 20000 ); + Long KK = new Long( k ); + String s = hmap.get( KK ); + + boolean contained = hmap.containsKey( KK ); + Assert.assertTrue( "containsKey missmatch (slow)", contained == cmap_slow.contains( k ) ); + Assert.assertTrue( "containsKey missmatch (fast)", contained == cmap_fast.contains( k ) ); + + if ( contained ) + { + Assert.assertEquals( "object missmatch (fast)", s, cmap_fast.get( k ) ); + Assert.assertEquals( "object missmatch (slow)", s, cmap_slow.get( k ) ); + } + } + } +} diff --git a/brouter-util/src/test/java/btools/util/CompactSetTest.java b/brouter-util/src/test/java/btools/util/CompactSetTest.java new file mode 100644 index 0000000..30987ec --- /dev/null +++ b/brouter-util/src/test/java/btools/util/CompactSetTest.java @@ -0,0 +1,60 @@ +package btools.util; + +import java.util.Random; +import java.util.HashSet; + +import org.junit.Assert; +import org.junit.Test; + +public class CompactSetTest +{ + @Test + public void hashSetComparisonTest() + { + hashSetComparison( 0, 1 ); + hashSetComparison( 1, 1 ); + hashSetComparison( 2, 2 ); + hashSetComparison( 3, 3 ); + hashSetComparison( 4, 4 ); + hashSetComparison( 5, 5 ); + hashSetComparison( 7, 10 ); + hashSetComparison( 8, 10 ); + hashSetComparison( 10000, 20000 ); + } + + private void hashSetComparison( int setsize, int trycount ) + { + Random rand = new Random( 12345 ); + HashSet hset = new HashSet(); + CompactLongSet cset_slow = new CompactLongSet(); + CompactLongSet cset_fast = new CompactLongSet(); + + for( int i=0; i blocklist = new ArrayList(1024); - - private static final int BLOCKSIZE = 0x10000; // 64k * 32 bits - private int valuebits; - private int maxvalue; - private long maxkey; - private long maxmemory; - - /** - * Creates a DenseLongMap for the given value range - * Note that one value is reserved for the "unset" state, - * so with 6 value bits you can store values in the - * range 0..62 only - * - * @param valuebits number of bits to use per value - */ - public DenseLongMap( int valuebits ) - { - if ( valuebits < 1 || valuebits > 32 ) - { - throw new IllegalArgumentException( "invalid valuebits (1..32): " + valuebits ); - } - this.valuebits = valuebits; - maxmemory = (Runtime.getRuntime().maxMemory() / 8) * 7; // assume most of it for our map - maxvalue = (1 << valuebits) - 2; - maxkey = ( maxmemory / valuebits ) * 8; - } - - - - public void put( long key, int value ) - { - if ( key < 0L || key > maxkey ) - { - throw new IllegalArgumentException( "key out of range (0.." + maxkey + "): " + key - + " give more memory (currently " + (maxmemory / 0x100000) - + "MB) to extend key range" ); - } - if ( value < 0 || value > maxvalue ) - { - throw new IllegalArgumentException( "value out of range (0.." + maxvalue + "): " + value ); - } - - int blockn = (int)(key >> 21); - int offset = (int)(key & 0x1fffff); - - int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null; - - if ( block == null ) - { - block = new int[BLOCKSIZE * valuebits]; - - while (blocklist.size() < blockn+1 ) - { - blocklist.add(null); - } - blocklist.set( blockn, block ); - } - - int bitmask = 1 << (offset & 0x1f); - int invmask = bitmask ^ 0xffffffff; - int probebit = 1; - int blockidx = (offset >> 5)*valuebits; - int blockend = blockidx + valuebits; - int v = value + 1; // 0 is reserved (=unset) - - while( blockidx < blockend ) - { - if ( ( v & probebit ) != 0 ) - { - block[blockidx] |= bitmask; - } - else - { - block[blockidx] &= invmask; - } - probebit <<= 1; - blockidx++; - } - } - - - public int getInt( long key ) - { - if ( key < 0 ) - { - return -1; - } - int blockn = (int)(key >> 21); - int offset = (int)(key & 0x1fffff); - - int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null; - - if ( block == null ) - { - return -1; - } - int bitmask = 1 << (offset & 0x1f); - int probebit = 1; - int blockidx = (offset >> 5)*valuebits; - int blockend = blockidx + valuebits; - int v = 0; // 0 is reserved (=unset) - - while( blockidx < blockend ) - { - if ( ( block[blockidx] & bitmask ) != 0 ) - { - v |= probebit; - } - probebit <<= 1; - blockidx++; - } - return v-1; - } - -} diff --git a/brouter-util/src/test/java/btools/util/DenseLongMapTest.java b/brouter-util/src/test/java/btools/util/DenseLongMapTest.java new file mode 100644 index 0000000..5c5b348 --- /dev/null +++ b/brouter-util/src/test/java/btools/util/DenseLongMapTest.java @@ -0,0 +1,113 @@ +package btools.util; + +import java.util.Random; +import java.util.HashMap; +import java.util.HashSet; + +import org.junit.Assert; +import org.junit.Test; + +public class DenseLongMapTest +{ + @Test + public void hashMapComparisonTest() + { + hashMapComparison( 100000, 100000, 100000 ); + hashMapComparison( 100000, 100000, 13000000 ); + } + + private void hashMapComparison( int mapsize, int trycount, long keyrange ) + { + Random rand = new Random( 12345 ); + HashMap hmap = new HashMap(); + DenseLongMap dmap = new DenseLongMap( 6 ); + + for( int i=0; i extends CompactLongMap -{ - private long[] faid; - private ArrayList flv; - private int size = 0; - private int p2size; // next power of 2 of size - - public FrozenLongMap( CompactLongMap map ) - { - size = map.size(); - - faid = new long[size]; - flv = new ArrayList(size); - - map.moveToFrozenArrays( faid, flv ); - - p2size = 0x40000000; - while( p2size > size ) p2size >>= 1; - } - - @Override - public boolean put( long id, V value ) - { - throw new RuntimeException( "cannot put on FrozenLongIntMap" ); - } - - @Override - public void fastPut( long id, V value ) - { - throw new RuntimeException( "cannot put on FrozenLongIntMap" ); - } - - /** - * @return the number of entries in this set - */ - @Override - public int size() - { - return size; - } - - - - /** - * @return true if "id" is contained in this set. - */ - @Override - protected boolean contains( long id, boolean doPut ) - { - if ( size == 0 ) - { - return false; - } - long[] a = faid; - int offset = p2size; - int n = 0; - - while ( offset> 0 ) - { - int nn = n + offset; - if ( nn < size && a[nn] <= id ) - { - n = nn; - } - offset >>= 1; - } - if ( a[n] == id ) - { - value_out = flv.get(n); - return true; - } - return false; - } - - /** - * @return the value for "id", - * Throw an exception if not contained in the map. - */ - @Override - public V get( long id ) - { - if ( size == 0 ) - { - return null; - } - long[] a = faid; - int offset = p2size; - int n = 0; - - while ( offset> 0 ) - { - int nn = n + offset; - if ( nn < size && a[nn] <= id ) - { - n = nn; - } - offset >>= 1; - } - if ( a[n] == id ) - { - return flv.get(n); - } - return null; - } - - public List getValueList() - { - return flv; - } -} diff --git a/brouter-util/src/test/java/btools/util/FrozenLongSet.java b/brouter-util/src/test/java/btools/util/FrozenLongSet.java deleted file mode 100644 index f6ef6b3..0000000 --- a/brouter-util/src/test/java/btools/util/FrozenLongSet.java +++ /dev/null @@ -1,81 +0,0 @@ -package btools.util; - -/** - * Frozen instance of Memory efficient Set - * - * This one is readily sorted into a singe array for faster access - * - * @author ab - */ -public class FrozenLongSet extends CompactLongSet -{ - private long[] faid; - private int size = 0; - private int p2size; // next power of 2 of size - - public FrozenLongSet( CompactLongSet set ) - { - size = set.size(); - - faid = new long[size]; - - set.moveToFrozenArray( faid ); - - p2size = 0x40000000; - while( p2size > size ) p2size >>= 1; - } - - @Override - public boolean add( long id ) - { - throw new RuntimeException( "cannot add on FrozenLongSet" ); - } - - @Override - public void fastAdd( long id ) - { - throw new RuntimeException( "cannot add on FrozenLongSet" ); - } - - /** - * @return the number of entries in this set - */ - @Override - public int size() - { - return size; - } - - - - /** - * @return true if "id" is contained in this set. - */ - @Override - public boolean contains( long id ) - { - if ( size == 0 ) - { - return false; - } - long[] a = faid; - int offset = p2size; - int n = 0; - - while ( offset> 0 ) - { - int nn = n + offset; - if ( nn < size && a[nn] <= id ) - { - n = nn; - } - offset >>= 1; - } - if ( a[n] == id ) - { - return true; - } - return false; - } - -} diff --git a/brouter-util/src/test/java/btools/util/LazyArrayOfLists.java b/brouter-util/src/test/java/btools/util/LazyArrayOfLists.java deleted file mode 100644 index 850504b..0000000 --- a/brouter-util/src/test/java/btools/util/LazyArrayOfLists.java +++ /dev/null @@ -1,53 +0,0 @@ -package btools.util; - -import java.util.List; -import java.util.ArrayList; - -/** - * Behaves like an Array of list - * with lazy list-allocation at getList - * - * @author ab - */ -public class LazyArrayOfLists -{ - private ArrayList> lists; - - public LazyArrayOfLists( int size ) - { - lists = new ArrayList>( size ); - for ( int i = 0; i< size; i++ ) - { - lists.add( null ); - } - } - - public List getList( int idx ) - { - ArrayList list = lists.get( idx ); - if ( list == null ) - { - list = new ArrayList(); - lists.set( idx, list ); - } - return list; - } - - public int getSize( int idx ) - { - List list = lists.get( idx ); - return list == null ? 0 : list.size(); - } - - public void trimAll() - { - for ( int idx = 0; idx< lists.size(); idx++ ) - { - ArrayList list = lists.get( idx ); - if ( list != null ) - { - list.trimToSize(); - } - } - } -} diff --git a/brouter-util/src/test/java/btools/util/LongList.java b/brouter-util/src/test/java/btools/util/LongList.java deleted file mode 100644 index 2da9914..0000000 --- a/brouter-util/src/test/java/btools/util/LongList.java +++ /dev/null @@ -1,43 +0,0 @@ -package btools.util; - -/** - * dynamic list of primitive longs - * - * @author ab - */ -public class LongList -{ - private long[] a; - private int size; - - public LongList( int capacity ) - { - a = capacity < 4 ? new long[4] : new long[capacity]; - } - - public void add( long value ) - { - if ( size == a.length ) - { - long[] aa = new long[2*size]; - System.arraycopy( a, 0, aa, 0, size ); - a = aa; - } - a[size++] = value; - } - - public long get( int idx ) - { - if ( idx >= size ) - { - throw new IndexOutOfBoundsException( "list size=" + size + " idx=" + idx ); - } - return a[idx]; - } - - public int size() - { - return size; - } - -} diff --git a/brouter-util/src/test/java/btools/util/TinyDenseLongMap.java b/brouter-util/src/test/java/btools/util/TinyDenseLongMap.java deleted file mode 100644 index 472b3c3..0000000 --- a/brouter-util/src/test/java/btools/util/TinyDenseLongMap.java +++ /dev/null @@ -1,206 +0,0 @@ -package btools.util; - -/** - * TinyDenseLongMap implements the DenseLongMap interface - * but actually is made for a medium count of non-dense keys - * - * It's used as a replacement for DenseLongMap where we - * have limited memory and far less keys than maykey - * - * @author ab - */ -public class TinyDenseLongMap extends DenseLongMap -{ - private long[][] al; - private int[] pa; - private int size = 0; - private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays - - protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE - - public TinyDenseLongMap() - { - super(1); - - // pointer array - pa = new int[MAXLISTS]; - - // allocate key lists - al = new long[MAXLISTS][]; - al[0] = new long[1]; // make the first array (the transient buffer) - - // same for the values - vla = new byte[MAXLISTS][]; - vla[0] = new byte[1]; - } - - - private byte[][] vla; // value list array - - private void fillReturnValue(byte[] rv, int idx, int p ) - { - rv[0] = vla[idx][p]; - if ( rv.length == 2 ) - { - vla[idx][p] = rv[1]; - } - } - - @Override - public void put( long id, int value ) - { - byte[] rv = new byte[2]; - rv[1] = (byte)value; - if ( contains( id, rv ) ) - { - return; - } - - vla[0][0] = (byte)value; - _add( id ); - } - - - /** - * Get the byte for the given id - * @param id the key to query - * @return the object - * @exception IllegalArgumentException if id is unknown - */ - @Override - public int getInt( long id ) - { - byte[] rv = new byte[1]; - if ( contains( id, rv ) ) - { - return rv[0]; - } - return -1; - } - - - private boolean _add( long id ) - { - if ( size == Integer.MAX_VALUE ) - { - throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" ); - } - - // put the new entry in the first array - al[0][0] = id; - - // determine the first empty array - int bp = size++; // treat size as bitpattern - int idx = 1; - int n = 1; - - pa[0] = 1; - pa[1] = 1; - - while ( (bp&1) == 1 ) - { - bp >>= 1; - pa[idx++] = n; - n <<= 1; - } - - // create it if not existant - if ( al[idx] == null ) - { - al[idx] = new long[n]; - vla[idx] = new byte[n]; - } - - // now merge the contents of arrays 0...idx-1 into idx - while ( n > 0 ) - { - long maxId = 0; - int maxIdx = -1; - - for ( int i=0; i 0 ) - { - long currentId = al[i][p-1]; - if ( maxIdx < 0 || currentId > maxId ) - { - maxIdx = i; - maxId = currentId; - } - } - } - - // current maximum found, copy to target array - if ( n < al[idx].length && maxId == al[idx][n] ) - { - throw new IllegalArgumentException( "duplicate key found in late check: " + maxId ); - } - --n; - al[idx][n] = maxId; - vla[idx][n] = vla[maxIdx][pa[maxIdx]-1]; - - --pa[maxIdx]; - } - - // de-allocate empty arrays of a certain size (fix at 64kByte) - while ( idx-- > _maxKeepExponent ) - { - al[idx] = null; - vla[idx] = null; - } - - return false; - } - - - private boolean contains( long id, byte[] rv ) - { - // determine the first empty array - int bp = size; // treat size as bitpattern - int idx = 1; - - while ( bp != 0 ) - { - if ( (bp&1) == 1 ) - { - // array at idx is valid, check - if ( contains( idx, id, rv ) ) - { - return true; - } - } - idx++; - bp >>= 1; - } - return false; - } - - - // does sorted array "a" contain "id" ? - private boolean contains( int idx, long id, byte[] rv ) - { - long[] a = al[idx]; - int offset = a.length; - int n = 0; - - while ( (offset >>= 1) > 0 ) - { - int nn = n + offset; - if ( a[nn] <= id ) - { - n = nn; - } - } - if ( a[n] == id ) - { - if ( rv != null ) - { - fillReturnValue( rv, idx, n ); - } - return true; - } - return false; - } - -} diff --git a/misc/pbfparser/.gitignore b/misc/pbfparser/.gitignore index f3d4651..4dd82da 100644 --- a/misc/pbfparser/.gitignore +++ b/misc/pbfparser/.gitignore @@ -1,2 +1,3 @@ *.jar +*.BAK btools/