added crc logic for datafile access
This commit is contained in:
Arndt Brenschede
parent
e88465f5ec
commit
b4aa961f2d
26 changed files with 621 additions and 1364 deletions
|
|
@ -27,5 +27,9 @@
|
|||
<artifactId>brouter-expressions</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -10,4 +10,12 @@
|
|||
</parent>
|
||||
<artifactId>brouter-expressions</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.btools</groupId>
|
||||
<artifactId>brouter-util</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ import java.util.Map;
|
|||
import java.util.StringTokenizer;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import btools.util.Crc32;
|
||||
|
||||
public final class BExpressionContext
|
||||
{
|
||||
private static final String CONTEXT_TAG = "---context:";
|
||||
|
|
@ -228,40 +230,6 @@ public final class BExpressionContext
|
|||
}
|
||||
}
|
||||
|
||||
private static int[] crctable = {
|
||||
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
|
||||
0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
|
||||
0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
|
||||
0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
|
||||
0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
|
||||
0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
|
||||
0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
|
||||
0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
|
||||
0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
|
||||
0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
|
||||
0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
|
||||
0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
|
||||
0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
|
||||
0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
|
||||
0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
|
||||
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
|
||||
0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
|
||||
0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
|
||||
0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
|
||||
0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
|
||||
0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
|
||||
0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
|
||||
0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
|
||||
0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
|
||||
0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
|
||||
0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
|
||||
0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
|
||||
0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
|
||||
0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
|
||||
0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
|
||||
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
|
||||
0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d,
|
||||
};
|
||||
|
||||
|
||||
public void evaluate( long bitmap, BExpressionReceiver receiver )
|
||||
|
|
@ -271,13 +239,7 @@ public final class BExpressionContext
|
|||
if ( currentBitmap != bitmap || currentHashBucket < 0 )
|
||||
{
|
||||
// calc hash bucket from crc
|
||||
int crc = 0xFFFFFFFF;
|
||||
long bm = bitmap;
|
||||
for( int j=0; j<8; j++ )
|
||||
{
|
||||
crc = (crc >>> 8) ^ crctable[(crc ^ (int)bm) & 0xff];
|
||||
bm >>= 8;
|
||||
}
|
||||
int crc = Crc32.crc( bitmap );
|
||||
int hashSize = _arrayBitmap.length;
|
||||
currentHashBucket = (crc & 0xfffffff) % hashSize;
|
||||
currentBitmap = bitmap;
|
||||
|
|
|
|||
|
|
@ -31,6 +31,8 @@ public class WayLinker extends MapCreatorBase
|
|||
private CompactLongSet borderSet;
|
||||
private short lookupVersion;
|
||||
|
||||
private long creationTimeStamp;
|
||||
|
||||
private BExpressionContext expctxWay;
|
||||
|
||||
private int minLon;
|
||||
|
|
@ -68,6 +70,8 @@ public class WayLinker extends MapCreatorBase
|
|||
lookupVersion = expctxWay.lookupVersion;
|
||||
expctxWay.parseFile( profileFile, "global" );
|
||||
|
||||
creationTimeStamp = System.currentTimeMillis();
|
||||
|
||||
// then process all segments
|
||||
new WayIterator( this, true ).processDir( wayTilesIn, ".wt5" );
|
||||
}
|
||||
|
|
@ -88,7 +92,7 @@ public class WayLinker extends MapCreatorBase
|
|||
|
||||
// read this tile's nodes
|
||||
readingBorder = false;
|
||||
new NodeIterator( this, true ).processFile( nodeFile );
|
||||
new NodeIterator( this, false ).processFile( nodeFile );
|
||||
|
||||
// freeze the nodes-map
|
||||
FrozenLongMap<OsmNodeP> nodesMapFrozen = new FrozenLongMap<OsmNodeP>( nodesMap );
|
||||
|
|
@ -203,8 +207,10 @@ public class WayLinker extends MapCreatorBase
|
|||
File outfile = fileFromTemplate( wayfile, dataTilesOut, dataTilesSuffix );
|
||||
DataOutputStream os = createOutStream( outfile );
|
||||
|
||||
// write 5*5 index dummy
|
||||
long[] fileIndex = new long[25];
|
||||
int[] fileHeaderCrcs = new int[25];
|
||||
|
||||
// write 5*5 index dummy
|
||||
for( int i55=0; i55<25; i55++)
|
||||
{
|
||||
os.writeLong( 0 );
|
||||
|
|
@ -243,7 +249,7 @@ public class WayLinker extends MapCreatorBase
|
|||
|
||||
ByteArrayOutputStream bos = new ByteArrayOutputStream( );
|
||||
DataOutputStream dos = new DataOutputStream( bos );
|
||||
dos.writeInt( subList.size() );
|
||||
dos.writeInt( subList.size() + 1 ); // reserve 1 dummy node for crc
|
||||
for( int ni=0; ni<subList.size(); ni++ )
|
||||
{
|
||||
OsmNodeP n = subList.get(ni);
|
||||
|
|
@ -251,21 +257,25 @@ public class WayLinker extends MapCreatorBase
|
|||
}
|
||||
dos.close();
|
||||
byte[] subBytes = bos.toByteArray();
|
||||
pos += subBytes.length;
|
||||
pos += subBytes.length + 12; // reserve 12 bytes for crc dummy node
|
||||
subByteArrays[si] = subBytes;
|
||||
}
|
||||
posIdx[si] = pos;
|
||||
}
|
||||
|
||||
byte[] abSubIndex = compileSubFileIndex( posIdx );
|
||||
fileHeaderCrcs[tileIndex] = Crc32.crc( abSubIndex, 0, abSubIndex.length );
|
||||
os.write( abSubIndex, 0, abSubIndex.length );
|
||||
for( int si=0; si<6400; si++)
|
||||
{
|
||||
os.writeInt( posIdx[si] );
|
||||
}
|
||||
for( int si=0; si<6400; si++)
|
||||
{
|
||||
if ( subByteArrays[si] != null )
|
||||
byte[] ab = subByteArrays[si];
|
||||
if ( ab != null )
|
||||
{
|
||||
os.write( subByteArrays[si] );
|
||||
os.write( ab );
|
||||
os.writeShort( Short.MAX_VALUE ); // write crc as a dummy node for compatibility
|
||||
os.writeShort( Short.MAX_VALUE );
|
||||
os.writeInt( 4 );
|
||||
os.writeInt( Crc32.crc( ab, 0 , ab.length ) );
|
||||
}
|
||||
}
|
||||
filepos += pos;
|
||||
|
|
@ -273,17 +283,49 @@ public class WayLinker extends MapCreatorBase
|
|||
fileIndex[ tileIndex ] = filepos;
|
||||
}
|
||||
}
|
||||
|
||||
byte[] abFileIndex = compileFileIndex( fileIndex, lookupVersion );
|
||||
|
||||
// write extra data: timestamp + index-checksums
|
||||
os.writeLong( creationTimeStamp );
|
||||
os.writeInt( Crc32.crc( abFileIndex, 0, abFileIndex.length ) );
|
||||
for( int i55=0; i55<25; i55++)
|
||||
{
|
||||
os.writeInt( fileHeaderCrcs[i55] );
|
||||
}
|
||||
|
||||
os.close();
|
||||
|
||||
// re-open random-access to write file-index
|
||||
RandomAccessFile ra = new RandomAccessFile( outfile, "rw" );
|
||||
long versionPrefix = lookupVersion;
|
||||
versionPrefix <<= 48;
|
||||
for( int i55=0; i55<25; i55++)
|
||||
{
|
||||
ra.writeLong( fileIndex[i55] | versionPrefix );
|
||||
}
|
||||
ra.write( abFileIndex, 0, abFileIndex.length );
|
||||
ra.close();
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] compileFileIndex( long[] fileIndex, short lookupVersion ) throws Exception
|
||||
{
|
||||
ByteArrayOutputStream bos = new ByteArrayOutputStream( );
|
||||
DataOutputStream dos = new DataOutputStream( bos );
|
||||
long versionPrefix = lookupVersion;
|
||||
versionPrefix <<= 48;
|
||||
for( int i55=0; i55<25; i55++)
|
||||
{
|
||||
dos.writeLong( fileIndex[i55] | versionPrefix );
|
||||
}
|
||||
dos.close();
|
||||
return bos.toByteArray();
|
||||
}
|
||||
|
||||
private byte[] compileSubFileIndex( int[] posIdx ) throws Exception
|
||||
{
|
||||
ByteArrayOutputStream bos = new ByteArrayOutputStream( );
|
||||
DataOutputStream dos = new DataOutputStream( bos );
|
||||
for( int si=0; si<6400; si++)
|
||||
{
|
||||
dos.writeInt( posIdx[si] );
|
||||
}
|
||||
dos.close();
|
||||
return bos.toByteArray();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,70 +0,0 @@
|
|||
package btools.mapcreator;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.HashMap;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import java.net.URL;
|
||||
import java.io.File;
|
||||
|
||||
public class MapcreatorTest
|
||||
{
|
||||
@Test
|
||||
public void mapcreatorTest() throws Exception
|
||||
{
|
||||
URL mapurl = this.getClass().getResource( "/dreieich.osm.gz" );
|
||||
Assert.assertTrue( "test-osm-map dreieich.osm not found", mapurl != null );
|
||||
File mapfile = new File(mapurl.getFile());
|
||||
File workingDir = mapfile.getParentFile();
|
||||
File tmpdir = new File( workingDir, "tmp" );
|
||||
tmpdir.mkdir();
|
||||
|
||||
// run OsmCutter
|
||||
File nodetiles = new File( tmpdir, "nodetiles" );
|
||||
nodetiles.mkdir();
|
||||
File lookupFile = new File( workingDir, "lookups.dat" );
|
||||
File wayFile = new File( tmpdir, "ways.dat" );
|
||||
File relFile = new File( tmpdir, "cycleways.dat" );
|
||||
new OsmCutter().process( lookupFile, nodetiles, wayFile, relFile, mapfile );
|
||||
|
||||
// run NodeFilter
|
||||
File ftiles = new File( tmpdir, "ftiles" );
|
||||
ftiles.mkdir();
|
||||
new NodeFilter().process( nodetiles, wayFile, ftiles );
|
||||
|
||||
// run WayCutter
|
||||
File waytiles = new File( tmpdir, "waytiles" );
|
||||
waytiles.mkdir();
|
||||
new WayCutter().process( ftiles, wayFile, waytiles, relFile );
|
||||
|
||||
// run WayCutter5
|
||||
File waytiles55 = new File( tmpdir, "waytiles55" );
|
||||
File bordernids = new File( tmpdir, "bordernids.dat" );
|
||||
waytiles55.mkdir();
|
||||
new WayCutter5().process( ftiles, waytiles, waytiles55, bordernids );
|
||||
|
||||
// run NodeCutter
|
||||
File nodes55 = new File( tmpdir, "nodes55" );
|
||||
nodes55.mkdir();
|
||||
new NodeCutter().process( ftiles, nodes55 );
|
||||
|
||||
// run PosUnifier
|
||||
File unodes55 = new File( tmpdir, "unodes55" );
|
||||
File bordernodes = new File( tmpdir, "bordernodes.dat" );
|
||||
unodes55.mkdir();
|
||||
new PosUnifier().process( nodes55, unodes55, bordernids, bordernodes, "/private-backup/srtm" );
|
||||
|
||||
// run WayLinker
|
||||
File segments = new File( tmpdir, "segments" );
|
||||
segments.mkdir();
|
||||
File profileAllFile = new File( workingDir, "all.brf" );
|
||||
new WayLinker().process( unodes55, waytiles55, bordernodes, lookupFile, segments, "rd5" );
|
||||
|
||||
// run WayLinker, car subset
|
||||
File carsubset = new File( segments, "carsubset" );
|
||||
carsubset.mkdir();
|
||||
File profileCarFile = new File( workingDir, "car-test.brf" );
|
||||
new WayLinker().process( unodes55, waytiles55, bordernodes, lookupFile, carsubset, "cd5" );
|
||||
}
|
||||
}
|
||||
|
|
@ -10,4 +10,12 @@
|
|||
</parent>
|
||||
<artifactId>brouter-mapaccess</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.btools</groupId>
|
||||
<artifactId>brouter-util</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ package btools.mapaccess;
|
|||
import java.util.*;
|
||||
import java.io.*;
|
||||
|
||||
import btools.util.Crc32;
|
||||
|
||||
final class MicroCache
|
||||
{
|
||||
private long[] faid;
|
||||
|
|
@ -32,10 +34,10 @@ final class MicroCache
|
|||
|
||||
int subIdx = (latIdx80-80*latDegree)*80 + (lonIdx80-80*lonDegree);
|
||||
|
||||
try
|
||||
{
|
||||
ab = iobuffer;
|
||||
int asize = segfile.getDataInputForSubIdx(subIdx, ab);
|
||||
|
||||
if ( asize == 0 )
|
||||
{
|
||||
return;
|
||||
|
|
@ -48,8 +50,30 @@ final class MicroCache
|
|||
aboffset = 0;
|
||||
size = readInt();
|
||||
|
||||
// get net size
|
||||
int nbytes = 0;
|
||||
for(int i = 0; i<size; i++)
|
||||
{
|
||||
int ilon = readShort();
|
||||
int ilat = readShort();
|
||||
int bodySize = readInt();
|
||||
if ( ilon == Short.MAX_VALUE )
|
||||
{
|
||||
int crc = Crc32.crc( ab, 0, aboffset-8 );
|
||||
if ( crc != readInt() )
|
||||
{
|
||||
throw new IOException( "checkum error" );
|
||||
}
|
||||
size = i;
|
||||
break;
|
||||
}
|
||||
aboffset += bodySize;
|
||||
nbytes += bodySize;
|
||||
}
|
||||
|
||||
// new array with only net data
|
||||
byte[] nab = new byte[asize - 4 - size*8];
|
||||
byte[] nab = new byte[nbytes];
|
||||
aboffset = 4;
|
||||
int noffset = 0;
|
||||
faid = new long[size];
|
||||
fapos = new int[size];
|
||||
|
|
@ -71,11 +95,9 @@ final class MicroCache
|
|||
aboffset += bodySize;
|
||||
noffset += bodySize;
|
||||
}
|
||||
|
||||
ab = nab;
|
||||
}
|
||||
catch( EOFException eof )
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
public int getSize()
|
||||
|
|
|
|||
|
|
@ -16,8 +16,7 @@ public final class NodesCache
|
|||
private boolean carMode;
|
||||
private String currentFileName;
|
||||
|
||||
private HashMap<String,RandomAccessFile> fileCache;
|
||||
private HashMap<String,long[]> indexCache;
|
||||
private HashMap<String,PhysicalFile> fileCache;
|
||||
private byte[] iobuffer;
|
||||
|
||||
private OsmFile[][] fileRows = new OsmFile[180][];
|
||||
|
|
@ -37,14 +36,12 @@ public final class NodesCache
|
|||
if ( oldCache != null )
|
||||
{
|
||||
fileCache = oldCache.fileCache;
|
||||
indexCache = oldCache.indexCache;
|
||||
iobuffer = oldCache.iobuffer;
|
||||
oom_carsubset_hint = oldCache.oom_carsubset_hint;
|
||||
}
|
||||
else
|
||||
{
|
||||
fileCache = new HashMap<String,RandomAccessFile>(4);
|
||||
indexCache = new HashMap<String,long[]>(4);
|
||||
fileCache = new HashMap<String,PhysicalFile>(4);
|
||||
iobuffer = new byte[65636];
|
||||
}
|
||||
}
|
||||
|
|
@ -144,6 +141,7 @@ public final class NodesCache
|
|||
|
||||
currentFileName = filenameBase + ".rd5/cd5";
|
||||
|
||||
PhysicalFile ra = null;
|
||||
if ( !fileCache.containsKey( filenameBase ) )
|
||||
{
|
||||
File f = null;
|
||||
|
|
@ -158,39 +156,17 @@ public final class NodesCache
|
|||
if ( fullFile.exists() ) f = fullFile;
|
||||
if ( carMode && f != null ) oom_carsubset_hint = true;
|
||||
}
|
||||
RandomAccessFile ra = f != null ? new RandomAccessFile( f, "r" ) : null;
|
||||
fileCache.put( filenameBase, ra );
|
||||
if ( ra != null )
|
||||
if ( f != null )
|
||||
{
|
||||
long[] fileIndex = new long[25];
|
||||
ra.readFully( iobuffer, 0, 200 );
|
||||
ByteDataReader dis = new ByteDataReader( iobuffer );
|
||||
for( int i=0; i<25; i++ )
|
||||
{
|
||||
long lv = dis.readLong();
|
||||
short readVersion = (short)(lv >> 48);
|
||||
if ( readVersion != lookupVersion )
|
||||
{
|
||||
throw new IllegalArgumentException( "lookup version mismatch (old rd5?) lookups.dat="
|
||||
+ lookupVersion + " " + f. getAbsolutePath() + "=" + readVersion );
|
||||
}
|
||||
fileIndex[i] = lv & 0xffffffffffffL;
|
||||
}
|
||||
indexCache.put( filenameBase, fileIndex );
|
||||
currentFileName = f.getName();
|
||||
ra = new PhysicalFile( f, iobuffer, lookupVersion );
|
||||
}
|
||||
fileCache.put( filenameBase, ra );
|
||||
}
|
||||
RandomAccessFile ra = fileCache.get( filenameBase );
|
||||
long startPos = 0L;
|
||||
if ( ra != null )
|
||||
{
|
||||
long[] index = indexCache.get( filenameBase );
|
||||
startPos = tileIndex > 0 ? index[ tileIndex-1 ] : 200L;
|
||||
if ( startPos == index[ tileIndex] ) ra = null;
|
||||
}
|
||||
OsmFile osmf = new OsmFile( ra, startPos, iobuffer );
|
||||
ra = fileCache.get( filenameBase );
|
||||
OsmFile osmf = new OsmFile( ra, tileIndex, iobuffer );
|
||||
osmf.lonDegree = lonDegree;
|
||||
osmf.latDegree = latDegree;
|
||||
osmf.filename = currentFileName;
|
||||
return osmf;
|
||||
}
|
||||
|
||||
|
|
@ -208,11 +184,11 @@ public final class NodesCache
|
|||
|
||||
public void close()
|
||||
{
|
||||
for( RandomAccessFile f: fileCache.values() )
|
||||
for( PhysicalFile f: fileCache.values() )
|
||||
{
|
||||
try
|
||||
{
|
||||
f.close();
|
||||
f.ra.close();
|
||||
}
|
||||
catch( IOException ioe )
|
||||
{
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ package btools.mapaccess;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import btools.util.Crc32;
|
||||
|
||||
final class OsmFile
|
||||
{
|
||||
|
|
@ -21,16 +22,31 @@ final class OsmFile
|
|||
|
||||
public String filename;
|
||||
|
||||
public OsmFile( RandomAccessFile rafile, long startPos, byte[] iobuffer ) throws Exception
|
||||
public OsmFile( PhysicalFile rafile, int tileIndex, byte[] iobuffer ) throws Exception
|
||||
{
|
||||
fileOffset = startPos;
|
||||
if ( rafile != null )
|
||||
{
|
||||
is = rafile;
|
||||
filename = rafile.fileName;
|
||||
|
||||
long[] index = rafile.fileIndex;
|
||||
fileOffset = tileIndex > 0 ? index[ tileIndex-1 ] : 200L;
|
||||
if ( fileOffset == index[ tileIndex] ) return; // empty
|
||||
|
||||
is = rafile.ra;
|
||||
posIdx = new int[6400];
|
||||
microCaches = new MicroCache[6400];
|
||||
is.seek( fileOffset );
|
||||
is.readFully( iobuffer, 0, 25600 );
|
||||
|
||||
if ( rafile.fileHeaderCrcs != null )
|
||||
{
|
||||
int headerCrc = Crc32.crc( iobuffer, 0, 25600 );
|
||||
if ( rafile.fileHeaderCrcs[tileIndex] != headerCrc )
|
||||
{
|
||||
throw new IOException( "sub index checksum error" );
|
||||
}
|
||||
}
|
||||
|
||||
ByteDataReader dis = new ByteDataReader( iobuffer );
|
||||
for( int i=0; i<6400; i++ )
|
||||
{
|
||||
|
|
@ -54,7 +70,7 @@ final class OsmFile
|
|||
is.seek( fileOffset + startPos );
|
||||
if ( size <= iobuffer.length )
|
||||
{
|
||||
is.readFully( iobuffer );
|
||||
is.readFully( iobuffer, 0, size );
|
||||
}
|
||||
}
|
||||
return size;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
/**
|
||||
* cache for a single square
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
package btools.mapaccess;
|
||||
|
||||
import java.io.*;
|
||||
import btools.util.Crc32;
|
||||
|
||||
final class PhysicalFile
|
||||
{
|
||||
RandomAccessFile ra = null;
|
||||
long[] fileIndex = new long[25];
|
||||
int[] fileHeaderCrcs;
|
||||
|
||||
private int fileIndexCrc;
|
||||
public long creationTime;
|
||||
|
||||
String fileName;
|
||||
|
||||
public PhysicalFile( File f, byte[] iobuffer, int lookupVersion ) throws Exception
|
||||
{
|
||||
fileName = f.getName();
|
||||
|
||||
ra = new RandomAccessFile( f, "r" );
|
||||
ra.readFully( iobuffer, 0, 200 );
|
||||
fileIndexCrc = Crc32.crc( iobuffer, 0, 200 );
|
||||
ByteDataReader dis = new ByteDataReader( iobuffer );
|
||||
for( int i=0; i<25; i++ )
|
||||
{
|
||||
long lv = dis.readLong();
|
||||
short readVersion = (short)(lv >> 48);
|
||||
if ( readVersion != lookupVersion )
|
||||
{
|
||||
throw new IllegalArgumentException( "lookup version mismatch (old rd5?) lookups.dat="
|
||||
+ lookupVersion + " " + f. getAbsolutePath() + "=" + readVersion );
|
||||
}
|
||||
fileIndex[i] = lv & 0xffffffffffffL;
|
||||
}
|
||||
|
||||
// read some extra info from the end of the file, if present
|
||||
long len = ra.length();
|
||||
|
||||
long pos = fileIndex[24];
|
||||
int extraLen = 8 + 26*4;
|
||||
|
||||
if ( len == pos ) return; // old format o.k.
|
||||
|
||||
if ( len < pos+extraLen ) // > is o.k. for future extensions!
|
||||
{
|
||||
throw new IOException( "file of size " + len + " + too short, should be " + (pos+extraLen) );
|
||||
}
|
||||
|
||||
ra.seek( pos );
|
||||
ra.readFully( iobuffer, 0, extraLen );
|
||||
dis = new ByteDataReader( iobuffer );
|
||||
creationTime = dis.readLong();
|
||||
if ( dis.readInt() != fileIndexCrc )
|
||||
{
|
||||
throw new IOException( "top index checksum error" );
|
||||
}
|
||||
fileHeaderCrcs = new int[25];
|
||||
for( int i=0; i<25; i++ )
|
||||
{
|
||||
fileHeaderCrcs[i] = dis.readInt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -51,5 +51,9 @@
|
|||
<artifactId>brouter-map-creator</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
|
|
|||
56
brouter-server/src/test/java/btools/server/RouterTest.java
Normal file
56
brouter-server/src/test/java/btools/server/RouterTest.java
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
package btools.server;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import java.net.URL;
|
||||
import java.io.File;
|
||||
|
||||
import btools.router.*;
|
||||
import btools.mapaccess.*;
|
||||
|
||||
public class RouterTest
|
||||
{
|
||||
@Test
|
||||
public void routerTest() throws Exception
|
||||
{
|
||||
URL resulturl = this.getClass().getResource( "/testtrack0.gpx" );
|
||||
Assert.assertTrue( "reference result not found: ", resulturl != null );
|
||||
File resultfile = new File(resulturl.getFile());
|
||||
File workingDir = resultfile.getParentFile();
|
||||
|
||||
String wd = workingDir.getAbsolutePath();
|
||||
|
||||
List<OsmNodeNamed> wplist = new ArrayList<OsmNodeNamed>();
|
||||
OsmNodeNamed n;
|
||||
n = new OsmNodeNamed();
|
||||
n.name = "from";
|
||||
n.ilon = 180000000 + 8720897;
|
||||
n.ilat = 90000000 + 50002515;
|
||||
wplist.add( n );
|
||||
|
||||
n = new OsmNodeNamed();
|
||||
n.name = "to";
|
||||
n.ilon = 180000000 + 8723658;
|
||||
n.ilat = 90000000 + 49997510;
|
||||
wplist.add( n );
|
||||
|
||||
RoutingContext rctx = new RoutingContext();
|
||||
rctx.localFunction = wd + "/../../../misc/profiles2/trekking.brf";
|
||||
// c.setAlternativeIdx( 1 );
|
||||
|
||||
RoutingEngine re = new RoutingEngine(
|
||||
wd + "/testtrack",
|
||||
wd + "/testlog",
|
||||
wd + "/../../../brouter-map-creator/target/test-classes/tmp/segments", wplist, rctx );
|
||||
re.doRun( 0 );
|
||||
|
||||
// error message from router?
|
||||
Assert.assertTrue( "routing failed: " + re.getErrorMessage(), re.getErrorMessage() == null );
|
||||
|
||||
// if the track didn't change, we expect the first alternative also
|
||||
File a1 = new File( workingDir, "testtrack1.gpx" );
|
||||
Assert.assertTrue( "result content missmatch", a1.exists() );
|
||||
}
|
||||
}
|
||||
44
brouter-server/src/test/resources/testtrack0.gpx
Normal file
44
brouter-server/src/test/resources/testtrack0.gpx
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- track-length = 728 filtered ascend = 0 plain-ascend = 0 cost=728 -->
|
||||
<gpx
|
||||
xmlns="http://www.topografix.com/GPX/1/1"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd"
|
||||
creator="BRouter-0.98" version="1.1">
|
||||
<trk>
|
||||
<name>brouter_trekking_0</name>
|
||||
<trkseg>
|
||||
<trkpt lon="8.720889" lat="50.002512"></trkpt>
|
||||
<trkpt lon="8.720897" lat="50.002503"></trkpt>
|
||||
<trkpt lon="8.720976" lat="50.002389"></trkpt>
|
||||
<trkpt lon="8.720787" lat="50.002390"></trkpt>
|
||||
<trkpt lon="8.720662" lat="50.002363"></trkpt>
|
||||
<trkpt lon="8.720553" lat="50.002331"></trkpt>
|
||||
<trkpt lon="8.720339" lat="50.002251"></trkpt>
|
||||
<trkpt lon="8.720158" lat="50.002159"></trkpt>
|
||||
<trkpt lon="8.719973" lat="50.002051"></trkpt>
|
||||
<trkpt lon="8.719729" lat="50.001823"></trkpt>
|
||||
<trkpt lon="8.719541" lat="50.001606"></trkpt>
|
||||
<trkpt lon="8.719807" lat="50.001508"></trkpt>
|
||||
<trkpt lon="8.719989" lat="50.001421"></trkpt>
|
||||
<trkpt lon="8.720085" lat="50.001367"></trkpt>
|
||||
<trkpt lon="8.720324" lat="50.001211"></trkpt>
|
||||
<trkpt lon="8.720482" lat="50.001041"></trkpt>
|
||||
<trkpt lon="8.720597" lat="50.000795"></trkpt>
|
||||
<trkpt lon="8.720672" lat="50.000551"></trkpt>
|
||||
<trkpt lon="8.720760" lat="50.000387"></trkpt>
|
||||
<trkpt lon="8.720921" lat="50.000228"></trkpt>
|
||||
<trkpt lon="8.721129" lat="50.000074"></trkpt>
|
||||
<trkpt lon="8.721409" lat="49.999885"></trkpt>
|
||||
<trkpt lon="8.721622" lat="49.999733"></trkpt>
|
||||
<trkpt lon="8.722176" lat="49.999309"></trkpt>
|
||||
<trkpt lon="8.722416" lat="49.999100"></trkpt>
|
||||
<trkpt lon="8.722474" lat="49.999038"></trkpt>
|
||||
<trkpt lon="8.722547" lat="49.998975"></trkpt>
|
||||
<trkpt lon="8.722669" lat="49.998853"></trkpt>
|
||||
<trkpt lon="8.723046" lat="49.998397"></trkpt>
|
||||
<trkpt lon="8.723659" lat="49.997526"></trkpt>
|
||||
<trkpt lon="8.723669" lat="49.997514"></trkpt>
|
||||
</trkseg>
|
||||
</trk>
|
||||
</gpx>
|
||||
67
brouter-util/src/main/java/btools/util/Crc32.java
Normal file
67
brouter-util/src/main/java/btools/util/Crc32.java
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* checksum stuff
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class Crc32
|
||||
{
|
||||
public static int crc( long bitmap )
|
||||
{
|
||||
int crc = 0xFFFFFFFF;
|
||||
long bm = bitmap;
|
||||
for( int j=0; j<8; j++ )
|
||||
{
|
||||
crc = (crc >>> 8) ^ crctable[(crc ^ (int)bm) & 0xff];
|
||||
bm >>= 8;
|
||||
}
|
||||
return crc;
|
||||
}
|
||||
|
||||
public static int crc( byte[] ab, int offset, int len )
|
||||
{
|
||||
int crc = 0xFFFFFFFF;
|
||||
for( int j=offset; j<len; j++ )
|
||||
{
|
||||
crc = (crc >>> 8) ^ crctable[(crc ^ ab[j]) & 0xff];
|
||||
}
|
||||
return crc;
|
||||
}
|
||||
|
||||
private static int[] crctable = {
|
||||
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
|
||||
0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
|
||||
0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
|
||||
0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
|
||||
0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
|
||||
0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
|
||||
0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
|
||||
0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
|
||||
0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
|
||||
0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
|
||||
0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
|
||||
0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
|
||||
0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
|
||||
0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
|
||||
0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
|
||||
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
|
||||
0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
|
||||
0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
|
||||
0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
|
||||
0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
|
||||
0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
|
||||
0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
|
||||
0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
|
||||
0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
|
||||
0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
|
||||
0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
|
||||
0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
|
||||
0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
|
||||
0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
|
||||
0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
|
||||
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
|
||||
0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d,
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -1,328 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
/**
|
||||
* Memory efficient Map to map a long-key to an object-value
|
||||
*
|
||||
* Implementation is such that basically the 12 bytes
|
||||
* per entry is allocated that's needed to store
|
||||
* a long- and an object-value.
|
||||
* This class does not implement the Map interface
|
||||
* because it's not complete (remove() is not implemented,
|
||||
* CompactLongMap can only grow.)
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class CompactLongMap<V>
|
||||
{
|
||||
private long[][] al;
|
||||
private int[] pa;
|
||||
private int size = 0;
|
||||
private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
|
||||
|
||||
private V value_in;
|
||||
protected V value_out;
|
||||
|
||||
protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
|
||||
private static boolean earlyDuplicateCheck;
|
||||
|
||||
public CompactLongMap()
|
||||
{
|
||||
// pointer array
|
||||
pa = new int[MAXLISTS];
|
||||
|
||||
// allocate key lists
|
||||
al = new long[MAXLISTS][];
|
||||
al[0] = new long[1]; // make the first array (the transient buffer)
|
||||
|
||||
// same for the values
|
||||
vla = new Object[MAXLISTS][];
|
||||
vla[0] = new Object[1];
|
||||
|
||||
earlyDuplicateCheck = Boolean.getBoolean( "earlyDuplicateCheck" );
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
*
|
||||
* The Map extension:
|
||||
* next 5 protected methods are needed to implement value-support
|
||||
* overwrite them all to support value structures other than the
|
||||
* long-values implemented here as a sample.
|
||||
*
|
||||
* Furthermore, put() and get() method need to be implemented
|
||||
* to access the values.
|
||||
*
|
||||
* Note that this map does not behave exactly like java.util.Map
|
||||
* - put(..) with already existing key throws exception
|
||||
* - get(..) with non-existing key thros exception
|
||||
*
|
||||
* If you have keys that cannot easily be mapped on long's, use
|
||||
* a hash-function to do the mapping. But note that, in comparison
|
||||
* to java.util.HashMap, in that case the keys itself are not saved,
|
||||
* only the hash-values, so you need to be sure that random duplicate
|
||||
* hashs are either excluded by the structure of your data or that
|
||||
* you can handle the possible IllegalArgumentException
|
||||
*
|
||||
*/
|
||||
|
||||
private Object[][] vla; // value list array
|
||||
|
||||
|
||||
public boolean put( long id, V value )
|
||||
{
|
||||
try
|
||||
{
|
||||
value_in = value;
|
||||
if ( contains( id, true ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
vla[0][0] = value;
|
||||
_add( id );
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
value_in = null;
|
||||
value_out = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Same as put( id, value ) but duplicate check
|
||||
* is skipped for performance. Be aware that you
|
||||
* can get a duplicate exception later on if the
|
||||
* map is restructured!
|
||||
* with System parameter earlyDuplicateCheck=true you
|
||||
* can enforce the early duplicate check for debugging
|
||||
*
|
||||
* @param id the key to insert
|
||||
* @param value the value to insert object
|
||||
* @exception IllegalArgumentException for duplicates if enabled
|
||||
*/
|
||||
public void fastPut( long id, V value )
|
||||
{
|
||||
if ( earlyDuplicateCheck && contains( id ) )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in early check: " + id );
|
||||
}
|
||||
vla[0][0] = value;
|
||||
_add( id );
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the value for the given id
|
||||
* @param id the key to query
|
||||
* @return the object, or null if id not known
|
||||
*/
|
||||
public V get( long id )
|
||||
{
|
||||
try
|
||||
{
|
||||
if ( contains( id, false ) )
|
||||
{
|
||||
return value_out;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
value_out = null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return the number of entries in this map
|
||||
*/
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
|
||||
private boolean _add( long id )
|
||||
{
|
||||
if ( size == Integer.MAX_VALUE )
|
||||
{
|
||||
throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
|
||||
}
|
||||
|
||||
// put the new entry in the first array
|
||||
al[0][0] = id;
|
||||
|
||||
// determine the first empty array
|
||||
int bp = size++; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
int n = 1;
|
||||
|
||||
pa[0] = 1;
|
||||
pa[1] = 1;
|
||||
|
||||
while ( (bp&1) == 1 )
|
||||
{
|
||||
bp >>= 1;
|
||||
pa[idx++] = n;
|
||||
n <<= 1;
|
||||
}
|
||||
|
||||
// create it if not existant
|
||||
if ( al[idx] == null )
|
||||
{
|
||||
al[idx] = new long[n];
|
||||
vla[idx] = new Object[n];
|
||||
}
|
||||
|
||||
// now merge the contents of arrays 0...idx-1 into idx
|
||||
while ( n > 0 )
|
||||
{
|
||||
long maxId = 0;
|
||||
int maxIdx = -1;
|
||||
|
||||
for ( int i=0; i<idx; i++ )
|
||||
{
|
||||
int p = pa[i];
|
||||
if ( p > 0 )
|
||||
{
|
||||
long currentId = al[i][p-1];
|
||||
if ( maxIdx < 0 || currentId > maxId )
|
||||
{
|
||||
maxIdx = i;
|
||||
maxId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// current maximum found, copy to target array
|
||||
if ( n < al[idx].length && maxId == al[idx][n] )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
|
||||
}
|
||||
--n;
|
||||
al[idx][n] = maxId;
|
||||
vla[idx][n] = vla[maxIdx][pa[maxIdx]-1];
|
||||
|
||||
--pa[maxIdx];
|
||||
}
|
||||
|
||||
// de-allocate empty arrays of a certain size (fix at 64kByte)
|
||||
while ( idx-- > _maxKeepExponent )
|
||||
{
|
||||
al[idx] = null;
|
||||
vla[idx] = null;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if "id" is contained in this set.
|
||||
*/
|
||||
public boolean contains( long id )
|
||||
{
|
||||
try
|
||||
{
|
||||
return contains( id, false );
|
||||
}
|
||||
finally
|
||||
{
|
||||
value_out = null;
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean contains( long id, boolean doPut )
|
||||
{
|
||||
// determine the first empty array
|
||||
int bp = size; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
// array at idx is valid, check
|
||||
if ( contains( idx, id, doPut ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// does sorted array "a" contain "id" ?
|
||||
private boolean contains( int idx, long id, boolean doPut )
|
||||
{
|
||||
long[] a = al[idx];
|
||||
int offset = a.length;
|
||||
int n = 0;
|
||||
|
||||
while ( (offset >>= 1) > 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
value_out = (V)vla[idx][n];
|
||||
if ( doPut ) vla[idx][n] = value_in;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
protected void moveToFrozenArrays( long[] faid, ArrayList<V> flv )
|
||||
{
|
||||
for( int i=1; i<MAXLISTS; i++ )
|
||||
{
|
||||
pa[i] = 0;
|
||||
}
|
||||
|
||||
for( int ti = 0; ti < size; ti++ ) // target-index
|
||||
{
|
||||
int bp = size; // treat size as bitpattern
|
||||
int minIdx = -1;
|
||||
long minId = 0;
|
||||
int idx = 1;
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
int p = pa[idx];
|
||||
if ( p < al[idx].length )
|
||||
{
|
||||
long currentId = al[idx][p];
|
||||
if ( minIdx < 0 || currentId < minId )
|
||||
{
|
||||
minIdx = idx;
|
||||
minId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
faid[ti] = minId;
|
||||
flv.add( (V)vla[minIdx][pa[minIdx]] );
|
||||
pa[minIdx]++;
|
||||
|
||||
if ( ti > 0 && faid[ti-1] == minId )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + minId );
|
||||
}
|
||||
}
|
||||
|
||||
// free the non-frozen arrays
|
||||
al = null;
|
||||
vla = null;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,220 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* Memory efficient Set for long-keys
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class CompactLongSet
|
||||
{
|
||||
private long[][] al;
|
||||
private int[] pa;
|
||||
private int size = 0;
|
||||
private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
|
||||
|
||||
protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
|
||||
private static boolean earlyDuplicateCheck;
|
||||
|
||||
public CompactLongSet()
|
||||
{
|
||||
// pointer array
|
||||
pa = new int[MAXLISTS];
|
||||
|
||||
// allocate key lists
|
||||
al = new long[MAXLISTS][];
|
||||
al[0] = new long[1]; // make the first array (the transient buffer)
|
||||
|
||||
earlyDuplicateCheck = Boolean.getBoolean( "earlyDuplicateCheck" );
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return the number of entries in this set
|
||||
*/
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
/**
|
||||
* add a long value to this set if not yet in.
|
||||
* @param id the value to add to this set.
|
||||
* @return true if "id" already contained in this set.
|
||||
*/
|
||||
public boolean add( long id )
|
||||
{
|
||||
if ( contains( id ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
_add( id );
|
||||
return false;
|
||||
}
|
||||
|
||||
public void fastAdd( long id )
|
||||
{
|
||||
_add( id );
|
||||
}
|
||||
|
||||
private void _add( long id )
|
||||
{
|
||||
if ( size == Integer.MAX_VALUE )
|
||||
{
|
||||
throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
|
||||
}
|
||||
|
||||
// put the new entry in the first array
|
||||
al[0][0] = id;
|
||||
|
||||
// determine the first empty array
|
||||
int bp = size++; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
int n = 1;
|
||||
|
||||
pa[0] = 1;
|
||||
pa[1] = 1;
|
||||
|
||||
while ( (bp&1) == 1 )
|
||||
{
|
||||
bp >>= 1;
|
||||
pa[idx++] = n;
|
||||
n <<= 1;
|
||||
}
|
||||
|
||||
// create it if not existant
|
||||
if ( al[idx] == null )
|
||||
{
|
||||
al[idx] = new long[n];
|
||||
}
|
||||
|
||||
// now merge the contents of arrays 0...idx-1 into idx
|
||||
while ( n > 0 )
|
||||
{
|
||||
long maxId = 0;
|
||||
int maxIdx = -1;
|
||||
|
||||
for ( int i=0; i<idx; i++ )
|
||||
{
|
||||
int p = pa[i];
|
||||
if ( p > 0 )
|
||||
{
|
||||
long currentId = al[i][p-1];
|
||||
if ( maxIdx < 0 || currentId > maxId )
|
||||
{
|
||||
maxIdx = i;
|
||||
maxId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// current maximum found, copy to target array
|
||||
if ( n < al[idx].length && maxId == al[idx][n] )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
|
||||
}
|
||||
--n;
|
||||
al[idx][n] = maxId;
|
||||
|
||||
--pa[maxIdx];
|
||||
}
|
||||
|
||||
// de-allocate empty arrays of a certain size (fix at 64kByte)
|
||||
while ( idx-- > _maxKeepExponent )
|
||||
{
|
||||
al[idx] = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if "id" is contained in this set.
|
||||
*/
|
||||
public boolean contains( long id )
|
||||
{
|
||||
// determine the first empty array
|
||||
int bp = size; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
// array at idx is valid, check
|
||||
if ( contains( idx, id ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// does sorted array "a" contain "id" ?
|
||||
private boolean contains( int idx, long id )
|
||||
{
|
||||
long[] a = al[idx];
|
||||
int offset = a.length;
|
||||
int n = 0;
|
||||
|
||||
while ( (offset >>= 1) > 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
protected void moveToFrozenArray( long[] faid )
|
||||
{
|
||||
for( int i=1; i<MAXLISTS; i++ )
|
||||
{
|
||||
pa[i] = 0;
|
||||
}
|
||||
|
||||
for( int ti = 0; ti < size; ti++ ) // target-index
|
||||
{
|
||||
int bp = size; // treat size as bitpattern
|
||||
int minIdx = -1;
|
||||
long minId = 0;
|
||||
int idx = 1;
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
int p = pa[idx];
|
||||
if ( p < al[idx].length )
|
||||
{
|
||||
long currentId = al[idx][p];
|
||||
if ( minIdx < 0 || currentId < minId )
|
||||
{
|
||||
minIdx = idx;
|
||||
minId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
faid[ti] = minId;
|
||||
pa[minIdx]++;
|
||||
|
||||
if ( ti > 0 && faid[ti-1] == minId )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + minId );
|
||||
}
|
||||
}
|
||||
|
||||
// free the non-frozen array
|
||||
al = null;
|
||||
}
|
||||
|
||||
}
|
||||
68
brouter-util/src/test/java/btools/util/CompactMapTest.java
Normal file
68
brouter-util/src/test/java/btools/util/CompactMapTest.java
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.HashMap;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class CompactMapTest
|
||||
{
|
||||
@Test
|
||||
public void hashMapComparisonTest()
|
||||
{
|
||||
hashMapComparison( 0, 1 );
|
||||
hashMapComparison( 1, 1 );
|
||||
hashMapComparison( 2, 2 );
|
||||
hashMapComparison( 3, 3 );
|
||||
hashMapComparison( 4, 4 );
|
||||
hashMapComparison( 5, 5 );
|
||||
hashMapComparison( 7, 10 );
|
||||
hashMapComparison( 8, 10 );
|
||||
hashMapComparison( 10000, 20000 );
|
||||
}
|
||||
|
||||
private void hashMapComparison( int mapsize, int trycount )
|
||||
{
|
||||
Random rand = new Random( 12345 );
|
||||
HashMap<Long,String> hmap = new HashMap<Long,String>();
|
||||
CompactLongMap<String> cmap_slow = new CompactLongMap<String>();
|
||||
CompactLongMap<String> cmap_fast = new CompactLongMap<String>();
|
||||
|
||||
for( int i=0; i<mapsize; i++ )
|
||||
{
|
||||
String s = "" + i;
|
||||
long k = mapsize < 10 ? i : rand.nextInt( 20000 );
|
||||
Long KK = new Long( k );
|
||||
|
||||
if ( !hmap.containsKey( KK ) )
|
||||
{
|
||||
hmap.put( KK, s );
|
||||
cmap_slow.put( k, s );
|
||||
cmap_fast.fastPut( k, s );
|
||||
}
|
||||
}
|
||||
|
||||
for( int i=0; i<trycount*2; i++ )
|
||||
{
|
||||
if ( i == trycount )
|
||||
{
|
||||
cmap_slow = new FrozenLongMap<String>( cmap_slow );
|
||||
cmap_fast = new FrozenLongMap<String>( cmap_fast );
|
||||
}
|
||||
long k = mapsize < 10 ? i : rand.nextInt( 20000 );
|
||||
Long KK = new Long( k );
|
||||
String s = hmap.get( KK );
|
||||
|
||||
boolean contained = hmap.containsKey( KK );
|
||||
Assert.assertTrue( "containsKey missmatch (slow)", contained == cmap_slow.contains( k ) );
|
||||
Assert.assertTrue( "containsKey missmatch (fast)", contained == cmap_fast.contains( k ) );
|
||||
|
||||
if ( contained )
|
||||
{
|
||||
Assert.assertEquals( "object missmatch (fast)", s, cmap_fast.get( k ) );
|
||||
Assert.assertEquals( "object missmatch (slow)", s, cmap_slow.get( k ) );
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
60
brouter-util/src/test/java/btools/util/CompactSetTest.java
Normal file
60
brouter-util/src/test/java/btools/util/CompactSetTest.java
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.HashSet;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class CompactSetTest
|
||||
{
|
||||
@Test
|
||||
public void hashSetComparisonTest()
|
||||
{
|
||||
hashSetComparison( 0, 1 );
|
||||
hashSetComparison( 1, 1 );
|
||||
hashSetComparison( 2, 2 );
|
||||
hashSetComparison( 3, 3 );
|
||||
hashSetComparison( 4, 4 );
|
||||
hashSetComparison( 5, 5 );
|
||||
hashSetComparison( 7, 10 );
|
||||
hashSetComparison( 8, 10 );
|
||||
hashSetComparison( 10000, 20000 );
|
||||
}
|
||||
|
||||
private void hashSetComparison( int setsize, int trycount )
|
||||
{
|
||||
Random rand = new Random( 12345 );
|
||||
HashSet<Long> hset = new HashSet<Long>();
|
||||
CompactLongSet cset_slow = new CompactLongSet();
|
||||
CompactLongSet cset_fast = new CompactLongSet();
|
||||
|
||||
for( int i=0; i<setsize; i++ )
|
||||
{
|
||||
long k = setsize < 10 ? i : rand.nextInt( 20000 );
|
||||
Long KK = new Long( k );
|
||||
|
||||
if ( !hset.contains( KK ) )
|
||||
{
|
||||
hset.add( KK );
|
||||
cset_slow.add( k );
|
||||
cset_fast.fastAdd( k );
|
||||
}
|
||||
}
|
||||
|
||||
for( int i=0; i<trycount*2; i++ )
|
||||
{
|
||||
if ( i == trycount )
|
||||
{
|
||||
cset_slow = new FrozenLongSet( cset_slow );
|
||||
cset_fast = new FrozenLongSet( cset_fast );
|
||||
}
|
||||
long k = setsize < 10 ? i : rand.nextInt( 20000 );
|
||||
Long KK = new Long( k );
|
||||
|
||||
boolean contained = hset.contains( KK );
|
||||
Assert.assertTrue( "contains missmatch (slow)", contained == cset_slow.contains( k ) );
|
||||
Assert.assertTrue( "contains missmatch (fast)", contained == cset_fast.contains( k ) );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,140 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
/**
|
||||
* Special Memory efficient Map to map a long-key to
|
||||
* a "small" value (some bits only) where it is expected
|
||||
* that the keys are dense, so that we can use more or less
|
||||
* a simple array as the best-fit data model (except for
|
||||
* the 32-bit limit of arrays!)
|
||||
*
|
||||
* Additionally, to enable small-memory unit testing
|
||||
* of code using this map this one has a fallback for a small map
|
||||
* where we have only few keys, but not dense. In this
|
||||
* case, we use the mechanics of the CompactLongMap
|
||||
*
|
||||
* Target application are osm-node ids which are in the
|
||||
* range 0...3 billion and basically dense (=only few
|
||||
* nodes deleted)
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class DenseLongMap
|
||||
{
|
||||
private ArrayList<int[]> blocklist = new ArrayList<int[]>(1024);
|
||||
|
||||
private static final int BLOCKSIZE = 0x10000; // 64k * 32 bits
|
||||
private int valuebits;
|
||||
private int maxvalue;
|
||||
private long maxkey;
|
||||
private long maxmemory;
|
||||
|
||||
/**
|
||||
* Creates a DenseLongMap for the given value range
|
||||
* Note that one value is reserved for the "unset" state,
|
||||
* so with 6 value bits you can store values in the
|
||||
* range 0..62 only
|
||||
*
|
||||
* @param valuebits number of bits to use per value
|
||||
*/
|
||||
public DenseLongMap( int valuebits )
|
||||
{
|
||||
if ( valuebits < 1 || valuebits > 32 )
|
||||
{
|
||||
throw new IllegalArgumentException( "invalid valuebits (1..32): " + valuebits );
|
||||
}
|
||||
this.valuebits = valuebits;
|
||||
maxmemory = (Runtime.getRuntime().maxMemory() / 8) * 7; // assume most of it for our map
|
||||
maxvalue = (1 << valuebits) - 2;
|
||||
maxkey = ( maxmemory / valuebits ) * 8;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void put( long key, int value )
|
||||
{
|
||||
if ( key < 0L || key > maxkey )
|
||||
{
|
||||
throw new IllegalArgumentException( "key out of range (0.." + maxkey + "): " + key
|
||||
+ " give more memory (currently " + (maxmemory / 0x100000)
|
||||
+ "MB) to extend key range" );
|
||||
}
|
||||
if ( value < 0 || value > maxvalue )
|
||||
{
|
||||
throw new IllegalArgumentException( "value out of range (0.." + maxvalue + "): " + value );
|
||||
}
|
||||
|
||||
int blockn = (int)(key >> 21);
|
||||
int offset = (int)(key & 0x1fffff);
|
||||
|
||||
int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
|
||||
|
||||
if ( block == null )
|
||||
{
|
||||
block = new int[BLOCKSIZE * valuebits];
|
||||
|
||||
while (blocklist.size() < blockn+1 )
|
||||
{
|
||||
blocklist.add(null);
|
||||
}
|
||||
blocklist.set( blockn, block );
|
||||
}
|
||||
|
||||
int bitmask = 1 << (offset & 0x1f);
|
||||
int invmask = bitmask ^ 0xffffffff;
|
||||
int probebit = 1;
|
||||
int blockidx = (offset >> 5)*valuebits;
|
||||
int blockend = blockidx + valuebits;
|
||||
int v = value + 1; // 0 is reserved (=unset)
|
||||
|
||||
while( blockidx < blockend )
|
||||
{
|
||||
if ( ( v & probebit ) != 0 )
|
||||
{
|
||||
block[blockidx] |= bitmask;
|
||||
}
|
||||
else
|
||||
{
|
||||
block[blockidx] &= invmask;
|
||||
}
|
||||
probebit <<= 1;
|
||||
blockidx++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public int getInt( long key )
|
||||
{
|
||||
if ( key < 0 )
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
int blockn = (int)(key >> 21);
|
||||
int offset = (int)(key & 0x1fffff);
|
||||
|
||||
int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
|
||||
|
||||
if ( block == null )
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
int bitmask = 1 << (offset & 0x1f);
|
||||
int probebit = 1;
|
||||
int blockidx = (offset >> 5)*valuebits;
|
||||
int blockend = blockidx + valuebits;
|
||||
int v = 0; // 0 is reserved (=unset)
|
||||
|
||||
while( blockidx < blockend )
|
||||
{
|
||||
if ( ( block[blockidx] & bitmask ) != 0 )
|
||||
{
|
||||
v |= probebit;
|
||||
}
|
||||
probebit <<= 1;
|
||||
blockidx++;
|
||||
}
|
||||
return v-1;
|
||||
}
|
||||
|
||||
}
|
||||
113
brouter-util/src/test/java/btools/util/DenseLongMapTest.java
Normal file
113
brouter-util/src/test/java/btools/util/DenseLongMapTest.java
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class DenseLongMapTest
|
||||
{
|
||||
@Test
|
||||
public void hashMapComparisonTest()
|
||||
{
|
||||
hashMapComparison( 100000, 100000, 100000 );
|
||||
hashMapComparison( 100000, 100000, 13000000 );
|
||||
}
|
||||
|
||||
private void hashMapComparison( int mapsize, int trycount, long keyrange )
|
||||
{
|
||||
Random rand = new Random( 12345 );
|
||||
HashMap<Long,Integer> hmap = new HashMap<Long,Integer>();
|
||||
DenseLongMap dmap = new DenseLongMap( 6 );
|
||||
|
||||
for( int i=0; i<mapsize; i++ )
|
||||
{
|
||||
int value = i%63;
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
Long KK = new Long( k );
|
||||
|
||||
hmap.put( KK, new Integer ( value ) );
|
||||
dmap.put( k, value ); // duplicate puts allowed!
|
||||
}
|
||||
|
||||
for( int i=0; i<trycount; i++ )
|
||||
{
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
Long KK = new Long( k );
|
||||
Integer VV = hmap.get( KK );
|
||||
int hvalue = VV == null ? -1 : VV.intValue();
|
||||
int dvalue = dmap.getInt( k );
|
||||
|
||||
if ( hvalue != dvalue )
|
||||
{
|
||||
Assert.fail( "value missmatch for key " + k + " hashmap=" + hvalue + " densemap=" + dvalue );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void oneBitTest()
|
||||
{
|
||||
int keyrange = 300000;
|
||||
int mapputs = 100000;
|
||||
int trycount = 100000;
|
||||
|
||||
Random rand = new Random( 12345 );
|
||||
HashSet<Long> hset = new HashSet<Long>();
|
||||
|
||||
DenseLongMap dmap = new DenseLongMap( 1 );
|
||||
for( int i=0; i<mapputs; i++ )
|
||||
{
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
hset.add( new Long( k ) );
|
||||
dmap.put( k, 0 );
|
||||
}
|
||||
for( int i=0; i<trycount; i++ )
|
||||
{
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
boolean hcontains = hset.contains( new Long( k ) );
|
||||
boolean dcontains = dmap.getInt( k ) == 0;
|
||||
|
||||
if ( hcontains != dcontains )
|
||||
{
|
||||
Assert.fail( "value missmatch for key " + k + " hashset=" + hcontains + " densemap=" + dcontains );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// @Test - memory test disabled for load reasons
|
||||
public void memoryUsageTest()
|
||||
{
|
||||
int keyrange = 32000000;
|
||||
int mapputs = keyrange * 2;
|
||||
|
||||
Random rand = new Random( 12345 );
|
||||
DenseLongMap dmap = new DenseLongMap( 6 );
|
||||
|
||||
System.gc();
|
||||
long mem1 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
|
||||
|
||||
for( int i=0; i<mapputs; i++ )
|
||||
{
|
||||
int value = i%63;
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
dmap.put( k, value );
|
||||
}
|
||||
|
||||
System.gc();
|
||||
long mem2 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
|
||||
|
||||
long memusage = mem2-mem1;
|
||||
|
||||
if ( memusage > (keyrange/8)*7 )
|
||||
{
|
||||
Assert.fail( "memory usage too high: " + memusage + " for keyrange " + keyrange );
|
||||
}
|
||||
|
||||
// need to use the map again for valid memory measure
|
||||
Assert.assertTrue( "out of range test", dmap.getInt(-1) == -1 );
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,122 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Frozen instance of Memory efficient Map
|
||||
*
|
||||
* This one is readily sorted into a singe array for faster access
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class FrozenLongMap<V> extends CompactLongMap<V>
|
||||
{
|
||||
private long[] faid;
|
||||
private ArrayList<V> flv;
|
||||
private int size = 0;
|
||||
private int p2size; // next power of 2 of size
|
||||
|
||||
public FrozenLongMap( CompactLongMap<V> map )
|
||||
{
|
||||
size = map.size();
|
||||
|
||||
faid = new long[size];
|
||||
flv = new ArrayList<V>(size);
|
||||
|
||||
map.moveToFrozenArrays( faid, flv );
|
||||
|
||||
p2size = 0x40000000;
|
||||
while( p2size > size ) p2size >>= 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean put( long id, V value )
|
||||
{
|
||||
throw new RuntimeException( "cannot put on FrozenLongIntMap" );
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fastPut( long id, V value )
|
||||
{
|
||||
throw new RuntimeException( "cannot put on FrozenLongIntMap" );
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the number of entries in this set
|
||||
*/
|
||||
@Override
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @return true if "id" is contained in this set.
|
||||
*/
|
||||
@Override
|
||||
protected boolean contains( long id, boolean doPut )
|
||||
{
|
||||
if ( size == 0 )
|
||||
{
|
||||
return false;
|
||||
}
|
||||
long[] a = faid;
|
||||
int offset = p2size;
|
||||
int n = 0;
|
||||
|
||||
while ( offset> 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( nn < size && a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
offset >>= 1;
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
value_out = flv.get(n);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the value for "id",
|
||||
* Throw an exception if not contained in the map.
|
||||
*/
|
||||
@Override
|
||||
public V get( long id )
|
||||
{
|
||||
if ( size == 0 )
|
||||
{
|
||||
return null;
|
||||
}
|
||||
long[] a = faid;
|
||||
int offset = p2size;
|
||||
int n = 0;
|
||||
|
||||
while ( offset> 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( nn < size && a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
offset >>= 1;
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
return flv.get(n);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public List<V> getValueList()
|
||||
{
|
||||
return flv;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* Frozen instance of Memory efficient Set
|
||||
*
|
||||
* This one is readily sorted into a singe array for faster access
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class FrozenLongSet extends CompactLongSet
|
||||
{
|
||||
private long[] faid;
|
||||
private int size = 0;
|
||||
private int p2size; // next power of 2 of size
|
||||
|
||||
public FrozenLongSet( CompactLongSet set )
|
||||
{
|
||||
size = set.size();
|
||||
|
||||
faid = new long[size];
|
||||
|
||||
set.moveToFrozenArray( faid );
|
||||
|
||||
p2size = 0x40000000;
|
||||
while( p2size > size ) p2size >>= 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean add( long id )
|
||||
{
|
||||
throw new RuntimeException( "cannot add on FrozenLongSet" );
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fastAdd( long id )
|
||||
{
|
||||
throw new RuntimeException( "cannot add on FrozenLongSet" );
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the number of entries in this set
|
||||
*/
|
||||
@Override
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @return true if "id" is contained in this set.
|
||||
*/
|
||||
@Override
|
||||
public boolean contains( long id )
|
||||
{
|
||||
if ( size == 0 )
|
||||
{
|
||||
return false;
|
||||
}
|
||||
long[] a = faid;
|
||||
int offset = p2size;
|
||||
int n = 0;
|
||||
|
||||
while ( offset> 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( nn < size && a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
offset >>= 1;
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
/**
|
||||
* Behaves like an Array of list
|
||||
* with lazy list-allocation at getList
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class LazyArrayOfLists<E>
|
||||
{
|
||||
private ArrayList<ArrayList<E>> lists;
|
||||
|
||||
public LazyArrayOfLists( int size )
|
||||
{
|
||||
lists = new ArrayList<ArrayList<E>>( size );
|
||||
for ( int i = 0; i< size; i++ )
|
||||
{
|
||||
lists.add( null );
|
||||
}
|
||||
}
|
||||
|
||||
public List<E> getList( int idx )
|
||||
{
|
||||
ArrayList<E> list = lists.get( idx );
|
||||
if ( list == null )
|
||||
{
|
||||
list = new ArrayList<E>();
|
||||
lists.set( idx, list );
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public int getSize( int idx )
|
||||
{
|
||||
List<E> list = lists.get( idx );
|
||||
return list == null ? 0 : list.size();
|
||||
}
|
||||
|
||||
public void trimAll()
|
||||
{
|
||||
for ( int idx = 0; idx< lists.size(); idx++ )
|
||||
{
|
||||
ArrayList<E> list = lists.get( idx );
|
||||
if ( list != null )
|
||||
{
|
||||
list.trimToSize();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* dynamic list of primitive longs
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class LongList
|
||||
{
|
||||
private long[] a;
|
||||
private int size;
|
||||
|
||||
public LongList( int capacity )
|
||||
{
|
||||
a = capacity < 4 ? new long[4] : new long[capacity];
|
||||
}
|
||||
|
||||
public void add( long value )
|
||||
{
|
||||
if ( size == a.length )
|
||||
{
|
||||
long[] aa = new long[2*size];
|
||||
System.arraycopy( a, 0, aa, 0, size );
|
||||
a = aa;
|
||||
}
|
||||
a[size++] = value;
|
||||
}
|
||||
|
||||
public long get( int idx )
|
||||
{
|
||||
if ( idx >= size )
|
||||
{
|
||||
throw new IndexOutOfBoundsException( "list size=" + size + " idx=" + idx );
|
||||
}
|
||||
return a[idx];
|
||||
}
|
||||
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,206 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* TinyDenseLongMap implements the DenseLongMap interface
|
||||
* but actually is made for a medium count of non-dense keys
|
||||
*
|
||||
* It's used as a replacement for DenseLongMap where we
|
||||
* have limited memory and far less keys than maykey
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class TinyDenseLongMap extends DenseLongMap
|
||||
{
|
||||
private long[][] al;
|
||||
private int[] pa;
|
||||
private int size = 0;
|
||||
private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
|
||||
|
||||
protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
|
||||
|
||||
public TinyDenseLongMap()
|
||||
{
|
||||
super(1);
|
||||
|
||||
// pointer array
|
||||
pa = new int[MAXLISTS];
|
||||
|
||||
// allocate key lists
|
||||
al = new long[MAXLISTS][];
|
||||
al[0] = new long[1]; // make the first array (the transient buffer)
|
||||
|
||||
// same for the values
|
||||
vla = new byte[MAXLISTS][];
|
||||
vla[0] = new byte[1];
|
||||
}
|
||||
|
||||
|
||||
private byte[][] vla; // value list array
|
||||
|
||||
private void fillReturnValue(byte[] rv, int idx, int p )
|
||||
{
|
||||
rv[0] = vla[idx][p];
|
||||
if ( rv.length == 2 )
|
||||
{
|
||||
vla[idx][p] = rv[1];
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void put( long id, int value )
|
||||
{
|
||||
byte[] rv = new byte[2];
|
||||
rv[1] = (byte)value;
|
||||
if ( contains( id, rv ) )
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
vla[0][0] = (byte)value;
|
||||
_add( id );
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get the byte for the given id
|
||||
* @param id the key to query
|
||||
* @return the object
|
||||
* @exception IllegalArgumentException if id is unknown
|
||||
*/
|
||||
@Override
|
||||
public int getInt( long id )
|
||||
{
|
||||
byte[] rv = new byte[1];
|
||||
if ( contains( id, rv ) )
|
||||
{
|
||||
return rv[0];
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
private boolean _add( long id )
|
||||
{
|
||||
if ( size == Integer.MAX_VALUE )
|
||||
{
|
||||
throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
|
||||
}
|
||||
|
||||
// put the new entry in the first array
|
||||
al[0][0] = id;
|
||||
|
||||
// determine the first empty array
|
||||
int bp = size++; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
int n = 1;
|
||||
|
||||
pa[0] = 1;
|
||||
pa[1] = 1;
|
||||
|
||||
while ( (bp&1) == 1 )
|
||||
{
|
||||
bp >>= 1;
|
||||
pa[idx++] = n;
|
||||
n <<= 1;
|
||||
}
|
||||
|
||||
// create it if not existant
|
||||
if ( al[idx] == null )
|
||||
{
|
||||
al[idx] = new long[n];
|
||||
vla[idx] = new byte[n];
|
||||
}
|
||||
|
||||
// now merge the contents of arrays 0...idx-1 into idx
|
||||
while ( n > 0 )
|
||||
{
|
||||
long maxId = 0;
|
||||
int maxIdx = -1;
|
||||
|
||||
for ( int i=0; i<idx; i++ )
|
||||
{
|
||||
int p = pa[i];
|
||||
if ( p > 0 )
|
||||
{
|
||||
long currentId = al[i][p-1];
|
||||
if ( maxIdx < 0 || currentId > maxId )
|
||||
{
|
||||
maxIdx = i;
|
||||
maxId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// current maximum found, copy to target array
|
||||
if ( n < al[idx].length && maxId == al[idx][n] )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
|
||||
}
|
||||
--n;
|
||||
al[idx][n] = maxId;
|
||||
vla[idx][n] = vla[maxIdx][pa[maxIdx]-1];
|
||||
|
||||
--pa[maxIdx];
|
||||
}
|
||||
|
||||
// de-allocate empty arrays of a certain size (fix at 64kByte)
|
||||
while ( idx-- > _maxKeepExponent )
|
||||
{
|
||||
al[idx] = null;
|
||||
vla[idx] = null;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
private boolean contains( long id, byte[] rv )
|
||||
{
|
||||
// determine the first empty array
|
||||
int bp = size; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
// array at idx is valid, check
|
||||
if ( contains( idx, id, rv ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// does sorted array "a" contain "id" ?
|
||||
private boolean contains( int idx, long id, byte[] rv )
|
||||
{
|
||||
long[] a = al[idx];
|
||||
int offset = a.length;
|
||||
int n = 0;
|
||||
|
||||
while ( (offset >>= 1) > 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
if ( rv != null )
|
||||
{
|
||||
fillReturnValue( rv, idx, n );
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
1
misc/pbfparser/.gitignore
vendored
1
misc/pbfparser/.gitignore
vendored
|
|
@ -1,2 +1,3 @@
|
|||
*.jar
|
||||
*.BAK
|
||||
btools/
|
||||
|
|
|
|||
Loading…
Reference in a new issue