pre-processor memory squeezing (dense map)

brouter-map-creator/src/main/java/btools/mapcreator/NodeFilter.java

@@ -38,7 +38,7 @@ public class NodeFilter extends MapCreatorBase
     this.nodeTilesOut = nodeTilesOut;
 
     // read the wayfile into a bitmap of used nodes
-    nodebitmap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap( 1 ) : new TinyDenseLongMap();
+    nodebitmap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap( 512 ) : new TinyDenseLongMap();
     new WayIterator( this, false ).processFile( wayFileIn );
 
     // finally filter all node files

brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java

@@ -34,7 +34,7 @@ public class WayCutter extends MapCreatorBase
     this.outTileDir = wayTilesOut;
 
     // *** read all nodes into tileIndexMap
-    tileIndexMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap( 6 ) : new TinyDenseLongMap();
+    tileIndexMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap() : new TinyDenseLongMap();
     new NodeIterator( this, false ).processDir( nodeTilesIn, ".tlf" );
 
     // *** finally process the way-file, cutting into pieces

brouter-map-creator/src/main/java/btools/mapcreator/WayCutter5.java

@@ -53,7 +53,7 @@ public class WayCutter5 extends MapCreatorBase
     String nodefilename = name.substring( 0, name.length()-3 ) + "tlf";
     File nodefile = new File( nodeTilesIn, nodefilename );
 
-    tileIndexMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap( 6 ) : new TinyDenseLongMap();
+    tileIndexMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap() : new TinyDenseLongMap();
     lonoffset = -1;
     latoffset = -1;
     new NodeIterator( this, false ).processFile( nodefile );

brouter-util/src/main/java/btools/util/DenseLongMap.java

@@ -9,11 +9,6 @@ import java.util.ArrayList;
  * a simple array as the best-fit data model (except for
  * the 32-bit limit of arrays!)
  *
- * Additionally, to enable small-memory unit testing 
- * of code using this map this one has a fallback for a small map
- * where we have only few keys, but not dense. In this
- * case, we use the mechanics of the CompactLongMap
- *
  * Target application are osm-node ids which are in the
  * range 0...3 billion and basically dense (=only few
  * nodes deleted)
@@ -22,57 +17,70 @@ import java.util.ArrayList;
  */
 public class DenseLongMap
 {
-  private ArrayList<int[]> blocklist = new ArrayList<int[]>(1024);
+  private ArrayList<byte[]> blocklist = new ArrayList<byte[]>(4096);
 
-  private static final int BLOCKSIZE = 0x10000; // 64k * 32 bits
+  private int blocksize; // bytes per bitplane in one block
-  private int valuebits;
+  private int blocksizeBits;
-  private int maxvalue;
+  private long blocksizeBitsMask;
-  private long maxkey;
+  private int maxvalue = 254; // fixed due to 8 bit lookup table
-  private long maxmemory;
+  private int[] bitplaneCount = new int[8];
+  private long putCount = 0L;
+  private long getCount = 0L;
 
   /**
-   * Creates a DenseLongMap for the given value range
+   * Creates a DenseLongMap for the default block size
-   * Note that one value is reserved for the "unset" state,
+   * ( 512 bytes per bitplane, covering a key range of 4096 keys )
-   * so with 6 value bits you can store values in the
+   * Note that one value range is limited to 0..254
-   * range 0..62 only
    *
    * @param valuebits number of bits to use per value
    */
-  public DenseLongMap( int valuebits )
+  public DenseLongMap()
   {
-    if ( valuebits < 1 || valuebits > 32 )
+    this(512);
-    {
+  }
-      throw new IllegalArgumentException( "invalid valuebits (1..32): " + valuebits );
+
-    }
+  /**
-    this.valuebits = valuebits;
+   * Creates a DenseLongMap for the given block size
-    maxmemory = (Runtime.getRuntime().maxMemory() / 8) * 7; // assume most of it for our map
+   *
-    maxvalue = (1 << valuebits) - 2;
+   * @param blocksize bytes per bit-plane
-    maxkey = ( maxmemory / valuebits ) * 8;
+   */
+  public DenseLongMap( int blocksize )
+  {
+	int bits = 4;
+	while( bits < 28 && (1 << bits) != blocksize  )
+	{
+	  bits++;
+	}
+	if ( bits == 28 )
+	{
+	  throw new RuntimeException( "not a valid blocksize: " +  blocksize + " ( expected 1 << bits with bits in (4..27) )");
+	}
+	blocksizeBits = bits + 3;
+	blocksizeBitsMask = (1L << blocksizeBits ) -1;
+	this.blocksize = blocksize;
   }
 
 
 
   public void put( long key, int value )
   {
-    if ( key < 0L || key > maxkey )
+	putCount++;
-    {
+
-      throw new IllegalArgumentException( "key out of range (0.." + maxkey + "): " + key
+	if ( value < 0 || value > maxvalue )
-            + " give more memory (currently " + (maxmemory / 0x100000)
-            + "MB) to extend key range" );
-    }
-    if ( value < 0 || value > maxvalue )
     {
       throw new IllegalArgumentException( "value out of range (0.." + maxvalue + "): " + value );
     }
 
-    int blockn = (int)(key >> 21);
+    int blockn = (int)(key >> blocksizeBits);
-    int offset = (int)(key & 0x1fffff);
+    int offset = (int)(key & blocksizeBitsMask);
 
-    int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
+    byte[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
 
+    int valuebits = 1;
     if ( block == null )
     {
-      block = new int[BLOCKSIZE * valuebits];
+      block = new byte[sizeForBits(valuebits)];
+      bitplaneCount[0] ++;
 
       while (blocklist.size() < blockn+1 )
       {
@@ -80,17 +88,49 @@ public class DenseLongMap
       }      
       blocklist.set( blockn, block );
     }
-    
+    else
-    int bitmask = 1 << (offset & 0x1f);
-    int invmask = bitmask ^ 0xffffffff;
-    int probebit = 1;
-    int blockidx = (offset >> 5)*valuebits;
-    int blockend = blockidx + valuebits;
-    int v = value + 1; // 0 is reserved (=unset)
-
-    while( blockidx < blockend )
     {
-      if ( ( v & probebit ) != 0 )
+      // check how many bitplanes we have from the arraysize
+      while( sizeForBits( valuebits) < block.length )
+      {
+        valuebits++;
+      }
+    }
+    int headersize = 1 << valuebits;
+
+    byte v = (byte)(value + 1); // 0 is reserved (=unset)
+    
+    // find the index in the lookup table or the first entry
+    int idx = 1;
+    while( idx < headersize )
+    {
+    	if ( block[idx] == 0 )
+    	{
+    	  block[idx] = v; // create new entry
+    	}
+    	if ( block[idx] == v )
+    	{
+    	  break;
+    	}
+    	idx++;
+    }
+    if ( idx == headersize )
+    {
+    	block = expandBlock( block, valuebits );
+  	    block[idx] = v; // create new entry
+        blocklist.set( blockn, block );
+    	valuebits++;
+    	headersize = 1 << valuebits;
+    }
+        
+    int bitmask = 1 << (offset & 0x7);
+    int invmask = bitmask ^ 0xff;
+    int probebit = 1;
+    int blockidx = (offset >> 3) + headersize;
+
+    for( int i=0; i < valuebits; i++ )
+    {
+      if ( ( idx & probebit ) != 0 )
       {
         block[blockidx] |= bitmask;
       }
@@ -99,42 +139,80 @@ public class DenseLongMap
         block[blockidx] &= invmask;
       }
       probebit <<= 1;
-      blockidx++;
+      blockidx += blocksize;
     }
   }
 
 
+  private int sizeForBits( int bits )
+  {
+	  // size is lookup table + datablocks
+	  return ( 1 << bits ) + blocksize * bits;
+  }
+  
+  private byte[] expandBlock( byte[] block, int valuebits )
+  {
+    bitplaneCount[valuebits] ++;
+	byte[] newblock = new byte[sizeForBits(valuebits+1)];
+	int headersize = 1 << valuebits;
+	System.arraycopy(block, 0, newblock, 0, headersize ); // copy header
+	System.arraycopy(block, headersize, newblock, 2*headersize, block.length - headersize ); // copy data
+	return newblock;
+  }
+
   public int getInt( long key )
   {
+	// bit-stats on first get
+    if ( getCount++ == 0L )
+    {
+    	System.out.println( "**** DenseLongMap stats ****" );
+    	System.out.println( "putCount=" + putCount );
+    	for( int i=0; i<8; i++ )
+    	{
+    	   	System.out.println( i + "-bitplanes=" +bitplaneCount[i]  );
+    	}
+    	System.out.println( "****************************" );
+    }
+	  
     if ( key < 0 )
     {
       return -1;
     }
-    int blockn = (int)(key >> 21);
+    int blockn = (int)(key >> blocksizeBits);
-    int offset = (int)(key & 0x1fffff);
+    int offset = (int)(key & blocksizeBitsMask);
 
-    int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
+    byte[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
 
     if ( block == null )
     {
       return -1;
     }
-    int bitmask = 1 << (offset & 0x1f);
+    
+    // check how many bitplanes we have from the arrayzize
+    int valuebits = 1;
+    while( sizeForBits( valuebits) < block.length )
+    {
+      valuebits++;
+    }
+    int headersize = 1 << valuebits;
+    
+    int bitmask = 1 << (offset & 7);
     int probebit = 1;
-    int blockidx = (offset >> 5)*valuebits;
+    int blockidx = (offset >> 3) + headersize;
-    int blockend = blockidx + valuebits;
+    int idx = 0; // 0 is reserved (=unset)
-    int v = 0; // 0 is reserved (=unset)
 
-    while( blockidx < blockend )
+    for( int i=0; i < valuebits; i++ )
     {
       if ( ( block[blockidx] & bitmask ) != 0 )
       {
-        v  |= probebit;
+        idx  |= probebit;
       }
       probebit <<= 1;
-      blockidx++;
+      blockidx += blocksize;
     }
-    return v-1;
+
+    // lookup that value in the lookup header
+    return ((256 + block[idx]) & 0xff ) -1;
   }
 
 }

brouter-util/src/main/java/btools/util/TinyDenseLongMap.java

@@ -20,7 +20,7 @@ public class TinyDenseLongMap extends DenseLongMap
 
   public TinyDenseLongMap()
   {
-    super(1);
+    super();
 
     // pointer array
     pa = new int[MAXLISTS];

brouter-util/src/test/java/btools/util/DenseLongMapTest.java

@@ -20,11 +20,11 @@ public class DenseLongMapTest
   {
     Random rand = new Random( 12345 );
     HashMap<Long,Integer> hmap = new HashMap<Long,Integer>();
-    DenseLongMap dmap = new DenseLongMap( 6 );
+    DenseLongMap dmap = new DenseLongMap( 512 );
 
     for( int i=0; i<mapsize; i++ )
     {
-      int value = i%63;
+      int value = i%255;
       long k = (long)(rand.nextDouble()*keyrange);
       Long KK = new Long( k );
 
@@ -57,7 +57,7 @@ public class DenseLongMapTest
     Random rand = new Random( 12345 );
     HashSet<Long> hset = new HashSet<Long>();
 
-    DenseLongMap dmap = new DenseLongMap( 1 );
+    DenseLongMap dmap = new DenseLongMap( 512 );
     for( int i=0; i<mapputs; i++ )
     {
       long k = (long)(rand.nextDouble()*keyrange);