direct weaving option

brouter-mapaccess/src/main/java/btools/mapaccess/DirectWeaver.java

@@ -0,0 +1,228 @@
+package btools.mapaccess;
+
+import btools.codec.DataBuffers;
+import btools.codec.NoisyDiffCoder;
+import btools.codec.StatCoderContext;
+import btools.codec.TagValueCoder;
+import btools.codec.TagValueValidator;
+import btools.codec.TagValueWrapper;
+import btools.codec.WaypointMatcher;
+import btools.util.ByteDataWriter;
+import btools.util.IByteArrayUnifier;
+
+/**
+ * MicroCache2 is the new format that uses statistical encoding and
+ * is able to do access filtering and waypoint matching during encoding
+ */
+public final class DirectWeaver extends ByteDataWriter
+{
+  private int lonBase;
+  private int latBase;
+  private int cellsize;
+
+  protected int[] faid;
+  protected int size = 0;
+
+  private static boolean debug = false;
+  
+  public DirectWeaver( DataBuffers dataBuffers, int lonIdx, int latIdx, int divisor, TagValueValidator wayValidator, WaypointMatcher waypointMatcher, OsmNodesMap hollowNodes ) throws Exception
+  {
+    super( null );
+    cellsize = 1000000 / divisor;
+    lonBase = lonIdx*cellsize;
+    latBase = latIdx*cellsize;
+
+    StatCoderContext bc = new StatCoderContext( dataBuffers.iobuffer );
+
+    TagValueCoder wayTagCoder = new TagValueCoder( bc, dataBuffers, wayValidator );
+    TagValueCoder nodeTagCoder = new TagValueCoder( bc, dataBuffers, null );
+    NoisyDiffCoder nodeIdxDiff = new NoisyDiffCoder( bc );
+    NoisyDiffCoder nodeEleDiff = new NoisyDiffCoder( bc );
+    NoisyDiffCoder extLonDiff = new NoisyDiffCoder(bc);
+    NoisyDiffCoder extLatDiff = new NoisyDiffCoder(bc);
+    NoisyDiffCoder transEleDiff = new NoisyDiffCoder( bc );
+
+    size = bc.decodeNoisyNumber( 5 );
+    faid = size > dataBuffers.ibuf2.length ? new int[size] : dataBuffers.ibuf2;
+    
+    if ( debug ) System.out.println( "*** decoding cache of size=" + size + " for lonIdx=" + lonIdx + " latIdx=" + latIdx );
+
+    bc.decodeSortedArray( faid, 0, size, 0x20000000, 0 );
+    
+    OsmNode[] nodes = new OsmNode[size];
+    for( int n = 0; n<size; n++ )
+    {
+      long id = expandId( faid[n] );
+      int ilon = (int) ( id >> 32 );
+      int ilat = (int) ( id & 0xffffffff );
+      OsmNode node = hollowNodes.get( ilon, ilat );
+      if ( node == null )
+      {
+        node = new OsmNode( ilon, ilat );
+      }
+      else
+      {
+        hollowNodes.remove( node );
+      }
+      nodes[n] = node;
+    }
+    
+    int netdatasize = bc.decodeNoisyNumber( 10 ); // (not needed for direct weaving)
+    ab = dataBuffers.bbuf1;
+    aboffset = 0;
+
+    int selev = 0;
+    for( int n=0; n<size; n++ ) // loop over nodes
+    { 
+      OsmNode node = nodes[n];
+      int ilon = node.ilon;
+      int ilat = node.ilat;
+    
+      // future escapes (turn restrictions?)
+      short trExceptions = 0;
+      for(;;)
+      {
+        int featureId = bc.decodeVarBits();
+        if ( featureId == 0 ) break;
+        int bitsize = bc.decodeNoisyNumber( 5 );
+
+        if ( featureId == 2 ) // exceptions to turn-restriction
+        {
+          trExceptions = (short)bc.decodeBounded( 1023 );
+        }
+        else if ( featureId == 1 ) // turn-restriction
+        {
+          TurnRestriction tr = new TurnRestriction();
+          tr.exceptions =  trExceptions;
+          trExceptions = 0;
+          tr.isPositive =  bc.decodeBit();
+          tr.fromLon = ilon + bc.decodeNoisyDiff( 10 );
+          tr.fromLat = ilat + bc.decodeNoisyDiff( 10 );
+          tr.toLon = ilon + bc.decodeNoisyDiff( 10 );
+          tr.toLat = ilat + bc.decodeNoisyDiff( 10 );
+          node.addTurnRestriction( tr );
+        }
+        else
+        {
+          for( int i=0; i< bitsize; i++ ) bc.decodeBit(); // unknown feature, just skip
+        }
+      }
+
+      selev += nodeEleDiff.decodeSignedValue();
+      node.selev = (short)selev;
+      TagValueWrapper nodeTags = nodeTagCoder.decodeTagValueSet();
+      node.nodeDescription = nodeTags == null ? null : nodeTags.data; // TODO: unified?
+
+      int links = bc.decodeNoisyNumber( 1 );
+      if ( debug ) System.out.println( "***   decoding node " + ilon + "/" + ilat + " with links=" + links );
+      for( int li=0; li<links; li++ )
+      {
+        int nodeIdx = n + nodeIdxDiff.decodeSignedValue();
+        
+        int dlon_remaining;
+        int dlat_remaining;
+
+        boolean isReverse = false;
+        if ( nodeIdx != n ) // internal (forward-) link
+        {
+          dlon_remaining = nodes[nodeIdx].ilon - ilon;
+          dlat_remaining = nodes[nodeIdx].ilat - ilat;
+        }
+        else
+        {
+          isReverse = bc.decodeBit();
+          dlon_remaining = extLonDiff.decodeSignedValue();
+          dlat_remaining = extLatDiff.decodeSignedValue();
+        }
+        if ( debug ) System.out.println( "***     decoding link to " + (ilon+dlon_remaining) + "/" + (ilat+dlat_remaining) + " extern=" + (nodeIdx == n) );
+
+        TagValueWrapper wayTags = wayTagCoder.decodeTagValueSet();
+
+        int linklon = ilon + dlon_remaining;
+        int linklat = ilat + dlat_remaining;
+        aboffset = 0;
+        if ( !isReverse ) // write geometry for forward links only
+        {
+          WaypointMatcher matcher = wayTags == null || wayTags.accessType < 2 ? null : waypointMatcher;
+          int ilontarget = ilon + dlon_remaining;
+          int ilattarget = ilat + dlat_remaining;
+          if ( matcher != null )
+          {
+            if ( !matcher.start( ilon, ilat, ilontarget, ilattarget ) )
+            {
+              matcher = null;
+            }
+          }
+          
+          int transcount = bc.decodeVarBits();
+          if ( debug ) System.out.println( "***       decoding geometry with count=" + transcount );
+          int count = transcount+1;
+          for( int i=0; i<transcount; i++ )
+          {
+            int dlon = bc.decodePredictedValue( dlon_remaining/count );
+            int dlat = bc.decodePredictedValue( dlat_remaining/count );
+            dlon_remaining -= dlon;
+            dlat_remaining -= dlat;
+            count--;
+            int elediff = transEleDiff.decodeSignedValue();
+            if ( wayTags != null )
+            {
+              writeVarLengthSigned( dlon );
+              writeVarLengthSigned( dlat );
+              writeVarLengthSigned( elediff );
+            }
+            
+            if ( matcher != null ) matcher.transferNode( ilontarget - dlon_remaining, ilattarget - dlat_remaining );
+          }
+          if ( matcher != null ) matcher.end();
+        }
+
+        if ( wayTags != null )
+        {
+          byte[] geometry = null;
+          if ( aboffset > 0 )
+          {
+            geometry = new byte[aboffset];
+            System.arraycopy( ab, 0, geometry, 0, aboffset );
+          }
+        
+          if ( nodeIdx != n ) // valid internal (forward-) link
+          {
+            OsmNode node2 = nodes[nodeIdx];
+            OsmLink link = node.isLinkUnused() ? node : ( node2.isLinkUnused() ? node2 : new OsmLink() );
+            link.descriptionBitmap = wayTags.data;
+            link.geometry = geometry;
+            node.addLink( link, isReverse, node2 );
+          }
+          else // weave external link
+          {
+            node.addLink( linklon, linklat, wayTags.data, geometry, hollowNodes, isReverse );
+          }
+        }
+          
+
+
+      }
+    }
+    
+  }
+
+  public long expandId( int id32 )
+  {
+    int dlon = 0;
+    int dlat = 0;
+
+    for( int bm = 1; bm < 0x8000; bm <<= 1 )
+    {
+      if ( (id32 & 1) != 0 ) dlon |= bm;
+      if ( (id32 & 2) != 0 ) dlat |= bm;
+      id32 >>= 2;
+    }
+
+    int lon32 = lonBase + dlon;
+    int lat32 = latBase + dlat;
+
+    return ((long)lon32)<<32 | lat32;
+  }
+
+}

brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java

@@ -49,6 +49,8 @@ public final class NodesCache
   private long ghostSum = 0;
   private long ghostWakeup = 0;
   
+  private boolean directWeaving = Boolean.getBoolean( "directWeaving" );
+  
   public String formatStatus()
   {
     return "collecting=" + garbageCollectionEnabled + " noGhosts=" + ghostCleaningDone + " cacheSum=" + cacheSum + " cacheSumClean=" + cacheSumClean + " ghostSum=" + ghostSum + " ghostWakeup=" + ghostWakeup ;
@@ -210,7 +212,7 @@ public final class NodesCache
       if ( segment == null )
       {
         checkEnableCacheCleaning();
-        segment = osmf.createMicroCache( ilon, ilat, dataBuffers, expCtxWay, waypointMatcher );
+        segment = osmf.createMicroCache( ilon, ilat, dataBuffers, expCtxWay, waypointMatcher, directWeaving ? nodesMap : null );
 
         cacheSum += segment.getDataSize();
       }
@@ -248,6 +250,10 @@ public final class NodesCache
     {
       return false;
     }
+    if ( !node.isHollow() )
+    {
+      return true; // direct weaving...
+    }
 
     long id = node.getIdFromPos();
     if ( segment.getAndClear( id ) )
@@ -292,6 +298,7 @@ public final class NodesCache
     // initialize the start-node
     OsmNode n = new OsmNode( id );
     n.setHollow();
+    nodesMap.put( n );
     if ( !obtainNonHollowNode( n ) )
     {
       return null;

brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java

@@ -94,12 +94,12 @@ final class OsmFile
     return microCaches[subIdx];
   }
 
-  public MicroCache createMicroCache( int ilon, int ilat, DataBuffers dataBuffers, TagValueValidator wayValidator, WaypointMatcher waypointMatcher )
+  public MicroCache createMicroCache( int ilon, int ilat, DataBuffers dataBuffers, TagValueValidator wayValidator, WaypointMatcher waypointMatcher, OsmNodesMap hollowNodes )
       throws Exception
   {
     int lonIdx = ilon / cellsize;
     int latIdx = ilat / cellsize;
-    MicroCache segment = createMicroCache( lonIdx, latIdx, dataBuffers, wayValidator, waypointMatcher, true );
+    MicroCache segment = createMicroCache( lonIdx, latIdx, dataBuffers, wayValidator, waypointMatcher, true, hollowNodes );
     int subIdx = ( latIdx - divisor * latDegree ) * divisor + ( lonIdx - divisor * lonDegree );
     microCaches[subIdx] = segment;
     return segment;
@@ -127,7 +127,7 @@ final class OsmFile
   }
 
   public MicroCache createMicroCache( int lonIdx, int latIdx, DataBuffers dataBuffers, TagValueValidator wayValidator,
-      WaypointMatcher waypointMatcher, boolean reallyDecode ) throws Exception
+      WaypointMatcher waypointMatcher, boolean reallyDecode, OsmNodesMap hollowNodes ) throws Exception
   {
     int subIdx = ( latIdx - divisor * latDegree ) * divisor + ( lonIdx - divisor * lonDegree );
 
@@ -155,7 +155,18 @@ final class OsmFile
     }
     else if ( ( crcData ^ 2 ) == crcFooter )
     {
-      return reallyDecode ? new MicroCache2( dataBuffers, lonIdx, latIdx, divisor, wayValidator, waypointMatcher ) : null;
+      if ( !reallyDecode )
+      {
+        return null;
+      }
+      if ( hollowNodes == null )
+      {
+        return new MicroCache2( dataBuffers, lonIdx, latIdx, divisor, wayValidator, waypointMatcher );
+      }
+      new DirectWeaver( dataBuffers, lonIdx, latIdx, divisor, wayValidator, waypointMatcher, hollowNodes );
+      MicroCache dummy = MicroCache.emptyCache();
+      dummy.virgin = false;
+      return dummy;
     }
     throw new IOException( "checkum error" );
   }

brouter-mapaccess/src/main/java/btools/mapaccess/OsmLink.java

@@ -45,8 +45,13 @@ public class OsmLink
    */
   public final OsmNode getTarget( OsmNode source )
   {
-    return n2 != source && n2 != null ? n2 : n1;
+/*    if ( isBidirectional() )
-/*    if ( n2 != null && n2 != source )
+    {
+      return n2 != source ? n2 : n1;
+    }
+    return n2 != null ? n2 : n1; */
+    // return n2 != source && n2 != null ? n2 : n1;
+    if ( n2 != null && n2 != source )
     {
       return n2;
     }
@@ -58,7 +63,7 @@ public class OsmLink
     {    
       new Throwable( "ups" ).printStackTrace();
       throw new IllegalArgumentException( "internal error: getTarget: unknown source; " + source + " n1=" + n1 + " n2=" + n2 );
-    } */
+    }
   }
   
   /**
@@ -66,8 +71,8 @@ public class OsmLink
    */
   public final OsmLink getNext( OsmNode source )
   {
-    return n2 != source && n2 != null ? next : previous;
+    // return n2 != source && n2 != null ? next : previous;
-/*    if ( n2 != null && n2 != source )
+    if ( n2 != null && n2 != source )
     {
       return next;
     }
@@ -78,7 +83,7 @@ public class OsmLink
     else
     {
       throw new IllegalArgumentException( "internal error: gextNext: unknown source" );
-    } */
+    }
   }
 
   /**
@@ -147,8 +152,8 @@ public class OsmLink
 
   public final boolean isReverse( OsmNode source )
   {
-    return n1 != source && n1 != null;
+  //  return n1 != source && n1 != null;
-/*    if ( n2 != null && n2 != source )
+    if ( n2 != null && n2 != source )
     {
       return false;
     }
@@ -159,7 +164,7 @@ public class OsmLink
     else
     {
       throw new IllegalArgumentException( "internal error: isReverse: unknown source" );
-    } */
+    }
   }
 
   public final boolean isBidirectional()
@@ -167,6 +172,11 @@ public class OsmLink
     return n1 != null && n2 != null;
   }
 
+  public final boolean isLinkUnused()
+  {
+    return n1 == null && n2 == null;
+  }
+
   public final void addLinkHolder( OsmLinkHolder holder, OsmNode source )
   {
     OsmLinkHolder firstHolder = getFirstLinkHolder( source );

brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java

@@ -35,6 +35,12 @@ public class OsmNode extends OsmLink implements OsmPos
 
   public TurnRestriction firstRestriction;
 
+  public void addTurnRestriction( TurnRestriction tr )
+  {
+    tr.next = firstRestriction;
+    firstRestriction = tr;
+  }
+
   /**
    * The links to other nodes
    */
@@ -78,8 +84,13 @@ public class OsmNode extends OsmLink implements OsmPos
     return selev / 4.;
   }
 
-  private void addLink( OsmLink link, boolean isReverse, OsmNode tn )
+  public final void addLink( OsmLink link, boolean isReverse, OsmNode tn )
   {
+    if ( link == firstlink )
+    {
+      throw new IllegalArgumentException( "UUUUPS" );
+    }
+  
     if ( isReverse )
     {
       link.n1 = tn;
@@ -107,7 +118,7 @@ public class OsmNode extends OsmLink implements OsmPos
 
   public String toString()
   {
-    return "" + getIdFromPos();
+    return "n_" + (ilon-180000000) + "_" + (ilat-90000000);
   }
 
   public final void parseNodeBody( MicroCache mc, OsmNodesMap hollowNodes, IByteArrayUnifier expCtxWay )
@@ -134,16 +145,13 @@ public class OsmNode extends OsmLink implements OsmPos
       tr.fromLat = mc.readInt();
       tr.toLon = mc.readInt();
       tr.toLat = mc.readInt();
-      tr.next = firstRestriction;
+      addTurnRestriction( tr );
-      firstRestriction = tr;
     }
 
     selev = mc.readShort();
     int nodeDescSize = mc.readVarLengthUnsigned();
     nodeDescription = nodeDescSize == 0 ? null : mc.readUnified( nodeDescSize, abUnifier );
 
-    OsmLink link0 = firstlink;
-
     while (mc.hasMoreData())
     {
       // read link data
@@ -160,16 +168,23 @@ public class OsmNode extends OsmLink implements OsmPos
       }
       byte[] geometry = mc.readDataUntil( endPointer );
 
+      addLink( linklon, linklat, description, geometry, hollowNodes, isReverse );
+    }
+    hollowNodes.remove( this );
+  }
+
+  public void addLink( int linklon, int linklat, byte[] description, byte[] geometry, OsmNodesMap hollowNodes, boolean isReverse )
+  {
       if ( linklon == ilon && linklat == ilat )
       {
-        continue; // skip self-ref
+        return; // skip self-ref
       }
 
       OsmNode tn = null; // find the target node
       OsmLink link = null;
 
       // ...in our known links
-      for ( OsmLink l = link0; l != null; l = l.getNext( this ) )
+      for ( OsmLink l = firstlink; l != null; l = l.getNext( this ) )
       {
         OsmNode t = l.getTarget( this );
         if ( t.ilon == linklon && t.ilat == linklat )
@@ -203,8 +218,6 @@ public class OsmNode extends OsmLink implements OsmPos
         link.geometry = geometry;
       }
   }
-    hollowNodes.remove( this );
-  }
 
 
   public final boolean isHollow()

brouter-mapaccess/src/main/java/btools/mapaccess/PhysicalFile.java

@@ -62,7 +62,7 @@ final public class PhysicalFile
           if ( osmf.hasData() )
             for ( int lonIdx = 0; lonIdx < div; lonIdx++ )
               for ( int latIdx = 0; latIdx < div; latIdx++ )
-                osmf.createMicroCache( lonDegree * div + lonIdx, latDegree * div + latIdx, dataBuffers, null, null, MicroCache.debug );
+                osmf.createMicroCache( lonDegree * div + lonIdx, latDegree * div + latIdx, dataBuffers, null, null, MicroCache.debug, null );
         }
       }
     }