diff --git a/brouter-codec/src/main/java/btools/codec/TagValueCoder.java b/brouter-codec/src/main/java/btools/codec/TagValueCoder.java index 3818d4e..5cddfb1 100644 --- a/brouter-codec/src/main/java/btools/codec/TagValueCoder.java +++ b/brouter-codec/src/main/java/btools/codec/TagValueCoder.java @@ -99,7 +99,12 @@ public final class TagValueCoder node.child2 = decodeTree( bc, buffer, validator ); return node; } - int startpos = bc.getReadingBitPosition(); + + BitCoderContext ctx = new BitCoderContext( buffer ); + + int inum = 0; + int lastEncodedInum = 0; + boolean hasdata = false; for ( ;; ) { @@ -110,22 +115,36 @@ public final class TagValueCoder { return null; } - hasdata = true; } if ( delta == 0 ) { + ctx.encodeVarBits( 0 ); break; } - bc.decodeVarBits(); + inum += delta; + + int data = bc.decodeVarBits(); + + if ( validator == null || validator.isLookupIdxUsed( inum ) ) + { + hasdata = true; + ctx.encodeVarBits( inum - lastEncodedInum ); + ctx.encodeVarBits( data ); + lastEncodedInum = inum; + } } - int endpos = bc.getReadingBitPosition(); - - int bitcount = endpos - startpos; - int bytecount = ( bitcount + 7 ) >> 3; - bc.setReadingBitPosition( startpos ); - byte[] res = new byte[bytecount]; - bc.copyBitsTo( res, bitcount ); + byte[] res; + int len = ctx.getEncodedLength(); + if ( validator == null ) + { + res = new byte[len]; + System.arraycopy( buffer, 0, res, 0, len ); + } + else + { + res = validator.unify( buffer, 0, len ); + } int accessType = validator == null ? 2 : validator.accessType( res ); if ( accessType > 0 ) diff --git a/brouter-codec/src/main/java/btools/codec/TagValueValidator.java b/brouter-codec/src/main/java/btools/codec/TagValueValidator.java index 6e0bf3a..b5a1a2f 100644 --- a/brouter-codec/src/main/java/btools/codec/TagValueValidator.java +++ b/brouter-codec/src/main/java/btools/codec/TagValueValidator.java @@ -8,4 +8,9 @@ public interface TagValueValidator * @return 0 = nothing, 1=no matching, 2=normal */ public int accessType( byte[] tagValueSet ); + + public byte[] unify( byte[] tagValueSet, int offset, int len ); + + public boolean isLookupIdxUsed( int idx ); + } diff --git a/brouter-core/src/main/java/btools/router/OsmTrack.java b/brouter-core/src/main/java/btools/router/OsmTrack.java index 9529fb7..5e872c5 100644 --- a/brouter-core/src/main/java/btools/router/OsmTrack.java +++ b/brouter-core/src/main/java/btools/router/OsmTrack.java @@ -17,6 +17,8 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.InputStreamReader; +import java.io.IOException; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -327,14 +329,28 @@ public final class OsmTrack public void writeGpx( String filename ) throws Exception { BufferedWriter bw = new BufferedWriter( new FileWriter( filename ) ); - - bw.write( formatAsGpx() ); + formatAsGpx( bw ); bw.close(); } public String formatAsGpx() { - StringBuilder sb = new StringBuilder( 8192 ); + try + { + StringWriter sw = new StringWriter( 8192 ); + BufferedWriter bw = new BufferedWriter( sw ); + formatAsGpx( bw ); + bw.close(); + return sw.toString(); + } + catch( Exception e ) + { + throw new RuntimeException( e ); + } + } + + public String formatAsGpx( BufferedWriter sb ) throws IOException + { int turnInstructionMode = voiceHints != null ? voiceHints.turnInstructionMode : 0; sb.append( "\n" ); @@ -385,8 +401,8 @@ public final class OsmTrack sb.append(" \n" ) .append ( " " ).append( hint.getMessageString() ).append( "\n \n " ) - .append( hint.getCommandString() ).append("\n ").append( hint.angle ) - .append("\n ").append( hint.indexInTrack ).append("\n \n \n"); + .append( hint.getCommandString() ).append("\n ").append( "" + hint.angle ) + .append("\n ").append( "" + hint.indexInTrack ).append("\n \n \n"); } sb.append("\n"); } @@ -399,8 +415,8 @@ public final class OsmTrack .append( formatILat( hint.ilat ) ).append( "\">" ) .append( hint.selev == Short.MIN_VALUE ? "" : "" + (hint.selev / 4.) + "" ) .append( "" ).append( hint.getMessageString() ).append( "" ) - .append( "" ).append( hint.distanceToNext ).append( "" ) - .append( "" ).append( hint.getLocusAction() ).append( "" ) + .append( "" ).append( "" + hint.distanceToNext ).append( "" ) + .append( "" ).append( "" + hint.getLocusAction() ).append( "" ) .append( "\n" ); } } @@ -425,7 +441,7 @@ public final class OsmTrack if ( turnInstructionMode == 2 ) { - sb.append( " " ).append( voiceHints.getLocusRouteType() ).append( "\n" ); + sb.append( " " ).append( "" + voiceHints.getLocusRouteType() ).append( "\n" ); sb.append( " 1\n" ); } diff --git a/brouter-core/src/main/java/btools/router/ProfileCache.java b/brouter-core/src/main/java/btools/router/ProfileCache.java index ff58887..34c71a3 100644 --- a/brouter-core/src/main/java/btools/router/ProfileCache.java +++ b/brouter-core/src/main/java/btools/router/ProfileCache.java @@ -63,8 +63,8 @@ public final class ProfileCache BExpressionMetaData meta = new BExpressionMetaData(); BExpressionContextGlobal expctxGlobal = new BExpressionContextGlobal( meta ); - rc.expctxWay = new BExpressionContextWay( rc.serversizing ? 131072 : 32768, meta ); - rc.expctxNode = new BExpressionContextNode( rc.serversizing ? 4096 : 1024, meta ); + rc.expctxWay = new BExpressionContextWay( rc.memoryclass * 512, meta ); + rc.expctxNode = new BExpressionContextNode( rc.memoryclass * 128, meta ); meta.readMetaData( new File( profileDir, "lookups.dat" ) ); diff --git a/brouter-core/src/main/java/btools/router/RoutingContext.java b/brouter-core/src/main/java/btools/router/RoutingContext.java index 621380e..bfd6ef8 100644 --- a/brouter-core/src/main/java/btools/router/RoutingContext.java +++ b/brouter-core/src/main/java/btools/router/RoutingContext.java @@ -41,7 +41,7 @@ public final class RoutingContext public BExpressionContextWay expctxWay; public BExpressionContextNode expctxNode; - public boolean serversizing = false; + public int memoryclass = 64; public int downhillcostdiv; public int downhillcutoff; diff --git a/brouter-core/src/main/java/btools/router/RoutingEngine.java b/brouter-core/src/main/java/btools/router/RoutingEngine.java index 0949da1..b51df5d 100644 --- a/brouter-core/src/main/java/btools/router/RoutingEngine.java +++ b/brouter-core/src/main/java/btools/router/RoutingEngine.java @@ -173,6 +173,7 @@ public class RoutingEngine extends Thread { continue; } + oldTrack = null; track.writeGpx( filename ); foundTrack = track; alternativeIndex = i; @@ -573,7 +574,10 @@ public class RoutingEngine extends Thread logInfo( "NodesCache status before reset=" + nodesCache.formatStatus() ); } nodesMap = new OsmNodesMap(); - nodesCache = new NodesCache(segmentDir, nodesMap, routingContext.expctxWay, routingContext.forceSecondaryData, nodesCache ); + + long maxmem = routingContext.memoryclass * 131072L; // 1/4 of total + + nodesCache = new NodesCache(segmentDir, nodesMap, routingContext.expctxWay, routingContext.forceSecondaryData, maxmem, nodesCache ); } private OsmNode getStartNode( long startId ) @@ -699,6 +703,20 @@ public class RoutingEngine extends Thread } private OsmTrack findTrack( String operationName, MatchedWaypoint startWp, MatchedWaypoint endWp, OsmTrack costCuttingTrack, OsmTrack refTrack, boolean fastPartialRecalc ) + { + try + { + resetCache(); + return _findTrack( operationName, startWp, endWp, costCuttingTrack, refTrack, fastPartialRecalc ); + } + finally + { + nodesCache.cleanNonVirgin(); + } + } + + + private OsmTrack _findTrack( String operationName, MatchedWaypoint startWp, MatchedWaypoint endWp, OsmTrack costCuttingTrack, OsmTrack refTrack, boolean fastPartialRecalc ) { boolean verbose = guideTrack != null; @@ -711,7 +729,6 @@ public class RoutingEngine extends Thread matchPath = null; int nodesVisited = 0; - resetCache(); long endNodeId1 = endWp == null ? -1L : endWp.node1.getIdFromPos(); long endNodeId2 = endWp == null ? -1L : endWp.node2.getIdFromPos(); long startNodeId1 = startWp.node1.getIdFromPos(); diff --git a/brouter-expressions/src/main/java/btools/expressions/BExpression.java b/brouter-expressions/src/main/java/btools/expressions/BExpression.java index 9fb9725..c66fc45 100644 --- a/brouter-expressions/src/main/java/btools/expressions/BExpression.java +++ b/brouter-expressions/src/main/java/btools/expressions/BExpression.java @@ -160,6 +160,7 @@ final class BExpression { throw new IllegalArgumentException( "unknown lookup name: " + name ); } + ctx.markLookupIdxUsed( exp.lookupNameIdx ); StringTokenizer tk = new StringTokenizer( values, "|" ); int nt = tk.countTokens(); int nt2 = nt == 0 ? 1 : nt; diff --git a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java index fc5ad6f..c1122bd 100644 --- a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java +++ b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java @@ -37,6 +37,7 @@ public abstract class BExpressionContext implements IByteArrayUnifier private ArrayList lookupValues = new ArrayList(); private ArrayList lookupNames = new ArrayList(); private ArrayList lookupHistograms = new ArrayList(); + private boolean[] lookupIdxUsed; private boolean lookupDataFrozen = false; @@ -255,6 +256,8 @@ public abstract class BExpressionContext implements IByteArrayUnifier // post-process metadata: lookupDataFrozen = true; + + lookupIdxUsed = new boolean[lookupValues.size()]; } public final void evaluate( int[] lookupData2 ) @@ -280,6 +283,8 @@ public abstract class BExpressionContext implements IByteArrayUnifier { return "requests=" + requests + " requests2=" + requests2 + " cachemisses=" + cachemisses; } + + private CacheNode lastCacheNode = new CacheNode(); // @Override public final byte[] unify( byte[] ab, int offset, int len ) @@ -303,6 +308,7 @@ public abstract class BExpressionContext implements IByteArrayUnifier } if ( cn != null ) { + lastCacheNode = cn; return cn.ab; } } @@ -318,10 +324,18 @@ public abstract class BExpressionContext implements IByteArrayUnifier requests++; lookupDataValid = false; // this is an assertion for a nasty pifall - probeCacheNode.ab = ab; - probeCacheNode.crc = Crc32.crc( ab, 0, ab.length ); + CacheNode cn; + if ( lastCacheNode.ab == ab ) + { + cn = lastCacheNode; + } + else + { + probeCacheNode.ab = ab; + probeCacheNode.crc = Crc32.crc( ab, 0, ab.length ); + cn = (CacheNode)cache.get( probeCacheNode ); + } - CacheNode cn = (CacheNode)cache.get( probeCacheNode ); if ( cn == null ) { cachemisses++; @@ -746,6 +760,16 @@ public abstract class BExpressionContext implements IByteArrayUnifier return num == null ? -1 : num.intValue(); } + public final void markLookupIdxUsed( int idx ) + { + lookupIdxUsed[ idx ] = true; + } + + public final boolean isLookupIdxUsed( int idx ) + { + return idx < lookupIdxUsed.length ? lookupIdxUsed[idx] : false; + } + int getLookupValueIdx( int nameIdx, String value ) { BExpressionLookupValue[] values = lookupValues.get( nameIdx ); diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java b/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java index 4e09088..8ccbc26 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java @@ -32,7 +32,6 @@ public final class NodesCache private DataBuffers dataBuffers; private OsmFile[][] fileRows; - private ArrayList segmentList = new ArrayList(); public WaypointMatcher waypointMatcher; @@ -40,14 +39,17 @@ public final class NodesCache public String first_file_access_name; private long cacheSum = 0; + private long maxmem; + private boolean garbageCollectionEnabled = false; + private boolean ghostCleaningDone = false; public String formatStatus() { - return "collecting=" + garbageCollectionEnabled + " cacheSum=" + cacheSum; + return "collecting=" + garbageCollectionEnabled + " noGhosts=" + ghostCleaningDone + " cacheSum=" + cacheSum; } - public NodesCache( String segmentDir, OsmNodesMap nodesMap, BExpressionContextWay ctxWay, boolean forceSecondaryData, NodesCache oldCache ) + public NodesCache( String segmentDir, OsmNodesMap nodesMap, BExpressionContextWay ctxWay, boolean forceSecondaryData, long maxmem, NodesCache oldCache ) { this.segmentDir = new File( segmentDir ); this.nodesMap = nodesMap; @@ -55,6 +57,7 @@ public final class NodesCache this.lookupVersion = ctxWay.meta.lookupVersion; this.lookupMinorVersion = ctxWay.meta.lookupMinorVersion; this.forceSecondaryData = forceSecondaryData; + this.maxmem = maxmem; first_file_access_failed = false; first_file_access_name = null; @@ -88,12 +91,25 @@ public final class NodesCache secondarySegmentsDir = StorageConfigHelper.getSecondarySegmentDir( segmentDir ); } } + + public void cleanNonVirgin() + { + for ( OsmFile[] fileRow : fileRows ) + { + if ( fileRow == null ) + continue; + for ( OsmFile osmf : fileRow ) + { + osmf.cleanNonVirgin(); + } + } + } // if the cache sum exceeded a threshold, // clean all ghosts and enable garbage collection private void checkEnableCacheCleaning() { - if ( cacheSum < 3000000 || garbageCollectionEnabled ) + if ( cacheSum < maxmem || ghostCleaningDone ) return; for ( int i = 0; i < fileRows.length; i++ ) @@ -104,10 +120,17 @@ public final class NodesCache int nghosts = 0; for ( OsmFile osmf : fileRow ) { - if ( osmf.ghost ) - nghosts++; + if ( garbageCollectionEnabled ) + { + if ( osmf.ghost ) + { + nghosts++; + } + } else + { osmf.cleanAll(); + } } if ( nghosts == 0 ) continue; @@ -121,7 +144,15 @@ public final class NodesCache } fileRows[i] = frow; } - garbageCollectionEnabled = true; + + if ( garbageCollectionEnabled ) + { + ghostCleaningDone = true; + } + else + { + garbageCollectionEnabled = true; + } } public int loadSegmentFor( int ilon, int ilat ) @@ -173,18 +204,10 @@ public final class NodesCache segment = osmf.createMicroCache( ilon, ilat, dataBuffers, expCtxWay, waypointMatcher ); cacheSum += segment.getDataSize(); - if ( segment.getSize() > 0 ) - { - segmentList.add( segment ); - } } else if ( segment.ghost ) { segment.unGhost(); - if ( segment.getSize() > 0 ) - { - segmentList.add( segment ); - } } return segment; } @@ -217,7 +240,7 @@ public final class NodesCache if ( garbageCollectionEnabled ) // garbage collection { - segment.collect( segment.getSize() >> 1 ); + segment.collect( segment.getSize() >> 2 ); // threshold = 1/4 of size is deleted } return !node.isHollow(); @@ -276,27 +299,6 @@ public final class NodesCache return osmf; } - public List getAllNodes() - { - List all = new ArrayList(); - for ( MicroCache segment : segmentList ) - { - ArrayList positions = new ArrayList(); - int size = segment.getSize(); - - for ( int i = 0; i < size; i++ ) - { - long id = segment.getIdForIndex( i ); - OsmNode n = new OsmNode( id ); - n.setHollow(); - nodesMap.put( n ); - positions.add( n ); - } - all.addAll( positions ); - } - return all; - } - public void close() { for ( PhysicalFile f : fileCache.values() ) diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java index 94ca165..6100fa6 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java @@ -201,4 +201,18 @@ final class OsmFile } } + void cleanNonVirgin() + { + int nc = microCaches == null ? 0 : microCaches.length; + for ( int i = 0; i < nc; i++ ) + { + MicroCache mc = microCaches[i]; + if ( mc == null ) + continue; + if ( !mc.virgin ) + { + microCaches[i] = null; + } + } + } } diff --git a/brouter-routing-app/src/main/java/btools/routingapp/BRouterActivity.java b/brouter-routing-app/src/main/java/btools/routingapp/BRouterActivity.java index acc35fe..417ccde 100644 --- a/brouter-routing-app/src/main/java/btools/routingapp/BRouterActivity.java +++ b/brouter-routing-app/src/main/java/btools/routingapp/BRouterActivity.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Set; import android.app.Activity; +import android.app.ActivityManager; import android.app.AlertDialog; import android.app.Dialog; import android.content.Context; @@ -57,8 +58,11 @@ public class BRouterActivity extends Activity implements OnInitListener // Create a bright wake lock mWakeLock = mPowerManager.newWakeLock( PowerManager.SCREEN_BRIGHT_WAKE_LOCK, getClass().getName() ); + ActivityManager am = (ActivityManager) getSystemService(ACTIVITY_SERVICE); + int memoryClass = am.getMemoryClass(); + // instantiate our simulation view and set it as the activity's content - mBRouterView = new BRouterView( this ); + mBRouterView = new BRouterView( this, memoryClass ); mBRouterView.init(); setContentView( mBRouterView ); } diff --git a/brouter-routing-app/src/main/java/btools/routingapp/BRouterView.java b/brouter-routing-app/src/main/java/btools/routingapp/BRouterView.java index 87077f5..73b58b1 100644 --- a/brouter-routing-app/src/main/java/btools/routingapp/BRouterView.java +++ b/brouter-routing-app/src/main/java/btools/routingapp/BRouterView.java @@ -75,14 +75,17 @@ public class BRouterView extends View private int[] imgPixels; + private int memoryClass; + public void stopRouting() { if ( cr != null ) cr.terminate(); } - public BRouterView( Context context ) + public BRouterView( Context context, int memoryClass ) { super( context ); + this.memoryClass = memoryClass; } public void init() @@ -221,10 +224,14 @@ public class BRouterView extends View } catch( Exception e ) { - tracksDir = basedir + "/brouter"; + tracksDir = null; } } } + if ( tracksDir == null ) + { + tracksDir = basedir + "/brouter"; // fallback + } String[] fileNames = new File( profileDir ).list(); ArrayList profiles = new ArrayList(); @@ -514,6 +521,17 @@ public class BRouterView extends View startTime = System.currentTimeMillis(); rc.prepareNogoPoints( nogoList ); rc.nogopoints = nogoList; + + rc.memoryclass = memoryClass; + if ( memoryClass < 16 ) + { + rc.memoryclass = 16; + } + else if ( memoryClass > 256 ) + { + rc.memoryclass = 256; + } + // for profile remote, use ref-track logic same as service interface rc.rawTrackPath = rawTrackPath; @@ -748,7 +766,7 @@ public class BRouterView extends View } else { - String result = "version = BRouter-1.4.4\n" + "distance = " + cr.getDistance() / 1000. + " km\n" + "filtered ascend = " + cr.getAscend() + String result = "version = BRouter-1.4.4\n" + "memory-class = " + memoryClass + "\ndistance = " + cr.getDistance() / 1000. + " km\n" + "filtered ascend = " + cr.getAscend() + " m\n" + "plain ascend = " + cr.getPlainAscend(); rawTrack = cr.getFoundRawTrack(); diff --git a/brouter-server/src/main/java/btools/server/request/ServerHandler.java b/brouter-server/src/main/java/btools/server/request/ServerHandler.java index 573accb..eb81816 100644 --- a/brouter-server/src/main/java/btools/server/request/ServerHandler.java +++ b/brouter-server/src/main/java/btools/server/request/ServerHandler.java @@ -41,7 +41,7 @@ public class ServerHandler extends RequestHandler { public RoutingContext readRoutingContext() { rc = new RoutingContext(); - rc.serversizing = true; + rc.memoryclass = 128; String profile = params.get( "profile" ); // when custom profile replace prefix with directory path diff --git a/brouter-util/src/main/java/btools/util/SortedHeap.java b/brouter-util/src/main/java/btools/util/SortedHeap.java index 943d340..e89ec85 100644 --- a/brouter-util/src/main/java/btools/util/SortedHeap.java +++ b/brouter-util/src/main/java/btools/util/SortedHeap.java @@ -34,35 +34,6 @@ public final class SortedHeap /** * @return the lowest key value, or null if none */ - public V popLowestKeyValue2() - { - int minId = 0; - int minIdx = -1; - for ( int i = 0;; i++ ) - { - int[] ali = al[i]; - if ( ali == null ) - break; - int lpi = lp[i]; - if ( lpi < 4 << i ) - { - int currentId = ali[lpi]; - if ( minIdx < 0 || currentId < minId ) - { - minIdx = i; - minId = currentId; - } - } - } - - if ( minIdx == -1 ) - return null; - - size--; - - return dropLowest( minIdx ); - } - public V popLowestKeyValue() { int idx = firstNonEmpty; @@ -286,24 +257,22 @@ public final class SortedHeap { int div = size / 1000 + 1; - ArrayList res = new ArrayList( size / div ); - int cnt = 0; + ArrayList res = new ArrayList( size / div + 1 ); + int lpi = 0; for ( int i = 1;; i++ ) { int[] ali = al[i]; if ( ali == null ) break; - int lpi = lp[i]; + lpi += lp[i]; Object[] vlai = vla[i]; int n = 4 << i; while (lpi < n) { - if ( ( ++cnt ) % div == 0 ) - { - res.add( (V) vla[i][lpi] ); - } - lpi++; + res.add( (V) vla[i][lpi] ); + lpi += div; } + lpi -= n; } return res; }