Fabio/brouter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

rd5 diff progress

Browse source
This commit is contained in:
Arndt Brenschede 2020-01-19 15:22:05 +01:00
parent db4757abbb
commit 43de4bb6ca
4 changed files with 227 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java
View file

@ -49,15 +49,20 @@ final public class Rd5DiffManager
// calculate MD5 of old file
String md5 = getMD5( fo );
String md5New = getMD5( fn );
System.out.println( "name=" + name + " md5=" + md5 );
File specificNewDiffs = new File( newDiffDir, basename );
specificNewDiffs.mkdirs();
String diffFileName = md5 + ".rd5diff";
File diffFile = new File( specificNewDiffs, diffFileName );
String dummyDiffFileName = md5New + ".rd5diff";
File dummyDiffFile = new File( specificNewDiffs, dummyDiffFileName );
dummyDiffFile.createNewFile();
// calc the new diff
Rd5DiffTool.diff2files( fo, fn, diffFile );
@ -72,15 +77,17 @@ final public class Rd5DiffManager
{
continue;
}
if ( System.currentTimeMillis() - od.lastModified() > 31*86400000L )
if ( System.currentTimeMillis() - od.lastModified() > 9*86400000L )
{
continue; // limit diff history to 31 days
continue; // limit diff history to 9 days
}
File updatedDiff = new File( specificNewDiffs, od.getName() );
Rd5DiffTool.addDeltas( od, diffFile, updatedDiff );
updatedDiff.setLastModified( od.lastModified() );
if ( !updatedDiff.exists() )
{
Rd5DiffTool.addDeltas( od, diffFile, updatedDiff );
updatedDiff.setLastModified( od.lastModified() );
}
}
}
}

113
brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java
View file

@ -19,8 +19,9 @@ import btools.codec.MicroCache;
import btools.codec.MicroCache2;
import btools.codec.StatCoderContext;
import btools.util.Crc32;
import btools.util.ProgressListener;
final public class Rd5DiffTool
final public class Rd5DiffTool implements ProgressListener
{
public static void main( String[] args ) throws Exception
{
@ -32,7 +33,7 @@ final public class Rd5DiffTool
}
else
{
recoverFromDelta( new File( args[0] ),new File( args[1] ), new File( args[2] ) /*, new File( args[3] ) */ );
recoverFromDelta( new File( args[0] ),new File( args[1] ), new File( args[2] ), new Rd5DiffTool() /*, new File( args[3] ) */ );
}
}
else
@ -41,6 +42,18 @@ final public class Rd5DiffTool
}
}
@Override
public void updateProgress( String progress )
{
System.out.println( progress );
}
@Override
public boolean isCanceled()
{
return false;
}
private static long[] readFileIndex( DataInputStream dis, DataOutputStream dos ) throws Exception
{
long[] fileIndex = new long[25];
@ -275,11 +288,21 @@ final public class Rd5DiffTool
}
public static void recoverFromDelta( File f1, File f2, File outFile /* , File cmpFile */ ) throws Exception
public static void recoverFromDelta( File f1, File f2, File outFile, ProgressListener progress /* , File cmpFile */ ) throws Exception
{
if ( f2.length() == 0L )
{
copyFile( f1, outFile, progress );
return;
}
byte[] abBuf1 = new byte[10 * 1024 * 1024];
byte[] abBuf2 = new byte[10 * 1024 * 1024];
boolean canceled = false;
long t0 = System.currentTimeMillis();
DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) );
// DataInputStream disCmp = new DataInputStream( new BufferedInputStream( new FileInputStream( cmpFile ) ) );
@ -290,6 +313,8 @@ final public class Rd5DiffTool
long[] fileIndex2 = readFileIndex( dis2, dos );
// long[] fileIndexCmp = readFileIndex( disCmp, null );
int lastPct = -1;
try
{
DataBuffers dataBuffers = new DataBuffers();
@ -307,6 +332,19 @@ final public class Rd5DiffTool
for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ )
{
if ( progress.isCanceled() )
{
canceled = true;
return;
}
double bytesProcessed = getTileStart( fileIndex1, subFileIdx ) + ( posIdx1 == null ? 0 : getPosIdx( posIdx1, tileIdx-1 ) );
int pct = (int)(100. * bytesProcessed / getTileEnd( fileIndex1, 24 ) + 0.5 );
if ( pct != lastPct )
{
progress.updateProgress( "Applying delta: " + pct + "%" );
lastPct = pct;
}
byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false );
byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true );
if ( ab2 == null )
@ -359,6 +397,7 @@ final public class Rd5DiffTool
*/
dos.write( abBuf1, 0, len );
dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 );
}
}
// write any remaining data to the output file
@ -371,6 +410,8 @@ final public class Rd5DiffTool
}
dos.write( abBuf1, 0, len );
}
long t1 = System.currentTimeMillis();
System.out.println( "recovering from diffs took " + (t1-t0) + "ms" );
}
finally
{
@ -403,6 +444,72 @@ final public class Rd5DiffTool
catch (Exception ee)
{
}
if ( canceled )
{
outFile.delete();
}
}
}
}
public static void copyFile( File f1, File outFile, ProgressListener progress ) throws Exception
{
boolean canceled = false;
DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
int lastPct = -1;
long sizeTotal = f1.length();
long sizeRead = 0L;
try
{
byte buf[] = new byte[65536];
for (;;)
{
if ( progress.isCanceled() )
{
canceled = true;
return;
}
int pct = (int)( (100. * sizeRead) / (sizeTotal+1) + 0.5 );
if ( pct != lastPct )
{
progress.updateProgress( "Copying: " + pct + "%" );
lastPct = pct;
}
int len = dis1.read( buf );
if ( len <= 0 )
{
break;
}
sizeRead += len;
dos.write( buf, 0, len );
}
}
finally
{
if ( dis1 != null )
{
try
{
dis1.close();
}
catch (Exception ee)
{
}
}
if ( dos != null )
{
try
{
dos.close();
}
catch (Exception ee)
{
}
if ( canceled )
{
outFile.delete();
}
}
}
}

116
brouter-routing-app/src/main/java/btools/routingapp/BInstallerView.java
View file

@ -25,7 +25,10 @@ import android.view.MotionEvent;
import android.view.View;
import android.widget.Toast;
import btools.mapaccess.PhysicalFile;
import btools.mapaccess.Rd5DiffManager;
import btools.mapaccess.Rd5DiffTool;
import btools.router.RoutingHelper;
import btools.util.ProgressListener;
public class BInstallerView extends View
{
@ -61,7 +64,9 @@ public class BInstallerView extends View
private long currentDownloadSize;
private String currentDownloadFile = "";
private volatile String currentDownloadOperation = "";
private String downloadAction = "";
private volatile String newDownloadAction = "";
private long totalSize = 0;
private long rd5Tiles = 0;
@ -144,6 +149,7 @@ public class BInstallerView extends View
String namebase = baseNameForTile( tileIndex );
String baseurl = "http://brouter.de/brouter/segments4/";
currentDownloadFile = namebase + ".rd5";
currentDownloadOperation = "Checking";
String url = baseurl + currentDownloadFile;
isDownloading = true;
downloadCanceled = false;
@ -241,7 +247,7 @@ public class BInstallerView extends View
tileStatus[tidx] |= MASK_INSTALLED_RD5;
long age = System.currentTimeMillis() - new File( dir, fileName ).lastModified();
if ( age < 86400000 ) tileStatus[tidx] |= MASK_CURRENT_RD5;
if ( age < 300000 ) tileStatus[tidx] |= MASK_CURRENT_RD5; // 5 minutes
}
}
}
@ -368,7 +374,7 @@ public class BInstallerView extends View
{
String sizeHint = currentDownloadSize > 0 ? " (" + ((currentDownloadSize + mb-1)/mb) + " MB)" : "";
paint.setTextSize(30);
canvas.drawText( "Loading " + currentDownloadFile + sizeHint, 30, (imgh/3)*2-30, paint);
canvas.drawText( currentDownloadOperation + " " + currentDownloadFile + sizeHint, 30, (imgh/3)*2-30, paint);
canvas.drawText( downloadAction, 30, (imgh/3)*2, paint);
}
if ( !tilesVisible )
@ -573,7 +579,7 @@ float tx, ty;
// usually, subclasses of AsyncTask are declared inside the activity class.
// that way, you can easily modify the UI thread from here
private class DownloadTask extends AsyncTask<String, Integer, String> {
private class DownloadTask extends AsyncTask<String, Integer, String> implements ProgressListener {
private Context context;
private PowerManager.WakeLock mWakeLock;
@ -583,6 +589,19 @@ float tx, ty;
}
@Override
public void updateProgress( String progress )
{
newDownloadAction = progress;
publishProgress( 0 );
}
@Override
public boolean isCanceled()
{
return isDownloadCanceled();
}
@Override
protected String doInBackground(String... sUrls)
{
InputStream input = null;
@ -595,16 +614,48 @@ float tx, ty;
{
try
{
URL url = new URL(surl);
int slidx = surl.lastIndexOf( "segments4/" );
String name = surl.substring( slidx+10 );
String surlBase = surl.substring( 0, slidx+10 );
fname = baseDir + "/brouter/segments4/" + name;
connection = (HttpURLConnection) url.openConnection();
connection.connect();
boolean delta = true;
File targetFile = new File( fname );
if ( targetFile.exists() )
{
updateProgress( "Calculating local checksum.." );
// first check for a delta file
String md5 = Rd5DiffManager.getMD5( targetFile );
String surlDelta = surlBase + "diff/" + name.replace( ".rd5", "/" + md5 + ".rd5diff" );
URL urlDelta = new URL(surlDelta);
updateProgress( "Connecting.." );
connection = (HttpURLConnection) urlDelta.openConnection();
connection.connect();
// 404 kind of expected here, means there's no delta file
if (connection.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND )
{
connection = null;
}
}
if ( connection == null )
{
delta = false;
URL url = new URL(surl);
connection = (HttpURLConnection) url.openConnection();
connection.connect();
}
// expect HTTP 200 OK, so we don't mistakenly save error report
// instead of the file
if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
return "Server returned HTTP " + connection.getResponseCode()
+ " " + connection.getResponseMessage();
+ " " + connection.getResponseMessage();
}
// this will be useful to display download percentage
@ -613,12 +664,12 @@ float tx, ty;
currentDownloadSize = fileLength;
if ( availableSize >= 0 && fileLength > availableSize ) return "not enough space on sd-card";
currentDownloadOperation = delta ? "Updating" : "Loading";
// download the file
input = connection.getInputStream();
int slidx = surl.lastIndexOf( "segments4/" );
fname = baseDir + "/brouter/segments4/" + surl.substring( slidx+10 );
tmp_file = new File( fname + "_tmp" );
tmp_file = new File( fname + ( delta ? "_diff" : "_tmp" ) );
output = new FileOutputStream( tmp_file );
byte data[] = new byte[4096];
@ -632,7 +683,15 @@ float tx, ty;
total += count;
// publishing the progress....
if (fileLength > 0) // only if total length is known
publishProgress((int) (total * 100 / fileLength));
{
int pct = (int) (total * 100 / fileLength);
updateProgress( "Progress " + pct + "%" );
}
else
{
updateProgress( "Progress (unnown size)" );
}
output.write(data, 0, count);
// enforce < 2 Mbit/s
@ -642,15 +701,33 @@ float tx, ty;
try { Thread.sleep( dt ); } catch( InterruptedException ie ) {}
}
}
publishProgress( 101 );
String check_result = PhysicalFile.checkFileIntegrity( tmp_file );
if ( check_result != null ) return check_result;
output.close();
output = null;
if ( !tmp_file.renameTo( new File( fname ) ) )
if ( delta )
{
return "Could not rename to " + fname;
updateProgress( "Applying delta.." );
File diffFile = tmp_file;
tmp_file = new File( fname + "_tmp" );
Rd5DiffTool.recoverFromDelta( targetFile, diffFile, tmp_file, this );
diffFile.delete();
}
if (isDownloadCanceled())
{
return "Canceled!";
}
if ( tmp_file != null )
{
updateProgress( "Verifying integrity.." );
String check_result = PhysicalFile.checkFileIntegrity( tmp_file );
if ( check_result != null ) return check_result;
if ( !tmp_file.renameTo( targetFile ) )
{
return "Could not rename to " + targetFile;
}
deleteRawTracks(); // invalidate raw-tracks after data update
}
deleteRawTracks(); // invalidate raw-tracks after data update
return null;
} catch (Exception e) {
return e.toString();
@ -685,10 +762,9 @@ float tx, ty;
@Override
protected void onProgressUpdate(Integer... progress) {
String newAction = progress[0] == 101 ? "Verifying.." : "Progress " + progress[0] + "%";
if ( !newAction.equals( downloadAction ) )
if ( !newDownloadAction.equals( downloadAction ) )
{
downloadAction = newAction;
downloadAction = newDownloadAction;
invalidate();
}
}

8
brouter-util/src/main/java/btools/util/ProgressListener.java Normal file
View file

@ -0,0 +1,8 @@
package btools.util;
public interface ProgressListener
{
public void updateProgress( String progress );
public boolean isCanceled();
}