Fabio/brouter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

pre-process speed: read pbf while it is written

Browse source
This commit is contained in:
Arndt Brenschede 2019-09-28 13:00:46 +02:00
parent 98b3f62efb
commit e30ea90c0e
1 changed files with 38 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
misc/pbfparser/OsmParser.java
View file

@ -33,13 +33,49 @@ public class OsmParser extends MapCreatorBase
// once more for testing // once more for testing
int rawBlobCount = 0; int rawBlobCount = 0;
long bytesRead = 0L;
// wait for file to become available
while( !mapFile.exists() )
{
System.out.println( "--- waiting for " + mapFile + " to become available" );
Thread.sleep( 10000 );
}
long currentSize = mapFile.length();
long currentSizeTime = System.currentTimeMillis();
DataInputStream dis = new DataInputStream( new BufferedInputStream ( new FileInputStream( mapFile ) ) ); DataInputStream dis = new DataInputStream( new BufferedInputStream ( new FileInputStream( mapFile ) ) );
for(;;) for(;;)
{ {
// continue reading if either more then a 100 MB unread, or the current-size is known for more then 2 Minutes
while ( currentSize-bytesRead < 100000000L )
{
long newSize = mapFile.length();
if ( newSize != currentSize )
{
currentSize = newSize;
currentSizeTime = System.currentTimeMillis();
}
else if ( System.currentTimeMillis() - currentSizeTime > 120000 )
{
break;
}
if ( currentSize-bytesRead < 100000000L )
{
System.out.println( "--- waiting for more data, currentSize=" + currentSize + " bytesRead=" + bytesRead );
Thread.sleep( 10000 );
}
}
int headerLength; int headerLength;
try try
{ {
headerLength = dis.readInt(); headerLength = dis.readInt();
bytesRead += 4;
} }
catch (EOFException e) catch (EOFException e)
{ {
@ -48,10 +84,12 @@ public class OsmParser extends MapCreatorBase
byte[] headerBuffer = new byte[headerLength]; byte[] headerBuffer = new byte[headerLength];
dis.readFully(headerBuffer); dis.readFully(headerBuffer);
bytesRead += headerLength;
Fileformat.BlobHeader blobHeader = Fileformat.BlobHeader.parseFrom(headerBuffer); Fileformat.BlobHeader blobHeader = Fileformat.BlobHeader.parseFrom(headerBuffer);
byte[] blobData = new byte[blobHeader.getDatasize()]; byte[] blobData = new byte[blobHeader.getDatasize()];
dis.readFully(blobData); dis.readFully(blobData);
bytesRead += blobData.length;
new BPbfBlobDecoder( blobHeader.getType(), blobData, this ).process(); new BPbfBlobDecoder( blobHeader.getType(), blobData, this ).process();