package btools.mapcreator; import com.google.protobuf.InvalidProtocolBufferException; import org.openstreetmap.osmosis.osmbinary.Fileformat; import org.openstreetmap.osmosis.osmbinary.Osmformat; import btools.util.LongList; import java.io.IOException; import java.util.*; import java.util.logging.Level; import java.util.zip.DataFormatException; import java.util.zip.Inflater; /** * Converts PBF block data into decoded entities ready to be passed into an Osmosis pipeline. This * class is designed to be passed into a pool of worker threads to allow multi-threaded decoding. *

* @author Brett Henderson */ public class BPbfBlobDecoder { private String blobType; private byte[] rawBlob; private OsmParser parser; /** * Creates a new instance. *

* @param blobType The type of blob. * @param rawBlob The raw data of the blob. * @param listener The listener for receiving decoding results. */ public BPbfBlobDecoder( String blobType, byte[] rawBlob, OsmParser parser ) { this.blobType = blobType; this.rawBlob = rawBlob; this.parser = parser; } public void process() throws Exception { if ("OSMHeader".equals(blobType)) { processOsmHeader(readBlobContent()); } else if ("OSMData".equals(blobType)) { processOsmPrimitives(readBlobContent()); } else { System.out.println("Skipping unrecognised blob type " + blobType); } } private byte[] readBlobContent() throws IOException { Fileformat.Blob blob = Fileformat.Blob.parseFrom(rawBlob); byte[] blobData; if (blob.hasRaw()) { blobData = blob.getRaw().toByteArray(); } else if (blob.hasZlibData()) { Inflater inflater = new Inflater(); inflater.setInput(blob.getZlibData().toByteArray()); blobData = new byte[blob.getRawSize()]; try { inflater.inflate(blobData); } catch (DataFormatException e) { throw new RuntimeException("Unable to decompress PBF blob.", e); } if (!inflater.finished()) { throw new RuntimeException("PBF blob contains incomplete compressed data."); } } else { throw new RuntimeException("PBF blob uses unsupported compression, only raw or zlib may be used."); } return blobData; } private void processOsmHeader( byte[] data ) throws InvalidProtocolBufferException { Osmformat.HeaderBlock header = Osmformat.HeaderBlock.parseFrom(data); // Build the list of active and unsupported features in the file. List supportedFeatures = Arrays.asList("OsmSchema-V0.6", "DenseNodes"); List activeFeatures = new ArrayList(); List unsupportedFeatures = new ArrayList(); for (String feature : header.getRequiredFeaturesList()) { if (supportedFeatures.contains(feature)) { activeFeatures.add(feature); } else { unsupportedFeatures.add(feature); } } // We can't continue if there are any unsupported features. We wait // until now so that we can display all unsupported features instead of // just the first one we encounter. if (unsupportedFeatures.size() > 0) { throw new RuntimeException("PBF file contains unsupported features " + unsupportedFeatures); } } private Map buildTags( List keys, List values, BPbfFieldDecoder fieldDecoder ) { Iterator keyIterator = keys.iterator(); Iterator valueIterator = values.iterator(); if (keyIterator.hasNext()) { Map tags = new HashMap(); while (keyIterator.hasNext()) { String key = fieldDecoder.decodeString(keyIterator.next()); String value = fieldDecoder.decodeString(valueIterator.next()); tags.put(key, value); } return tags; } return null; } private void processNodes( List nodes, BPbfFieldDecoder fieldDecoder ) { for (Osmformat.Node node : nodes) { Map tags = buildTags(node.getKeysList(), node.getValsList(), fieldDecoder); parser.addNode( node.getId(), tags, fieldDecoder.decodeLatitude(node .getLat()), fieldDecoder.decodeLatitude(node.getLon())); } } private void processNodes( Osmformat.DenseNodes nodes, BPbfFieldDecoder fieldDecoder ) { List idList = nodes.getIdList(); List latList = nodes.getLatList(); List lonList = nodes.getLonList(); Iterator keysValuesIterator = nodes.getKeysValsList().iterator(); long nodeId = 0; long latitude = 0; long longitude = 0; for (int i = 0; i < idList.size(); i++) { // Delta decode node fields. nodeId += idList.get(i); latitude += latList.get(i); longitude += lonList.get(i); // Build the tags. The key and value string indexes are sequential // in the same PBF array. Each set of tags is delimited by an index // with a value of 0. Map tags = null; while (keysValuesIterator.hasNext()) { int keyIndex = keysValuesIterator.next(); if (keyIndex == 0) { break; } int valueIndex = keysValuesIterator.next(); if (tags == null) { tags = new HashMap(); } tags.put(fieldDecoder.decodeString(keyIndex), fieldDecoder.decodeString(valueIndex)); } parser.addNode( nodeId, tags, ((double) latitude) / 10000000, ((double) longitude) / 10000000); } } private void processWays( List ways, BPbfFieldDecoder fieldDecoder ) { for (Osmformat.Way way : ways) { Map tags = buildTags(way.getKeysList(), way.getValsList(), fieldDecoder); // Build up the list of way nodes for the way. The node ids are // delta encoded meaning that each id is stored as a delta against // the previous one. long nodeId = 0; LongList wayNodes = new LongList( 16 ); for (long nodeIdOffset : way.getRefsList()) { nodeId += nodeIdOffset; wayNodes.add(nodeId); } parser.addWay( way.getId(), tags, wayNodes ); } } private LongList buildRelationMembers( List memberIds, List memberRoles, List memberTypes, BPbfFieldDecoder fieldDecoder ) { LongList wayIds = new LongList( 16 ); Iterator memberIdIterator = memberIds.iterator(); Iterator memberRoleIterator = memberRoles.iterator(); Iterator memberTypeIterator = memberTypes.iterator(); // Build up the list of relation members for the way. The member ids are // delta encoded meaning that each id is stored as a delta against // the previous one. long refId = 0; while (memberIdIterator.hasNext()) { Osmformat.Relation.MemberType memberType = memberTypeIterator.next(); refId += memberIdIterator.next(); String role = fieldDecoder.decodeString( memberRoleIterator.next() ); if ( memberType == Osmformat.Relation.MemberType.WAY ) // currently just waymembers { wayIds.add( refId ); } } return wayIds; } private void processRelations( List relations, BPbfFieldDecoder fieldDecoder ) { for (Osmformat.Relation relation : relations) { Map tags = buildTags(relation.getKeysList(), relation.getValsList(), fieldDecoder); LongList wayIds = buildRelationMembers( relation.getMemidsList(), relation.getRolesSidList(), relation.getTypesList(), fieldDecoder); parser.addRelation( relation.getId(), tags, wayIds ); } } private void processOsmPrimitives( byte[] data ) throws InvalidProtocolBufferException { Osmformat.PrimitiveBlock block = Osmformat.PrimitiveBlock.parseFrom(data); BPbfFieldDecoder fieldDecoder = new BPbfFieldDecoder(block); for (Osmformat.PrimitiveGroup primitiveGroup : block.getPrimitivegroupList()) { processNodes(primitiveGroup.getDense(), fieldDecoder); processNodes(primitiveGroup.getNodesList(), fieldDecoder); processWays(primitiveGroup.getWaysList(), fieldDecoder); processRelations(primitiveGroup.getRelationsList(), fieldDecoder); } } }