tileserver-gl/src/serve_rendered.js

832 lines
26 KiB
JavaScript

'use strict';
const advancedPool = require('advanced-pool');
const fs = require('fs');
const path = require('path');
const url = require('url');
const util = require('util');
const zlib = require('zlib');
// sharp has to be required before node-canvas
// see https://github.com/lovell/sharp/issues/371
const sharp = require('sharp');
const { createCanvas } = require('canvas');
const clone = require('clone');
const Color = require('color');
const express = require('express');
const mercator = new (require('@mapbox/sphericalmercator'))();
const mbgl = require('@mapbox/mapbox-gl-native');
const MBTiles = require('@mapbox/mbtiles');
const proj4 = require('proj4');
const request = require('request');
const utils = require('./utils');
const FLOAT_PATTERN = '[+-]?(?:\\d+|\\d+\.?\\d+)';
const httpTester = /^(http(s)?:)?\/\//;
const getScale = scale => (scale || '@1x').slice(1, 2) | 0;
mbgl.on('message', e => {
if (e.severity === 'WARNING' || e.severity === 'ERROR') {
console.log('mbgl:', e);
}
});
/**
* Lookup of sharp output formats by file extension.
*/
const extensionToFormat = {
'.jpg': 'jpeg',
'.jpeg': 'jpeg',
'.png': 'png',
'.webp': 'webp'
};
/**
* Cache of response data by sharp output format and color. Entry for empty
* string is for unknown or unsupported formats.
*/
const cachedEmptyResponses = {
'': Buffer.alloc(0)
};
/**
* Create an appropriate mbgl response for http errors.
* @param {string} format The format (a sharp format or 'pbf').
* @param {string} color The background color (or empty string for transparent).
* @param {Function} callback The mbgl callback.
*/
function createEmptyResponse(format, color, callback) {
if (!format || format === 'pbf') {
callback(null, { data: cachedEmptyResponses[''] });
return;
}
if (format === 'jpg') {
format = 'jpeg';
}
if (!color) {
color = 'rgba(255,255,255,0)';
}
const cacheKey = `${format},${color}`;
const data = cachedEmptyResponses[cacheKey];
if (data) {
callback(null, { data: data });
return;
}
// create an "empty" response image
color = new Color(color);
const array = color.array();
const channels = array.length === 4 && format !== 'jpeg' ? 4 : 3;
sharp(Buffer.from(array), {
raw: {
width: 1,
height: 1,
channels: channels
}
}).toFormat(format).toBuffer((err, buffer, info) => {
if (!err) {
cachedEmptyResponses[cacheKey] = buffer;
}
callback(null, { data: buffer });
});
}
const extractPathFromQuery = (query, transformer) => {
const pathParts = (query.path || '').split('|');
const path = [];
for (const pair of pathParts) {
const pairParts = pair.split(',');
if (pairParts.length === 2) {
let pair;
if (query.latlng === '1' || query.latlng === 'true') {
pair = [+(pairParts[1]), +(pairParts[0])];
} else {
pair = [+(pairParts[0]), +(pairParts[1])];
}
if (transformer) {
pair = transformer(pair);
}
path.push(pair);
}
}
return path;
};
const renderOverlay = (z, x, y, bearing, pitch, w, h, scale,
path, query) => {
if (!path || path.length < 2) {
return null;
}
const precisePx = (ll, zoom) => {
const px = mercator.px(ll, 20);
const scale = Math.pow(2, zoom - 20);
return [px[0] * scale, px[1] * scale];
};
const center = precisePx([x, y], z);
const mapHeight = 512 * (1 << z);
const maxEdge = center[1] + h / 2;
const minEdge = center[1] - h / 2;
if (maxEdge > mapHeight) {
center[1] -= (maxEdge - mapHeight);
} else if (minEdge < 0) {
center[1] -= minEdge;
}
const canvas = createCanvas(scale * w, scale * h);
const ctx = canvas.getContext('2d');
ctx.scale(scale, scale);
if (bearing) {
ctx.translate(w / 2, h / 2);
ctx.rotate(-bearing / 180 * Math.PI);
ctx.translate(-center[0], -center[1]);
} else {
// optimized path
ctx.translate(-center[0] + w / 2, -center[1] + h / 2);
}
const lineWidth = query.width !== undefined ?
parseFloat(query.width) : 1;
ctx.lineWidth = lineWidth;
ctx.strokeStyle = query.stroke || 'rgba(0,64,255,0.7)';
ctx.fillStyle = query.fill || 'rgba(255,255,255,0.4)';
ctx.beginPath();
for (const pair of path) {
const px = precisePx(pair, z);
ctx.lineTo(px[0], px[1]);
}
if (path[0][0] === path[path.length - 1][0] &&
path[0][1] === path[path.length - 1][1]) {
ctx.closePath();
}
ctx.fill();
if (lineWidth > 0) {
ctx.stroke();
}
return canvas.toBuffer();
};
const calcZForBBox = (bbox, w, h, query) => {
let z = 25;
const padding = query.padding !== undefined ?
parseFloat(query.padding) : 0.1;
const minCorner = mercator.px([bbox[0], bbox[3]], z),
maxCorner = mercator.px([bbox[2], bbox[1]], z);
const w_ = w / (1 + 2 * padding);
const h_ = h / (1 + 2 * padding);
z -= Math.max(
Math.log((maxCorner[0] - minCorner[0]) / w_),
Math.log((maxCorner[1] - minCorner[1]) / h_)
) / Math.LN2;
z = Math.max(Math.log(Math.max(w, h) / 256) / Math.LN2, Math.min(25, z));
return z;
};
const existingFonts = {};
let maxScaleFactor = 2;
module.exports = {
init: (options, repo) => {
const fontListingPromise = new Promise((resolve, reject) => {
fs.readdir(options.paths.fonts, (err, files) => {
if (err) {
reject(err);
return;
}
for (const file of files) {
fs.stat(path.join(options.paths.fonts, file), (err, stats) => {
if (err) {
reject(err);
return;
}
if (stats.isDirectory()) {
existingFonts[path.basename(file)] = true;
}
});
}
resolve();
});
});
maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9);
let scalePattern = '';
for (let i = 2; i <= maxScaleFactor; i++) {
scalePattern += i.toFixed();
}
scalePattern = `@[${scalePattern}]x`;
const app = express().disable('x-powered-by');
const respondImage = (item, z, lon, lat, bearing, pitch,
width, height, scale, format, res, next,
opt_overlay) => {
if (Math.abs(lon) > 180 || Math.abs(lat) > 85.06 ||
lon !== lon || lat !== lat) {
return res.status(400).send('Invalid center');
}
if (Math.min(width, height) <= 0 ||
Math.max(width, height) * scale > (options.maxSize || 2048) ||
width !== width || height !== height) {
return res.status(400).send('Invalid size');
}
if (format === 'png' || format === 'webp') {
} else if (format === 'jpg' || format === 'jpeg') {
format = 'jpeg';
} else {
return res.status(400).send('Invalid format');
}
const pool = item.map.renderers[scale];
pool.acquire((err, renderer) => {
const mbglZ = Math.max(0, z - 1);
const params = {
zoom: mbglZ,
center: [lon, lat],
bearing: bearing,
pitch: pitch,
width: width,
height: height
};
if (z === 0) {
params.width *= 2;
params.height *= 2;
}
const tileMargin = Math.max(options.tileMargin || 0, 0);
if (z > 2 && tileMargin > 0) {
params.width += tileMargin * 2;
params.height += tileMargin * 2;
}
renderer.render(params, (err, data) => {
pool.release(renderer);
if (err) {
console.error(err);
return;
}
// Fix semi-transparent outlines on raw, premultiplied input
// https://github.com/maptiler/tileserver-gl/issues/350#issuecomment-477857040
for (var i = 0; i < data.length; i += 4) {
var alpha = data[i + 3];
var norm = alpha / 255;
if (alpha === 0) {
data[i] = 0;
data[i + 1] = 0;
data[i + 2] = 0;
} else {
data[i] = data[i] / norm;
data[i + 1] = data[i + 1] / norm;
data[i + 2] = data[i + 2] / norm;
}
}
const image = sharp(data, {
raw: {
width: params.width * scale,
height: params.height * scale,
channels: 4
}
});
if (z > 2 && tileMargin > 0) {
image.extract({
left: tileMargin * scale,
top: tileMargin * scale,
width: width * scale,
height: height * scale
});
}
if (z === 0) {
// HACK: when serving zoom 0, resize the 0 tile from 512 to 256
image.resize(width * scale, height * scale);
}
if (opt_overlay) {
image.composite([{ input: opt_overlay }]);
}
if (item.watermark) {
const canvas = createCanvas(scale * width, scale * height);
const ctx = canvas.getContext('2d');
ctx.scale(scale, scale);
ctx.font = '10px sans-serif';
ctx.strokeWidth = '1px';
ctx.strokeStyle = 'rgba(255,255,255,.4)';
ctx.strokeText(item.watermark, 5, height - 5);
ctx.fillStyle = 'rgba(0,0,0,.4)';
ctx.fillText(item.watermark, 5, height - 5);
image.composite([{ input: canvas.toBuffer() }]);
}
const formatQuality = (options.formatQuality || {})[format];
if (format === 'png') {
image.png({ adaptiveFiltering: false });
} else if (format === 'jpeg') {
image.jpeg({ quality: formatQuality || 80 });
} else if (format === 'webp') {
image.webp({ quality: formatQuality || 90 });
}
image.toBuffer((err, buffer, info) => {
if (!buffer) {
return res.status(404).send('Not found');
}
res.set({
'Last-Modified': item.lastModified,
'Content-Type': `image/${format}`
});
return res.status(200).send(buffer);
});
});
});
};
app.get(`/:id/:z(\\d+)/:x(\\d+)/:y(\\d+):scale(${scalePattern})?.:format([\\w]+)`, (req, res, next) => {
const item = repo[req.params.id];
if (!item) {
return res.sendStatus(404);
}
const modifiedSince = req.get('if-modified-since'), cc = req.get('cache-control');
if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) {
if (new Date(item.lastModified) <= new Date(modifiedSince)) {
return res.sendStatus(304);
}
}
const z = req.params.z | 0,
x = req.params.x | 0,
y = req.params.y | 0,
scale = getScale(req.params.scale),
format = req.params.format;
if (z < 0 || x < 0 || y < 0 ||
z > 20 || x >= Math.pow(2, z) || y >= Math.pow(2, z)) {
return res.status(404).send('Out of bounds');
}
const tileSize = 256;
const tileCenter = mercator.ll([
((x + 0.5) / (1 << z)) * (256 << z),
((y + 0.5) / (1 << z)) * (256 << z)
], z);
return respondImage(item, z, tileCenter[0], tileCenter[1], 0, 0,
tileSize, tileSize, scale, format, res, next);
});
if (options.serveStaticMaps !== false) {
const staticPattern =
`/:id/static/:raw(raw)?/%s/:width(\\d+)x:height(\\d+):scale(${scalePattern})?.:format([\\w]+)`;
const centerPattern =
util.format(':x(%s),:y(%s),:z(%s)(@:bearing(%s)(,:pitch(%s))?)?',
FLOAT_PATTERN, FLOAT_PATTERN, FLOAT_PATTERN,
FLOAT_PATTERN, FLOAT_PATTERN);
app.get(util.format(staticPattern, centerPattern), (req, res, next) => {
const item = repo[req.params.id];
if (!item) {
return res.sendStatus(404);
}
const raw = req.params.raw;
let z = +req.params.z,
x = +req.params.x,
y = +req.params.y,
bearing = +(req.params.bearing || '0'),
pitch = +(req.params.pitch || '0'),
w = req.params.width | 0,
h = req.params.height | 0,
scale = getScale(req.params.scale),
format = req.params.format;
if (z < 0) {
return res.status(404).send('Invalid zoom');
}
const transformer = raw ?
mercator.inverse.bind(mercator) : item.dataProjWGStoInternalWGS;
if (transformer) {
const ll = transformer([x, y]);
x = ll[0];
y = ll[1];
}
const path = extractPathFromQuery(req.query, transformer);
const overlay = renderOverlay(z, x, y, bearing, pitch, w, h, scale,
path, req.query);
return respondImage(item, z, x, y, bearing, pitch, w, h, scale, format,
res, next, overlay);
});
const serveBounds = (req, res, next) => {
const item = repo[req.params.id];
if (!item) {
return res.sendStatus(404);
}
const raw = req.params.raw;
const bbox = [+req.params.minx, +req.params.miny,
+req.params.maxx, +req.params.maxy];
let center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2];
const transformer = raw ?
mercator.inverse.bind(mercator) : item.dataProjWGStoInternalWGS;
if (transformer) {
const minCorner = transformer(bbox.slice(0, 2));
const maxCorner = transformer(bbox.slice(2));
bbox[0] = minCorner[0];
bbox[1] = minCorner[1];
bbox[2] = maxCorner[0];
bbox[3] = maxCorner[1];
center = transformer(center);
}
const w = req.params.width | 0,
h = req.params.height | 0,
scale = getScale(req.params.scale),
format = req.params.format;
const z = calcZForBBox(bbox, w, h, req.query),
x = center[0],
y = center[1],
bearing = 0,
pitch = 0;
const path = extractPathFromQuery(req.query, transformer);
const overlay = renderOverlay(z, x, y, bearing, pitch, w, h, scale,
path, req.query);
return respondImage(item, z, x, y, bearing, pitch, w, h, scale, format,
res, next, overlay);
};
const boundsPattern =
util.format(':minx(%s),:miny(%s),:maxx(%s),:maxy(%s)',
FLOAT_PATTERN, FLOAT_PATTERN, FLOAT_PATTERN, FLOAT_PATTERN);
app.get(util.format(staticPattern, boundsPattern), serveBounds);
app.get('/:id/static/', (req, res, next) => {
for (let key in req.query) {
req.query[key.toLowerCase()] = req.query[key];
}
req.params.raw = true;
req.params.format = (req.query.format || 'image/png').split('/').pop();
const bbox = (req.query.bbox || '').split(',');
req.params.minx = bbox[0];
req.params.miny = bbox[1];
req.params.maxx = bbox[2];
req.params.maxy = bbox[3];
req.params.width = req.query.width || '256';
req.params.height = req.query.height || '256';
if (req.query.scale) {
req.params.width /= req.query.scale;
req.params.height /= req.query.scale;
req.params.scale = `@${req.query.scale}`;
}
return serveBounds(req, res, next);
});
const autoPattern = 'auto';
app.get(util.format(staticPattern, autoPattern), (req, res, next) => {
const item = repo[req.params.id];
if (!item) {
return res.sendStatus(404);
}
const raw = req.params.raw;
const w = req.params.width | 0,
h = req.params.height | 0,
bearing = 0,
pitch = 0,
scale = getScale(req.params.scale),
format = req.params.format;
const transformer = raw ?
mercator.inverse.bind(mercator) : item.dataProjWGStoInternalWGS;
const path = extractPathFromQuery(req.query, transformer);
if (path.length < 2) {
return res.status(400).send('Invalid path');
}
const bbox = [Infinity, Infinity, -Infinity, -Infinity];
for (const pair of path) {
bbox[0] = Math.min(bbox[0], pair[0]);
bbox[1] = Math.min(bbox[1], pair[1]);
bbox[2] = Math.max(bbox[2], pair[0]);
bbox[3] = Math.max(bbox[3], pair[1]);
}
const bbox_ = mercator.convert(bbox, '900913');
const center = mercator.inverse(
[(bbox_[0] + bbox_[2]) / 2, (bbox_[1] + bbox_[3]) / 2]
);
const z = calcZForBBox(bbox, w, h, req.query),
x = center[0],
y = center[1];
const overlay = renderOverlay(z, x, y, bearing, pitch, w, h, scale,
path, req.query);
return respondImage(item, z, x, y, bearing, pitch, w, h, scale, format,
res, next, overlay);
});
}
app.get('/:id.json', (req, res, next) => {
const item = repo[req.params.id];
if (!item) {
return res.sendStatus(404);
}
const info = clone(item.tileJSON);
info.tiles = utils.getTileUrls(req, info.tiles,
`styles/${req.params.id}`, info.format, item.publicUrl);
return res.send(info);
});
return Promise.all([fontListingPromise]).then(() => app);
},
add: (options, repo, params, id, publicUrl, dataResolver) => {
const map = {
renderers: [],
sources: {}
};
let styleJSON;
const createPool = (ratio, min, max) => {
const createRenderer = (ratio, createCallback) => {
const renderer = new mbgl.Map({
mode: "tile",
ratio: ratio,
request: (req, callback) => {
const protocol = req.url.split(':')[0];
//console.log('Handling request:', req);
if (protocol === 'sprites') {
const dir = options.paths[protocol];
const file = unescape(req.url).substring(protocol.length + 3);
fs.readFile(path.join(dir, file), (err, data) => {
callback(err, { data: data });
});
} else if (protocol === 'fonts') {
const parts = req.url.split('/');
const fontstack = unescape(parts[2]);
const range = parts[3].split('.')[0];
utils.getFontsPbf(
null, options.paths[protocol], fontstack, range, existingFonts
).then(concated => {
callback(null, { data: concated });
}, err => {
callback(err, { data: null });
});
} else if (protocol === 'mbtiles') {
const parts = req.url.split('/');
const sourceId = parts[2];
const source = map.sources[sourceId];
const sourceInfo = styleJSON.sources[sourceId];
const z = parts[3] | 0,
x = parts[4] | 0,
y = parts[5].split('.')[0] | 0,
format = parts[5].split('.')[1];
source.getTile(z, x, y, (err, data, headers) => {
if (err) {
if (options.verbose) console.log('MBTiles error, serving empty', err);
createEmptyResponse(sourceInfo.format, sourceInfo.color, callback);
return;
}
const response = {};
if (headers['Last-Modified']) {
response.modified = new Date(headers['Last-Modified']);
}
if (format === 'pbf') {
try {
response.data = zlib.unzipSync(data);
} catch (err) {
console.log("Skipping incorrect header for tile mbtiles://%s/%s/%s/%s.pbf", id, z, x, y);
}
if (options.dataDecoratorFunc) {
response.data = options.dataDecoratorFunc(
sourceId, 'data', response.data, z, x, y);
}
} else {
response.data = data;
}
callback(null, response);
});
} else if (protocol === 'http' || protocol === 'https') {
request({
url: req.url,
encoding: null,
gzip: true
}, (err, res, body) => {
const parts = url.parse(req.url);
const extension = path.extname(parts.pathname).toLowerCase();
const format = extensionToFormat[extension] || '';
if (err || res.statusCode < 200 || res.statusCode >= 300) {
// console.log('HTTP error', err || res.statusCode);
createEmptyResponse(format, '', callback);
return;
}
const response = {};
if (res.headers.modified) {
response.modified = new Date(res.headers.modified);
}
if (res.headers.expires) {
response.expires = new Date(res.headers.expires);
}
if (res.headers.etag) {
response.etag = res.headers.etag;
}
response.data = body;
callback(null, response);
});
}
}
});
renderer.load(styleJSON);
createCallback(null, renderer);
};
return new advancedPool.Pool({
min: min,
max: max,
create: createRenderer.bind(null, ratio),
destroy: renderer => {
renderer.release();
}
});
};
const styleFile = params.style;
const styleJSONPath = path.resolve(options.paths.styles, styleFile);
try {
styleJSON = JSON.parse(fs.readFileSync(styleJSONPath));
} catch (e) {
console.log('Error parsing style file');
return false;
}
if (styleJSON.sprite && !httpTester.test(styleJSON.sprite)) {
styleJSON.sprite = 'sprites://' +
styleJSON.sprite
.replace('{style}', path.basename(styleFile, '.json'))
.replace('{styleJsonFolder}', path.relative(options.paths.sprites, path.dirname(styleJSONPath)));
}
if (styleJSON.glyphs && !httpTester.test(styleJSON.glyphs)) {
styleJSON.glyphs = `fonts://${styleJSON.glyphs}`;
}
for (const layer of (styleJSON.layers || [])) {
if (layer && layer.paint) {
// Remove (flatten) 3D buildings
if (layer.paint['fill-extrusion-height']) {
layer.paint['fill-extrusion-height'] = 0;
}
if (layer.paint['fill-extrusion-base']) {
layer.paint['fill-extrusion-base'] = 0;
}
}
}
const tileJSON = {
'tilejson': '2.0.0',
'name': styleJSON.name,
'attribution': '',
'minzoom': 0,
'maxzoom': 20,
'bounds': [-180, -85.0511, 180, 85.0511],
'format': 'png',
'type': 'baselayer'
};
const attributionOverride = params.tilejson && params.tilejson.attribution;
Object.assign(tileJSON, params.tilejson || {});
tileJSON.tiles = params.domains || options.domains;
utils.fixTileJSONCenter(tileJSON);
repo[id] = {
tileJSON,
publicUrl,
map,
dataProjWGStoInternalWGS: null,
lastModified: new Date().toUTCString(),
watermark: params.watermark || options.watermark
};
const queue = [];
for (const name of Object.keys(styleJSON.sources)) {
let source = styleJSON.sources[name];
const url = source.url;
if (url && url.lastIndexOf('mbtiles:', 0) === 0) {
// found mbtiles source, replace with info from local file
delete source.url;
let mbtilesFile = url.substring('mbtiles://'.length);
const fromData = mbtilesFile[0] === '{' &&
mbtilesFile[mbtilesFile.length - 1] === '}';
if (fromData) {
mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2);
const mapsTo = (params.mapping || {})[mbtilesFile];
if (mapsTo) {
mbtilesFile = mapsTo;
}
mbtilesFile = dataResolver(mbtilesFile);
if (!mbtilesFile) {
console.error(`ERROR: data "${mbtilesFile}" not found!`);
process.exit(1);
}
}
queue.push(new Promise((resolve, reject) => {
mbtilesFile = path.resolve(options.paths.mbtiles, mbtilesFile);
const mbtilesFileStats = fs.statSync(mbtilesFile);
if (!mbtilesFileStats.isFile() || mbtilesFileStats.size === 0) {
throw Error(`Not valid MBTiles file: ${mbtilesFile}`);
}
map.sources[name] = new MBTiles(mbtilesFile, err => {
map.sources[name].getInfo((err, info) => {
if (err) {
console.error(err);
return;
}
if (!repo[id].dataProjWGStoInternalWGS && info.proj4) {
// how to do this for multiple sources with different proj4 defs?
const to3857 = proj4('EPSG:3857');
const toDataProj = proj4(info.proj4);
repo[id].dataProjWGStoInternalWGS = xy => to3857.inverse(toDataProj.forward(xy));
}
const type = source.type;
Object.assign(source, info);
source.type = type;
source.tiles = [
// meta url which will be detected when requested
`mbtiles://${name}/{z}/{x}/{y}.${info.format || 'pbf'}`
];
delete source.scheme;
if (options.dataDecoratorFunc) {
source = options.dataDecoratorFunc(name, 'tilejson', source);
}
if (!attributionOverride &&
source.attribution && source.attribution.length > 0) {
if (tileJSON.attribution.length > 0) {
tileJSON.attribution += '; ';
}
tileJSON.attribution += source.attribution;
}
resolve();
});
});
}));
}
}
const renderersReadyPromise = Promise.all(queue).then(() => {
// standard and @2x tiles are much more usual -> default to larger pools
const minPoolSizes = options.minRendererPoolSizes || [8, 4, 2];
const maxPoolSizes = options.maxRendererPoolSizes || [16, 8, 4];
for (let s = 1; s <= maxScaleFactor; s++) {
const i = Math.min(minPoolSizes.length - 1, s - 1);
const j = Math.min(maxPoolSizes.length - 1, s - 1);
const minPoolSize = minPoolSizes[i];
const maxPoolSize = Math.max(minPoolSize, maxPoolSizes[j]);
map.renderers[s] = createPool(s, minPoolSize, maxPoolSize);
}
});
return Promise.all([renderersReadyPromise]);
},
remove: (repo, id) => {
let item = repo[id];
if (item) {
item.map.renderers.forEach(pool => {
pool.close();
});
}
delete repo[id];
},
};