Fabio/tileserver-gl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

move tile fetch and add fix verbose logging

Browse source 
Co-Authored-By: Andrew Calcutt <acalcutt@techidiots.net>
This commit is contained in:
acalcutt 2024-12-30 11:12:20 -05:00
parent d6f7f5e926
commit 99afa33e9d
5 changed files with 108 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
src/serve_data.js
View file

@ -11,7 +11,12 @@ import SphericalMercator from '@mapbox/sphericalmercator';
import { Image, createCanvas } from 'canvas';
import sharp from 'sharp';
import { fixTileJSONCenter, getTileUrls, isValidHttpUrl } from './utils.js';
import {
fixTileJSONCenter,
getTileUrls,
isValidHttpUrl,
fetchTileData,
} from './utils.js';
import {
getPMtilesInfo,
getPMtilesTile,
@ -64,31 +69,17 @@ export const serve_data = {
return res.status(404).send('Out of bounds');
}
let getTile;
if (item.sourceType === 'pmtiles') {
const tileinfo = await getPMtilesTile(item.source, z, x, y);
if (!tileinfo?.data) return res.status(204).send();
getTile = { data: tileinfo.data, header: tileinfo.header };
} else if (item.sourceType === 'mbtiles') {
try {
getTile = await new Promise((resolve, reject) => {
item.source.getTile(z, x, y, (err, tileData, tileHeader) => {
if (err) {
return /does not exist/.test(err.message)
? resolve(null)
: reject(err);
}
resolve({ data: tileData, header: tileHeader });
});
});
} catch (e) {
return res.status(500).send(e.message);
}
}
if (getTile == null) return res.status(204).send();
const fetchTile = await fetchTileData(
item.source,
item.sourceType,
z,
x,
y,
);
if (fetchTile == null) return res.status(204).send();
let data = getTile.data;
let headers = getTile.header;
let data = fetchTile.data;
let headers = fetchTile.headers;
let isGzipped = data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0;
if (tileJSONFormat === 'pbf') {
@ -121,7 +112,6 @@ export const serve_data = {
}
data = JSON.stringify(geojson);
}
console.log(headers);
delete headers['ETag']; // do not trust the tile ETag -- regenerate
headers['Content-Encoding'] = 'gzip';
res.set(headers);
@ -162,10 +152,11 @@ export const serve_data = {
* @param {object} repo Repository object.
* @param {object} params Parameters object.
* @param {string} id ID of the data source.
* @param {string} publicUrl Public URL of the data.
* @param {object} programOpts - An object containing the program options
* @returns {Promise<void>}
*/
add: async function (options, repo, params, id, publicUrl) {
add: async function (options, repo, params, id, programOpts) {
const { publicUrl } = programOpts;
let inputFile;
let inputType;
if (params.pmtiles) {

140
src/serve_rendered.js
View file

@ -33,12 +33,9 @@ import {
getTileUrls,
isValidHttpUrl,
fixTileJSONCenter,
fetchTileData,
} from './utils.js';
import {
openPMtiles,
getPMtilesInfo,
getPMtilesTile,
} from './pmtiles_adapter.js';
import { openPMtiles, getPMtilesInfo } from './pmtiles_adapter.js';
import { renderOverlay, renderWatermark, renderAttribution } from './render.js';
import fsp from 'node:fs/promises';
import { existsP, gunzipP } from './promises.js';
@ -951,11 +948,11 @@ export const serve_rendered = {
* @param {object} repo Repository object.
* @param {object} params Parameters object.
* @param {string} id ID of the item.
* @param {string} publicUrl Public URL.
* @param {object} programOpts - An object containing the program options
* @param {Function} dataResolver Function to resolve data.
* @returns {Promise<void>}
*/
add: async function (options, repo, params, id, publicUrl, dataResolver) {
add: async function (options, repo, params, id, programOpts, dataResolver) {
const map = {
renderers: [],
renderersStatic: [],
@ -963,6 +960,8 @@ export const serve_rendered = {
sourceTypes: {},
};
const { publicUrl, verbose } = programOpts;
let styleJSON;
/**
* Creates a pool of renderers.
@ -1023,88 +1022,57 @@ export const serve_rendered = {
const y = parts[5].split('.')[0] | 0;
const format = parts[5].split('.')[1];
if (sourceType === 'pmtiles') {
let tileinfo = await getPMtilesTile(source, z, x, y);
let data = tileinfo.data;
let headers = tileinfo.header;
if (data == undefined) {
if (options.verbose)
console.log('MBTiles error, serving empty', err);
createEmptyResponse(
sourceInfo.format,
sourceInfo.color,
callback,
const fetchTile = await fetchTileData(
source,
sourceType,
z,
x,
y,
);
if (fetchTile == null) {
if (verbose) {
console.log(
'fetchTile error on %s, serving empty response',
req.url,
);
return;
} else {
const response = {};
response.data = data;
if (headers['Last-Modified']) {
response.modified = new Date(headers['Last-Modified']);
}
if (format === 'pbf') {
if (options.dataDecoratorFunc) {
response.data = options.dataDecoratorFunc(
sourceId,
'data',
response.data,
z,
x,
y,
);
}
}
callback(null, response);
}
} else if (sourceType === 'mbtiles') {
source.getTile(z, x, y, async (err, data, headers) => {
if (err) {
if (options.verbose)
console.log('MBTiles error, serving empty', err);
createEmptyResponse(
sourceInfo.format,
sourceInfo.color,
callback,
);
return;
}
const response = {};
if (headers['Last-Modified']) {
response.modified = new Date(headers['Last-Modified']);
}
if (format === 'pbf') {
try {
response.data = await gunzipP(data);
} catch (err) {
console.log(
'Skipping incorrect header for tile mbtiles://%s/%s/%s/%s.pbf',
id,
z,
x,
y,
);
}
if (options.dataDecoratorFunc) {
response.data = options.dataDecoratorFunc(
sourceId,
'data',
response.data,
z,
x,
y,
);
}
} else {
response.data = data;
}
callback(null, response);
});
createEmptyResponse(
sourceInfo.format,
sourceInfo.color,
callback,
);
return;
}
const response = {};
response.data = fetchTile.data;
let headers = fetchTile.headers;
let isGzipped =
response.data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) ===
0;
if (headers['Last-Modified']) {
response.modified = new Date(headers['Last-Modified']);
}
if (format === 'pbf') {
if (isGzipped) {
response.data = await gunzipP(response.data);
isGzipped = false;
}
if (options.dataDecoratorFunc) {
response.data = options.dataDecoratorFunc(
sourceId,
'data',
response.data,
z,
x,
y,
);
}
}
callback(null, response);
} else if (protocol === 'http' || protocol === 'https') {
try {
const response = await axios.get(req.url, {

5
src/serve_style.js
View file

@ -112,7 +112,7 @@ export const serve_style = {
* @param {object} repo Repository object.
* @param {object} params Parameters object containing style path
* @param {string} id ID of the style.
* @param {string} publicUrl Public URL of the data.
* @param {object} programOpts - An object containing the program options
* @param {Function} reportTiles Function for reporting tile sources.
* @param {Function} reportFont Function for reporting font usage
* @returns {boolean} true if add is succesful
@ -122,10 +122,11 @@ export const serve_style = {
repo,
params,
id,
publicUrl,
programOpts,
reportTiles,
reportFont,
) {
const { publicUrl } = programOpts;
const styleFile = path.resolve(options.paths.styles, params.style);
let styleFileData;

8
src/server.js
View file

@ -193,7 +193,7 @@ async function start(opts) {
serving.styles,
item,
id,
opts.publicUrl,
opts,
(styleSourceId, protocol) => {
let dataItemId;
for (const id of Object.keys(data)) {
@ -250,7 +250,7 @@ async function start(opts) {
serving.rendered,
item,
id,
opts.publicUrl,
opts,
function dataResolver(styleSourceId) {
let fileType;
let inputFile;
@ -301,9 +301,7 @@ async function start(opts) {
);
continue;
}
startupPromises.push(
serve_data.add(options, serving.data, item, id, opts.publicUrl),
);
startupPromises.push(serve_data.add(options, serving.data, item, id, opts));
}
if (options.serveAllStyles) {
fs.readdir(options.paths.styles, { withFileTypes: true }, (err, files) => {

29
src/utils.js
View file

@ -6,6 +6,7 @@ import fs from 'node:fs';
import clone from 'clone';
import { combine } from '@jsse/pbfont';
import { existsP } from './promises.js';
import { getPMtilesTile } from './pmtiles_adapter.js';
/**
* Restrict user input to an allowed set of options.
@ -322,3 +323,31 @@ export function isValidHttpUrl(string) {
return url.protocol === 'http:' || url.protocol === 'https:';
}
/**
* Fetches tile data from either PMTiles or MBTiles source.
* @param {object} source - The source object, which may contain a mbtiles object, or pmtiles object.
* @param {string} sourceType - The source type, which should be `pmtiles` or `mbtiles`
* @param {number} z - The zoom level.
* @param {number} x - The x coordinate of the tile.
* @param {number} y - The y coordinate of the tile.
* @returns {Promise<object | null>} - A promise that resolves to an object with data and headers or null if no data is found.
*/
export async function fetchTileData(source, sourceType, z, x, y) {
if (sourceType === 'pmtiles') {
return await new Promise(async (resolve) => {
const tileinfo = await getPMtilesTile(source, z, x, y);
if (!tileinfo?.data) return resolve(null);
resolve({ data: tileinfo.data, headers: tileinfo.header });
});
} else if (sourceType === 'mbtiles') {
return await new Promise((resolve) => {
source.getTile(z, x, y, (err, tileData, tileHeader) => {
if (err) {
return resolve(null);
}
resolve({ data: tileData, headers: tileHeader });
});
});
}
}