From 3aaab828cb51c5cdd9b61caf5631d38c24d9a7fe Mon Sep 17 00:00:00 2001 From: acalcutt Date: Mon, 30 Dec 2024 11:12:20 -0500 Subject: [PATCH] move tile fetch and add fix verbose logging Co-Authored-By: Andrew Calcutt --- src/serve_data.js | 47 ++++++-------- src/serve_rendered.js | 140 ++++++++++++++++-------------------------- src/serve_style.js | 5 +- src/server.js | 8 +-- src/utils.js | 29 +++++++++ 5 files changed, 108 insertions(+), 121 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index 0a8bfae..81e2574 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -8,7 +8,12 @@ import express from 'express'; import Pbf from 'pbf'; import { VectorTile } from '@mapbox/vector-tile'; -import { fixTileJSONCenter, getTileUrls, isValidHttpUrl } from './utils.js'; +import { + fixTileJSONCenter, + getTileUrls, + isValidHttpUrl, + fetchTileData, +} from './utils.js'; import { getPMtilesInfo, getPMtilesTile, @@ -61,31 +66,17 @@ export const serve_data = { return res.status(404).send('Out of bounds'); } - let getTile; - if (item.sourceType === 'pmtiles') { - const tileinfo = await getPMtilesTile(item.source, z, x, y); - if (!tileinfo?.data) return res.status(204).send(); - getTile = { data: tileinfo.data, header: tileinfo.header }; - } else if (item.sourceType === 'mbtiles') { - try { - getTile = await new Promise((resolve, reject) => { - item.source.getTile(z, x, y, (err, tileData, tileHeader) => { - if (err) { - return /does not exist/.test(err.message) - ? resolve(null) - : reject(err); - } - resolve({ data: tileData, header: tileHeader }); - }); - }); - } catch (e) { - return res.status(500).send(e.message); - } - } - if (getTile == null) return res.status(204).send(); + const fetchTile = await fetchTileData( + item.source, + item.sourceType, + z, + x, + y, + ); + if (fetchTile == null) return res.status(204).send(); - let data = getTile.data; - let headers = getTile.header; + let data = fetchTile.data; + let headers = fetchTile.headers; let isGzipped = data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0; if (tileJSONFormat === 'pbf') { @@ -118,7 +109,6 @@ export const serve_data = { } data = JSON.stringify(geojson); } - console.log(headers); delete headers['ETag']; // do not trust the tile ETag -- regenerate headers['Content-Encoding'] = 'gzip'; res.set(headers); @@ -159,10 +149,11 @@ export const serve_data = { * @param {object} repo Repository object. * @param {object} params Parameters object. * @param {string} id ID of the data source. - * @param {string} publicUrl Public URL of the data. + * @param {object} programOpts - An object containing the program options * @returns {Promise} */ - add: async function (options, repo, params, id, publicUrl) { + add: async function (options, repo, params, id, programOpts) { + const { publicUrl } = programOpts; let inputFile; let inputType; if (params.pmtiles) { diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 3964947..944b841 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -33,12 +33,9 @@ import { getTileUrls, isValidHttpUrl, fixTileJSONCenter, + fetchTileData, } from './utils.js'; -import { - openPMtiles, - getPMtilesInfo, - getPMtilesTile, -} from './pmtiles_adapter.js'; +import { openPMtiles, getPMtilesInfo } from './pmtiles_adapter.js'; import { renderOverlay, renderWatermark, renderAttribution } from './render.js'; import fsp from 'node:fs/promises'; import { existsP, gunzipP } from './promises.js'; @@ -951,11 +948,11 @@ export const serve_rendered = { * @param {object} repo Repository object. * @param {object} params Parameters object. * @param {string} id ID of the item. - * @param {string} publicUrl Public URL. + * @param {object} programOpts - An object containing the program options * @param {Function} dataResolver Function to resolve data. * @returns {Promise} */ - add: async function (options, repo, params, id, publicUrl, dataResolver) { + add: async function (options, repo, params, id, programOpts, dataResolver) { const map = { renderers: [], renderersStatic: [], @@ -963,6 +960,8 @@ export const serve_rendered = { sourceTypes: {}, }; + const { publicUrl, verbose } = programOpts; + let styleJSON; /** * Creates a pool of renderers. @@ -1023,88 +1022,57 @@ export const serve_rendered = { const y = parts[5].split('.')[0] | 0; const format = parts[5].split('.')[1]; - if (sourceType === 'pmtiles') { - let tileinfo = await getPMtilesTile(source, z, x, y); - let data = tileinfo.data; - let headers = tileinfo.header; - if (data == undefined) { - if (options.verbose) - console.log('MBTiles error, serving empty', err); - createEmptyResponse( - sourceInfo.format, - sourceInfo.color, - callback, + const fetchTile = await fetchTileData( + source, + sourceType, + z, + x, + y, + ); + if (fetchTile == null) { + if (verbose) { + console.log( + 'fetchTile error on %s, serving empty response', + req.url, ); - return; - } else { - const response = {}; - response.data = data; - if (headers['Last-Modified']) { - response.modified = new Date(headers['Last-Modified']); - } - - if (format === 'pbf') { - if (options.dataDecoratorFunc) { - response.data = options.dataDecoratorFunc( - sourceId, - 'data', - response.data, - z, - x, - y, - ); - } - } - - callback(null, response); } - } else if (sourceType === 'mbtiles') { - source.getTile(z, x, y, async (err, data, headers) => { - if (err) { - if (options.verbose) - console.log('MBTiles error, serving empty', err); - createEmptyResponse( - sourceInfo.format, - sourceInfo.color, - callback, - ); - return; - } - - const response = {}; - if (headers['Last-Modified']) { - response.modified = new Date(headers['Last-Modified']); - } - - if (format === 'pbf') { - try { - response.data = await gunzipP(data); - } catch (err) { - console.log( - 'Skipping incorrect header for tile mbtiles://%s/%s/%s/%s.pbf', - id, - z, - x, - y, - ); - } - if (options.dataDecoratorFunc) { - response.data = options.dataDecoratorFunc( - sourceId, - 'data', - response.data, - z, - x, - y, - ); - } - } else { - response.data = data; - } - - callback(null, response); - }); + createEmptyResponse( + sourceInfo.format, + sourceInfo.color, + callback, + ); + return; } + + const response = {}; + response.data = fetchTile.data; + let headers = fetchTile.headers; + let isGzipped = + response.data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === + 0; + + if (headers['Last-Modified']) { + response.modified = new Date(headers['Last-Modified']); + } + + if (format === 'pbf') { + if (isGzipped) { + response.data = await gunzipP(response.data); + isGzipped = false; + } + if (options.dataDecoratorFunc) { + response.data = options.dataDecoratorFunc( + sourceId, + 'data', + response.data, + z, + x, + y, + ); + } + } + + callback(null, response); } else if (protocol === 'http' || protocol === 'https') { try { const response = await axios.get(req.url, { diff --git a/src/serve_style.js b/src/serve_style.js index c26eb94..5a74782 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -112,7 +112,7 @@ export const serve_style = { * @param {object} repo Repository object. * @param {object} params Parameters object containing style path * @param {string} id ID of the style. - * @param {string} publicUrl Public URL of the data. + * @param {object} programOpts - An object containing the program options * @param {Function} reportTiles Function for reporting tile sources. * @param {Function} reportFont Function for reporting font usage * @returns {boolean} true if add is succesful @@ -122,10 +122,11 @@ export const serve_style = { repo, params, id, - publicUrl, + programOpts, reportTiles, reportFont, ) { + const { publicUrl } = programOpts; const styleFile = path.resolve(options.paths.styles, params.style); let styleFileData; diff --git a/src/server.js b/src/server.js index 9b6ca07..a49fbac 100644 --- a/src/server.js +++ b/src/server.js @@ -193,7 +193,7 @@ async function start(opts) { serving.styles, item, id, - opts.publicUrl, + opts, (styleSourceId, protocol) => { let dataItemId; for (const id of Object.keys(data)) { @@ -250,7 +250,7 @@ async function start(opts) { serving.rendered, item, id, - opts.publicUrl, + opts, function dataResolver(styleSourceId) { let fileType; let inputFile; @@ -301,9 +301,7 @@ async function start(opts) { ); continue; } - startupPromises.push( - serve_data.add(options, serving.data, item, id, opts.publicUrl), - ); + startupPromises.push(serve_data.add(options, serving.data, item, id, opts)); } if (options.serveAllStyles) { fs.readdir(options.paths.styles, { withFileTypes: true }, (err, files) => { diff --git a/src/utils.js b/src/utils.js index cd3d17c..bf2d15f 100644 --- a/src/utils.js +++ b/src/utils.js @@ -6,6 +6,7 @@ import fs from 'node:fs'; import clone from 'clone'; import { combine } from '@jsse/pbfont'; import { existsP } from './promises.js'; +import { getPMtilesTile } from './pmtiles_adapter.js'; /** * Restrict user input to an allowed set of options. @@ -310,3 +311,31 @@ export function isValidHttpUrl(string) { return url.protocol === 'http:' || url.protocol === 'https:'; } + +/** + * Fetches tile data from either PMTiles or MBTiles source. + * @param {object} source - The source object, which may contain a mbtiles object, or pmtiles object. + * @param {string} sourceType - The source type, which should be `pmtiles` or `mbtiles` + * @param {number} z - The zoom level. + * @param {number} x - The x coordinate of the tile. + * @param {number} y - The y coordinate of the tile. + * @returns {Promise} - A promise that resolves to an object with data and headers or null if no data is found. + */ +export async function fetchTileData(source, sourceType, z, x, y) { + if (sourceType === 'pmtiles') { + return await new Promise(async (resolve) => { + const tileinfo = await getPMtilesTile(source, z, x, y); + if (!tileinfo?.data) return resolve(null); + resolve({ data: tileinfo.data, headers: tileinfo.header }); + }); + } else if (sourceType === 'mbtiles') { + return await new Promise((resolve) => { + source.getTile(z, x, y, (err, tileData, tileHeader) => { + if (err) { + return resolve(null); + } + resolve({ data: tileData, headers: tileHeader }); + }); + }); + } +}