From 345b831eddbb970ecae24c9bba3fabd302352403 Mon Sep 17 00:00:00 2001 From: Martin d'Allens Date: Mon, 20 Nov 2023 22:39:14 +0100 Subject: [PATCH 1/8] chore: convert some promises to async/await Signed-off-by: Martin d'Allens --- src/main.js | 31 ++++++++++--------- src/serve_font.js | 45 ++++++++++++++------------- src/serve_rendered.js | 72 ++++++++++++++++++++----------------------- src/server.js | 7 ++--- src/utils.js | 5 +-- 5 files changed, 79 insertions(+), 81 deletions(-) diff --git a/src/main.js b/src/main.js index 5bf4e0a..f3bb71b 100644 --- a/src/main.js +++ b/src/main.js @@ -241,7 +241,7 @@ const StartWithInputFile = async (inputFile) => { } }; -fs.stat(path.resolve(opts.config), (err, stats) => { +fs.stat(path.resolve(opts.config), async (err, stats) => { if (err || !stats.isFile() || stats.size === 0) { let inputFile; if (opts.file) { @@ -274,21 +274,22 @@ fs.stat(path.resolve(opts.config), (err, stats) => { const writer = fs.createWriteStream(filename); console.log(`No input file found`); console.log(`[DEMO] Downloading sample data (${filename}) from ${url}`); - axios({ - url, - method: 'GET', - responseType: 'stream', - }) - .then((response) => { - response.data.pipe(writer); - writer.on('finish', () => StartWithInputFile(filename)); - writer.on('error', (err) => - console.error(`Error writing file: ${err}`), - ); - }) - .catch((error) => { - console.error(`Error downloading file: ${error}`); + + try { + const response = await axios({ + url, + method: 'GET', + responseType: 'stream', }); + + response.data.pipe(writer); + writer.on('finish', () => StartWithInputFile(filename)); + writer.on('error', (err) => + console.error(`Error writing file: ${err}`), + ); + } catch (error) { + console.error(`Error downloading file: ${error}`); + } } } } else { diff --git a/src/serve_font.js b/src/serve_font.js index e21bf22..02f46dc 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -4,7 +4,7 @@ import express from 'express'; import { getFontsPbf, listFonts } from './utils.js'; -export const serve_font = (options, allowedFonts) => { +export const serve_font = async (options, allowedFonts) => { const app = express().disable('x-powered-by'); const lastModified = new Date().toUTCString(); @@ -13,25 +13,29 @@ export const serve_font = (options, allowedFonts) => { const existingFonts = {}; - app.get('/fonts/:fontstack/:range([\\d]+-[\\d]+).pbf', (req, res, next) => { - const fontstack = decodeURI(req.params.fontstack); - const range = req.params.range; + app.get( + '/fonts/:fontstack/:range([\\d]+-[\\d]+).pbf', + async (req, res, next) => { + const fontstack = decodeURI(req.params.fontstack); + const range = req.params.range; + + try { + const concatenated = await getFontsPbf( + options.serveAllFonts ? null : allowedFonts, + fontPath, + fontstack, + range, + existingFonts, + ); - getFontsPbf( - options.serveAllFonts ? null : allowedFonts, - fontPath, - fontstack, - range, - existingFonts, - ).then( - (concated) => { res.header('Content-type', 'application/x-protobuf'); res.header('Last-Modified', lastModified); - return res.send(concated); - }, - (err) => res.status(400).header('Content-Type', 'text/plain').send(err), - ); - }); + return res.send(concatenated); + } catch (err) { + res.status(400).header('Content-Type', 'text/plain').send(err); + } + }, + ); app.get('/fonts.json', (req, res, next) => { res.header('Content-type', 'application/json'); @@ -40,8 +44,7 @@ export const serve_font = (options, allowedFonts) => { ); }); - return listFonts(options.paths.fonts).then((fonts) => { - Object.assign(existingFonts, fonts); - return app; - }); + const fonts = await listFonts(options.paths.fonts); + Object.assign(existingFonts, fonts); + return app; }; diff --git a/src/serve_rendered.js b/src/serve_rendered.js index fbdd1f0..bb0006c 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -524,7 +524,7 @@ const existingFonts = {}; let maxScaleFactor = 2; export const serve_rendered = { - init: (options, repo) => { + init: async (options, repo) => { maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9); let scalePattern = ''; for (let i = 2; i <= maxScaleFactor; i++) { @@ -909,10 +909,9 @@ export const serve_rendered = { return res.send(info); }); - return listFonts(options.paths.fonts).then((fonts) => { - Object.assign(existingFonts, fonts); - return app; - }); + const fonts = await listFonts(options.paths.fonts); + Object.assign(existingFonts, fonts); + return app; }, add: async (options, repo, params, id, publicUrl, dataResolver) => { const map = { @@ -941,20 +940,19 @@ export const serve_rendered = { const parts = req.url.split('/'); const fontstack = unescape(parts[2]); const range = parts[3].split('.')[0]; - getFontsPbf( - null, - options.paths[protocol], - fontstack, - range, - existingFonts, - ).then( - (concated) => { - callback(null, { data: concated }); - }, - (err) => { - callback(err, { data: null }); - }, - ); + + try { + const concatenated = await getFontsPbf( + null, + options.paths[protocol], + fontstack, + range, + existingFonts, + ); + callback(null, { data: concatenated }); + } catch (err) { + callback(err, { data: null }); + } } else if (protocol === 'mbtiles' || protocol === 'pmtiles') { const parts = req.url.split('/'); const sourceId = parts[2]; @@ -1296,26 +1294,24 @@ export const serve_rendered = { } } - const renderersReadyPromise = Promise.all(queue).then(() => { - // standard and @2x tiles are much more usual -> default to larger pools - const minPoolSizes = options.minRendererPoolSizes || [8, 4, 2]; - const maxPoolSizes = options.maxRendererPoolSizes || [16, 8, 4]; - for (let s = 1; s <= maxScaleFactor; s++) { - const i = Math.min(minPoolSizes.length - 1, s - 1); - const j = Math.min(maxPoolSizes.length - 1, s - 1); - const minPoolSize = minPoolSizes[i]; - const maxPoolSize = Math.max(minPoolSize, maxPoolSizes[j]); - map.renderers[s] = createPool(s, 'tile', minPoolSize, maxPoolSize); - map.renderers_static[s] = createPool( - s, - 'static', - minPoolSize, - maxPoolSize, - ); - } - }); + await Promise.all(queue); - return renderersReadyPromise; + // standard and @2x tiles are much more usual -> default to larger pools + const minPoolSizes = options.minRendererPoolSizes || [8, 4, 2]; + const maxPoolSizes = options.maxRendererPoolSizes || [16, 8, 4]; + for (let s = 1; s <= maxScaleFactor; s++) { + const i = Math.min(minPoolSizes.length - 1, s - 1); + const j = Math.min(maxPoolSizes.length - 1, s - 1); + const minPoolSize = minPoolSizes[i]; + const maxPoolSize = Math.max(minPoolSize, maxPoolSizes[j]); + map.renderers[s] = createPool(s, 'tile', minPoolSize, maxPoolSize); + map.renderers_static[s] = createPool( + s, + 'static', + minPoolSize, + maxPoolSize, + ); + } }, remove: (repo, id) => { const item = repo[id]; diff --git a/src/server.js b/src/server.js index 1e570eb..a4a3760 100644 --- a/src/server.js +++ b/src/server.js @@ -141,11 +141,8 @@ function start(opts) { // Load all available icons into a settings object startupPromises.push( - new Promise((resolve) => { - getFiles(paths.icons).then((files) => { - paths.availableIcons = files; - resolve(); - }); + getFiles(paths.icons).then((files) => { + paths.availableIcons = files; }), ); diff --git a/src/utils.js b/src/utils.js index 3f5cb0d..8ed2d9c 100644 --- a/src/utils.js +++ b/src/utils.js @@ -139,7 +139,7 @@ const getFontPbf = (allowedFonts, fontPath, name, range, fallbacks) => } }); -export const getFontsPbf = ( +export const getFontsPbf = async ( allowedFonts, fontPath, names, @@ -160,7 +160,8 @@ export const getFontsPbf = ( ); } - return Promise.all(queue).then((values) => glyphCompose.combine(values)); + const values = await Promise.all(queue); + return glyphCompose.combine(values); }; export const listFonts = async (fontPath) => { From d9c2128ddf88de7e6f135091cd773a10945e11eb Mon Sep 17 00:00:00 2001 From: Martin d'Allens Date: Tue, 21 Nov 2023 11:38:50 +0100 Subject: [PATCH 2/8] chore: compact arguments for renderOverlay and respondImage Signed-off-by: Martin d'Allens --- src/serve_rendered.js | 105 ++++++++---------------------------------- 1 file changed, 18 insertions(+), 87 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index bb0006c..adfe255 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -573,19 +573,10 @@ export const serve_rendered = { ], z, ); + + // prettier-ignore return respondImage( - options, - item, - z, - tileCenter[0], - tileCenter[1], - 0, - 0, - tileSize, - tileSize, - scale, - format, - res, + options, item, z, tileCenter[0], tileCenter[1], 0, 0, tileSize, tileSize, scale, format, res, ); }, ); @@ -641,35 +632,15 @@ export const serve_rendered = { options, transformer, ); + + // prettier-ignore const overlay = await renderOverlay( - z, - x, - y, - bearing, - pitch, - w, - h, - scale, - paths, - markers, - req.query, + z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, ); + // prettier-ignore return respondImage( - options, - item, - z, - x, - y, - bearing, - pitch, - w, - h, - scale, - format, - res, - overlay, - 'static', + options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', ); } catch (e) { next(e); @@ -723,34 +694,15 @@ export const serve_rendered = { options, transformer, ); + + // prettier-ignore const overlay = await renderOverlay( - z, - x, - y, - bearing, - pitch, - w, - h, - scale, - paths, - markers, - req.query, + z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, ); + + // prettier-ignore return respondImage( - options, - item, - z, - x, - y, - bearing, - pitch, - w, - h, - scale, - format, - res, - overlay, - 'static', + options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', ); } catch (e) { next(e); @@ -856,35 +808,14 @@ export const serve_rendered = { const x = center[0]; const y = center[1]; + // prettier-ignore const overlay = await renderOverlay( - z, - x, - y, - bearing, - pitch, - w, - h, - scale, - paths, - markers, - req.query, + z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, ); + // prettier-ignore return respondImage( - options, - item, - z, - x, - y, - bearing, - pitch, - w, - h, - scale, - format, - res, - overlay, - 'static', + options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', ); } catch (e) { next(e); From f69a2339b8201a573e9fe6d5f984d542067287a2 Mon Sep 17 00:00:00 2001 From: Martin d'Allens Date: Tue, 21 Nov 2023 11:49:45 +0100 Subject: [PATCH 3/8] chore: remove duplicated field names to simplify code Signed-off-by: Martin d'Allens --- src/main.js | 8 ++++---- src/serve_rendered.js | 24 ++++++++++++------------ src/server.js | 10 +++++----- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/main.js b/src/main.js index f3bb71b..d32f11a 100644 --- a/src/main.js +++ b/src/main.js @@ -68,8 +68,8 @@ const StartServer = (configPath, config) => { publicUrl += '/'; } return server({ - configPath: configPath, - config: config, + configPath, + config, bind: opts.bind, port: opts.port, cors: opts.cors, @@ -77,7 +77,7 @@ const StartServer = (configPath, config) => { silent: opts.silent, logFile: opts.log_file, logFormat: opts.log_format, - publicUrl: publicUrl, + publicUrl, }); }; @@ -215,7 +215,7 @@ const StartWithInputFile = async (inputFile) => { config['styles'][styleName] = { style: styleFileRel, tilejson: { - bounds: bounds, + bounds, }, }; } diff --git a/src/serve_rendered.js b/src/serve_rendered.js index adfe255..4ff0e8b 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -97,7 +97,7 @@ function createEmptyResponse(format, color, callback) { raw: { width: 1, height: 1, - channels: channels, + channels, }, }) .toFormat(format) @@ -405,10 +405,10 @@ const respondImage = ( const params = { zoom: mlglZ, center: [lon, lat], - bearing: bearing, - pitch: pitch, - width: width, - height: height, + bearing, + pitch, + width, + height, }; if (z === 0) { @@ -856,8 +856,8 @@ export const serve_rendered = { const createPool = (ratio, mode, min, max) => { const createRenderer = (ratio, createCallback) => { const renderer = new mlgl.Map({ - mode: mode, - ratio: ratio, + mode, + ratio, request: async (req, callback) => { const protocol = req.url.split(':')[0]; // console.log('Handling request:', req); @@ -1016,8 +1016,8 @@ export const serve_rendered = { createCallback(null, renderer); }; return new advancedPool.Pool({ - min: min, - max: max, + min, + max, create: createRenderer.bind(null, ratio), destroy: (renderer) => { renderer.release(); @@ -1114,9 +1114,9 @@ export const serve_rendered = { let inputFile; const DataInfo = dataResolver(dataId); - if (DataInfo.inputfile) { - inputFile = DataInfo.inputfile; - source_type = DataInfo.filetype; + if (DataInfo.inputFile) { + inputFile = DataInfo.inputFile; + source_type = DataInfo.fileType; } else { console.error(`ERROR: data "${inputFile}" not found!`); process.exit(1); diff --git a/src/server.js b/src/server.js index a4a3760..6b10644 100644 --- a/src/server.js +++ b/src/server.js @@ -253,7 +253,7 @@ function start(opts) { inputFile = path.resolve(options.paths[fileType], inputFile); } - return { inputfile: inputFile, filetype: fileType }; + return { inputFile, fileType }; }, ), ); @@ -344,7 +344,7 @@ function start(opts) { result.push({ version: styleJSON.version, name: styleJSON.name, - id: id, + id, url: `${getPublicUrl( opts.publicUrl, req, @@ -630,9 +630,9 @@ function start(opts) { enableShutdown(server); return { - app: app, - server: server, - startupPromise: startupPromise, + app, + server, + startupPromise, }; } From 9a4ef3dedf680f9336fc3c65b4b97556fd4af25e Mon Sep 17 00:00:00 2001 From: Martin d'Allens Date: Tue, 21 Nov 2023 15:06:11 +0100 Subject: [PATCH 4/8] chore: remove unused opt_nokey arg Signed-off-by: Martin d'Allens --- src/serve_style.js | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index de4ccdb..d497e54 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -11,12 +11,12 @@ import { getPublicUrl } from './utils.js'; const httpTester = /^(http(s)?:)?\/\//; -const fixUrl = (req, url, publicUrl, opt_nokey) => { +const fixUrl = (req, url, publicUrl) => { if (!url || typeof url !== 'string' || url.indexOf('local://') !== 0) { return url; } const queryParams = []; - if (!opt_nokey && req.query.key) { + if (req.query.key) { queryParams.unshift(`key=${encodeURIComponent(req.query.key)}`); } let query = ''; @@ -42,20 +42,10 @@ export const serve_style = { } // mapbox-gl-js viewer cannot handle sprite urls with query if (styleJSON_.sprite) { - styleJSON_.sprite = fixUrl( - req, - styleJSON_.sprite, - item.publicUrl, - false, - ); + styleJSON_.sprite = fixUrl(req, styleJSON_.sprite, item.publicUrl); } if (styleJSON_.glyphs) { - styleJSON_.glyphs = fixUrl( - req, - styleJSON_.glyphs, - item.publicUrl, - false, - ); + styleJSON_.glyphs = fixUrl(req, styleJSON_.glyphs, item.publicUrl); } return res.send(styleJSON_); }); From 36d3e9d35621757df68164506a1afd40e1a53671 Mon Sep 17 00:00:00 2001 From: Martin d'Allens Date: Tue, 21 Nov 2023 22:19:34 +0100 Subject: [PATCH 5/8] chore: apply camelCase to function and variable names Signed-off-by: Martin d'Allens --- public/templates/index.tmpl | 2 +- src/main.js | 22 +++++++-------- src/pmtiles_adapter.js | 18 ++++++------ src/serve_data.js | 24 ++++++++-------- src/serve_rendered.js | 56 ++++++++++++++++++------------------- src/server.js | 24 ++++++++-------- 6 files changed, 73 insertions(+), 73 deletions(-) diff --git a/public/templates/index.tmpl b/public/templates/index.tmpl index 2de6634..250d056 100644 --- a/public/templates/index.tmpl +++ b/public/templates/index.tmpl @@ -79,7 +79,7 @@

{{tileJSON.name}}

identifier: {{@key}}{{#if formatted_filesize}} | size: {{formatted_filesize}}{{/if}}
-
type: {{#is_vector}}vector{{/is_vector}}{{^is_vector}}raster{{/is_vector}} data {{#if source_type}} | ext: {{source_type}}{{/if}}
+
type: {{#is_vector}}vector{{/is_vector}}{{^is_vector}}raster{{/is_vector}} data {{#if sourceType}} | ext: {{sourceType}}{{/if}}

services: TileJSON {{#if xyz_link}} diff --git a/src/main.js b/src/main.js index d32f11a..5077ba0 100644 --- a/src/main.js +++ b/src/main.js @@ -9,7 +9,7 @@ import axios from 'axios'; import { server } from './server.js'; import MBTiles from '@mapbox/mbtiles'; import { isValidHttpUrl } from './utils.js'; -import { PMtilesOpen, GetPMtilesInfo } from './pmtiles_adapter.js'; +import { openPMtiles, getPMtilesInfo } from './pmtiles_adapter.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -62,7 +62,7 @@ const opts = program.opts(); console.log(`Starting ${packageJson.name} v${packageJson.version}`); -const StartServer = (configPath, config) => { +const startServer = (configPath, config) => { let publicUrl = opts.public_url; if (publicUrl && publicUrl.lastIndexOf('/') !== publicUrl.length - 1) { publicUrl += '/'; @@ -81,7 +81,7 @@ const StartServer = (configPath, config) => { }); }; -const StartWithInputFile = async (inputFile) => { +const startWithInputFile = async (inputFile) => { console.log(`[INFO] Automatically creating config file for ${inputFile}`); console.log(`[INFO] Only a basic preview style will be used.`); console.log( @@ -123,8 +123,8 @@ const StartWithInputFile = async (inputFile) => { const extension = inputFile.split('.').pop().toLowerCase(); if (extension === 'pmtiles') { - let FileOpenInfo = PMtilesOpen(inputFile); - const metadata = await GetPMtilesInfo(FileOpenInfo); + const fileOpenInfo = openPMtiles(inputFile); + const metadata = await getPMtilesInfo(fileOpenInfo); if ( metadata.format === 'pbf' && @@ -174,7 +174,7 @@ const StartWithInputFile = async (inputFile) => { console.log('Run with --verbose to see the config file here.'); } - return StartServer(null, config); + return startServer(null, config); } else { if (isValidHttpUrl(inputFile)) { console.log( @@ -235,7 +235,7 @@ const StartWithInputFile = async (inputFile) => { console.log('Run with --verbose to see the config file here.'); } - return StartServer(null, config); + return startServer(null, config); }); }); } @@ -251,7 +251,7 @@ fs.stat(path.resolve(opts.config), async (err, stats) => { } if (inputFile) { - return StartWithInputFile(inputFile); + return startWithInputFile(inputFile); } else { // try to find in the cwd const files = fs.readdirSync(process.cwd()); @@ -266,7 +266,7 @@ fs.stat(path.resolve(opts.config), async (err, stats) => { } if (inputFile) { console.log(`No input file specified, using ${inputFile}`); - return StartWithInputFile(inputFile); + return startWithInputFile(inputFile); } else { const url = 'https://github.com/maptiler/tileserver-gl/releases/download/v1.3.0/zurich_switzerland.mbtiles'; @@ -283,7 +283,7 @@ fs.stat(path.resolve(opts.config), async (err, stats) => { }); response.data.pipe(writer); - writer.on('finish', () => StartWithInputFile(filename)); + writer.on('finish', () => startWithInputFile(filename)); writer.on('error', (err) => console.error(`Error writing file: ${err}`), ); @@ -294,6 +294,6 @@ fs.stat(path.resolve(opts.config), async (err, stats) => { } } else { console.log(`Using specified config file from ${opts.config}`); - return StartServer(opts.config, null); + return startServer(opts.config, null); } }); diff --git a/src/pmtiles_adapter.js b/src/pmtiles_adapter.js index a827584..cab3f54 100644 --- a/src/pmtiles_adapter.js +++ b/src/pmtiles_adapter.js @@ -11,7 +11,7 @@ class PMTilesFileSource { } async getBytes(offset, length) { const buffer = Buffer.alloc(length); - await ReadFileBytes(this.fd, buffer, offset); + await readFileBytes(this.fd, buffer, offset); const ab = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength, @@ -26,7 +26,7 @@ class PMTilesFileSource { * @param buffer * @param offset */ -async function ReadFileBytes(fd, buffer, offset) { +async function readFileBytes(fd, buffer, offset) { return new Promise((resolve, reject) => { fs.read(fd, buffer, 0, buffer.length, offset, (err) => { if (err) { @@ -41,7 +41,7 @@ async function ReadFileBytes(fd, buffer, offset) { * * @param FilePath */ -export function PMtilesOpen(FilePath) { +export function openPMtiles(FilePath) { let pmtiles = undefined; if (isValidHttpUrl(FilePath)) { @@ -59,12 +59,12 @@ export function PMtilesOpen(FilePath) { * * @param pmtiles */ -export async function GetPMtilesInfo(pmtiles) { +export async function getPMtilesInfo(pmtiles) { const header = await pmtiles.getHeader(); const metadata = await pmtiles.getMetadata(); //Add missing metadata from header - metadata['format'] = GetPmtilesTileType(header.tileType).type; + metadata['format'] = getPmtilesTileType(header.tileType).type; metadata['minzoom'] = header.minZoom; metadata['maxzoom'] = header.maxZoom; @@ -103,23 +103,23 @@ export async function GetPMtilesInfo(pmtiles) { * @param x * @param y */ -export async function GetPMtilesTile(pmtiles, z, x, y) { +export async function getPMtilesTile(pmtiles, z, x, y) { const header = await pmtiles.getHeader(); - const TileType = GetPmtilesTileType(header.tileType); + const tileType = getPmtilesTileType(header.tileType); let zxyTile = await pmtiles.getZxy(z, x, y); if (zxyTile && zxyTile.data) { zxyTile = Buffer.from(zxyTile.data); } else { zxyTile = undefined; } - return { data: zxyTile, header: TileType.header }; + return { data: zxyTile, header: tileType.header }; } /** * * @param typenum */ -function GetPmtilesTileType(typenum) { +function getPmtilesTileType(typenum) { let head = {}; let tileType; switch (typenum) { diff --git a/src/serve_data.js b/src/serve_data.js index 0221369..4a95b1f 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -12,9 +12,9 @@ import { VectorTile } from '@mapbox/vector-tile'; import { getTileUrls, isValidHttpUrl, fixTileJSONCenter } from './utils.js'; import { - PMtilesOpen, - GetPMtilesInfo, - GetPMtilesTile, + openPMtiles, + getPMtilesInfo, + getPMtilesTile, } from './pmtiles_adapter.js'; export const serve_data = { @@ -53,8 +53,8 @@ export const serve_data = { ) { return res.status(404).send('Out of bounds'); } - if (item.source_type === 'pmtiles') { - let tileinfo = await GetPMtilesTile(item.source, z, x, y); + if (item.sourceType === 'pmtiles') { + let tileinfo = await getPMtilesTile(item.source, z, x, y); if (tileinfo == undefined || tileinfo.data == undefined) { return res.status(404).send('Not found'); } else { @@ -99,7 +99,7 @@ export const serve_data = { return res.status(200).send(data); } - } else if (item.source_type === 'mbtiles') { + } else if (item.sourceType === 'mbtiles') { item.source.getTile(z, x, y, (err, data, headers) => { let isGzipped; if (err) { @@ -223,11 +223,11 @@ export const serve_data = { } let source; - let source_type; + let sourceType; if (inputType === 'pmtiles') { - source = PMtilesOpen(inputFile); - source_type = 'pmtiles'; - const metadata = await GetPMtilesInfo(source); + source = openPMtiles(inputFile); + sourceType = 'pmtiles'; + const metadata = await getPMtilesInfo(source); tileJSON['name'] = id; tileJSON['format'] = 'pbf'; @@ -245,7 +245,7 @@ export const serve_data = { tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON); } } else if (inputType === 'mbtiles') { - source_type = 'mbtiles'; + sourceType = 'mbtiles'; const sourceInfoPromise = new Promise((resolve, reject) => { source = new MBTiles(inputFile + '?mode=ro', (err) => { if (err) { @@ -285,7 +285,7 @@ export const serve_data = { tileJSON, publicUrl, source, - source_type, + sourceType, }; }, }; diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 4ff0e8b..2a82804 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -25,9 +25,9 @@ import { fixTileJSONCenter, } from './utils.js'; import { - PMtilesOpen, - GetPMtilesInfo, - GetPMtilesTile, + openPMtiles, + getPMtilesInfo, + getPMtilesTile, } from './pmtiles_adapter.js'; import { renderOverlay, renderWatermark, renderAttribution } from './render.js'; @@ -398,7 +398,7 @@ const respondImage = ( if (mode === 'tile' && tileMargin === 0) { pool = item.map.renderers[scale]; } else { - pool = item.map.renderers_static[scale]; + pool = item.map.renderersStatic[scale]; } pool.acquire((err, renderer) => { const mlglZ = Math.max(0, z - 1); @@ -471,14 +471,14 @@ const respondImage = ( image.resize(width * scale, height * scale); } - const composite_array = []; + const composites = []; if (overlay) { - composite_array.push({ input: overlay }); + composites.push({ input: overlay }); } if (item.watermark) { const canvas = renderWatermark(width, height, scale, item.watermark); - composite_array.push({ input: canvas.toBuffer() }); + composites.push({ input: canvas.toBuffer() }); } if (mode === 'static' && item.staticAttributionText) { @@ -489,11 +489,11 @@ const respondImage = ( item.staticAttributionText, ); - composite_array.push({ input: canvas.toBuffer() }); + composites.push({ input: canvas.toBuffer() }); } - if (composite_array.length > 0) { - image.composite(composite_array); + if (composites.length > 0) { + image.composite(composites); } const formatQuality = (options.formatQuality || {})[format]; @@ -847,9 +847,9 @@ export const serve_rendered = { add: async (options, repo, params, id, publicUrl, dataResolver) => { const map = { renderers: [], - renderers_static: [], + renderersStatic: [], sources: {}, - source_types: {}, + sourceTypes: {}, }; let styleJSON; @@ -888,7 +888,7 @@ export const serve_rendered = { const parts = req.url.split('/'); const sourceId = parts[2]; const source = map.sources[sourceId]; - const source_type = map.source_types[sourceId]; + const sourceType = map.sourceTypes[sourceId]; const sourceInfo = styleJSON.sources[sourceId]; const z = parts[3] | 0; @@ -896,8 +896,8 @@ export const serve_rendered = { const y = parts[5].split('.')[0] | 0; const format = parts[5].split('.')[1]; - if (source_type === 'pmtiles') { - let tileinfo = await GetPMtilesTile(source, z, x, y); + if (sourceType === 'pmtiles') { + let tileinfo = await getPMtilesTile(source, z, x, y); let data = tileinfo.data; let headers = tileinfo.header; if (data == undefined) { @@ -931,7 +931,7 @@ export const serve_rendered = { callback(null, response); } - } else if (source_type === 'mbtiles') { + } else if (sourceType === 'mbtiles') { source.getTile(z, x, y, (err, data, headers) => { if (err) { if (options.verbose) @@ -1092,7 +1092,7 @@ export const serve_rendered = { const queue = []; for (const name of Object.keys(styleJSON.sources)) { - let source_type; + let sourceType; let source = styleJSON.sources[name]; let url = source.url; if ( @@ -1113,10 +1113,10 @@ export const serve_rendered = { } let inputFile; - const DataInfo = dataResolver(dataId); - if (DataInfo.inputFile) { - inputFile = DataInfo.inputFile; - source_type = DataInfo.fileType; + const dataInfo = dataResolver(dataId); + if (dataInfo.inputFile) { + inputFile = dataInfo.inputFile; + sourceType = dataInfo.fileType; } else { console.error(`ERROR: data "${inputFile}" not found!`); process.exit(1); @@ -1129,10 +1129,10 @@ export const serve_rendered = { } } - if (source_type === 'pmtiles') { - map.sources[name] = PMtilesOpen(inputFile); - map.source_types[name] = 'pmtiles'; - const metadata = await GetPMtilesInfo(map.sources[name]); + if (sourceType === 'pmtiles') { + map.sources[name] = openPMtiles(inputFile); + map.sourceTypes[name] = 'pmtiles'; + const metadata = await getPMtilesInfo(map.sources[name]); if (!repoobj.dataProjWGStoInternalWGS && metadata.proj4) { // how to do this for multiple sources with different proj4 defs? @@ -1177,7 +1177,7 @@ export const serve_rendered = { console.error(err); return; } - map.source_types[name] = 'mbtiles'; + map.sourceTypes[name] = 'mbtiles'; if (!repoobj.dataProjWGStoInternalWGS && info.proj4) { // how to do this for multiple sources with different proj4 defs? @@ -1236,7 +1236,7 @@ export const serve_rendered = { const minPoolSize = minPoolSizes[i]; const maxPoolSize = Math.max(minPoolSize, maxPoolSizes[j]); map.renderers[s] = createPool(s, 'tile', minPoolSize, maxPoolSize); - map.renderers_static[s] = createPool( + map.renderersStatic[s] = createPool( s, 'static', minPoolSize, @@ -1250,7 +1250,7 @@ export const serve_rendered = { item.map.renderers.forEach((pool) => { pool.close(); }); - item.map.renderers_static.forEach((pool) => { + item.map.renderersStatic.forEach((pool) => { pool.close(); }); } diff --git a/src/server.js b/src/server.js index 6b10644..2b382a0 100644 --- a/src/server.js +++ b/src/server.js @@ -179,15 +179,15 @@ function start(opts) { item, id, opts.publicUrl, - (StyleSourceId, protocol) => { + (styleSourceId, protocol) => { let dataItemId; for (const id of Object.keys(data)) { - if (id === StyleSourceId) { + if (id === styleSourceId) { // Style id was found in data ids, return that id dataItemId = id; } else { const fileType = Object.keys(data[id])[0]; - if (data[id][fileType] === StyleSourceId) { + if (data[id][fileType] === styleSourceId) { // Style id was found in data filename, return the id that filename belong to dataItemId = id; } @@ -199,21 +199,21 @@ function start(opts) { } else { if (!allowMoreData) { console.log( - `ERROR: style "${item.style}" using unknown file "${StyleSourceId}"! Skipping...`, + `ERROR: style "${item.style}" using unknown file "${styleSourceId}"! Skipping...`, ); return undefined; } else { let id = - StyleSourceId.substr(0, StyleSourceId.lastIndexOf('.')) || - StyleSourceId; - if (isValidHttpUrl(StyleSourceId)) { + styleSourceId.substr(0, styleSourceId.lastIndexOf('.')) || + styleSourceId; + if (isValidHttpUrl(styleSourceId)) { id = - fnv1a(StyleSourceId) + '_' + id.replace(/^.*\/(.*)$/, '$1'); + fnv1a(styleSourceId) + '_' + id.replace(/^.*\/(.*)$/, '$1'); } while (data[id]) id += '_'; //if the data source id already exists, add a "_" untill it doesn't //Add the new data source to the data array. data[id] = { - [protocol]: StyleSourceId, + [protocol]: styleSourceId, }; return id; @@ -236,15 +236,15 @@ function start(opts) { item, id, opts.publicUrl, - (StyleSourceId) => { + function dataResolver(styleSourceId) { let fileType; let inputFile; for (const id of Object.keys(data)) { fileType = Object.keys(data[id])[0]; - if (StyleSourceId == id) { + if (styleSourceId == id) { inputFile = data[id][fileType]; break; - } else if (data[id][fileType] == StyleSourceId) { + } else if (data[id][fileType] == styleSourceId) { inputFile = data[id][fileType]; break; } From b25a6420dda4000d6512929930cf9d2608cd148a Mon Sep 17 00:00:00 2001 From: Martin d'Allens Date: Thu, 23 Nov 2023 18:16:31 +0100 Subject: [PATCH 6/8] chore: simplify alpha premultiplication, now supported by sharp (#1073) Maplibre-native outputs premultiplied pixels values. The sharp library did not support it so we added code to cancel the alpha premultiplication. Note that this can only visible onr raster tiles (and probably static maps). The sharp library now supports premultiplied pixels with the right config. Let's use it: it should be faster and easie to maintain. Feature announced here: https://github.com/lovell/sharp/issues/1599#issuecomment-837004081 Feature developped here by @mnutt: https://github.com/lovell/sharp/pull/2685 Signed-off-by: Martin d'Allens --- src/serve_rendered.js | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index fbdd1f0..0beb383 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -428,24 +428,9 @@ const respondImage = ( return res.status(500).header('Content-Type', 'text/plain').send(err); } - // Fix semi-transparent outlines on raw, premultiplied input - // https://github.com/maptiler/tileserver-gl/issues/350#issuecomment-477857040 - for (let i = 0; i < data.length; i += 4) { - const alpha = data[i + 3]; - const norm = alpha / 255; - if (alpha === 0) { - data[i] = 0; - data[i + 1] = 0; - data[i + 2] = 0; - } else { - data[i] = data[i] / norm; - data[i + 1] = data[i + 1] / norm; - data[i + 2] = data[i + 2] / norm; - } - } - const image = sharp(data, { raw: { + premultiplied: true, width: params.width * scale, height: params.height * scale, channels: 4, From c9aa26a6de4dfe9fdafa62f1f989b8fdcd66096c Mon Sep 17 00:00:00 2001 From: Martin d'Allens Date: Fri, 24 Nov 2023 17:48:50 +0100 Subject: [PATCH 7/8] fix: listFonts was broken, missing fonts could not fallback (#1076) Promise code never worked: listFonts did not wait for fs.stat() to resolve(). This was not evident because late results were still used. A recent PR made it worse: late results are now ignored. This manifested for styles with missing fonts, no fallback could be used. Signed-off-by: Martin d'Allens --- src/utils.js | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/src/utils.js b/src/utils.js index 3f5cb0d..97ed7ef 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,7 +1,8 @@ 'use strict'; import path from 'path'; -import fs from 'node:fs'; +import fsPromises from 'fs/promises'; +import fs, { existsSync } from 'node:fs'; import clone from 'clone'; import glyphCompose from '@mapbox/glyph-pbf-composite'; @@ -165,30 +166,18 @@ export const getFontsPbf = ( export const listFonts = async (fontPath) => { const existingFonts = {}; - const fontListingPromise = new Promise((resolve, reject) => { - fs.readdir(fontPath, (err, files) => { - if (err) { - reject(err); - return; - } - for (const file of files) { - fs.stat(path.join(fontPath, file), (err, stats) => { - if (err) { - reject(err); - return; - } - if ( - stats.isDirectory() && - fs.existsSync(path.join(fontPath, file, '0-255.pbf')) - ) { - existingFonts[path.basename(file)] = true; - } - }); - } - resolve(); - }); - }); - await fontListingPromise; + + const files = await fsPromises.readdir(fontPath); + for (const file of files) { + const stats = await fsPromises.stat(path.join(fontPath, file)); + if ( + stats.isDirectory() && + existsSync(path.join(fontPath, file, '0-255.pbf')) + ) { + existingFonts[path.basename(file)] = true; + } + } + return existingFonts; }; From 407f6b2e30a1d2b0be265b2c751bf75642ea9f5e Mon Sep 17 00:00:00 2001 From: Martin d'Allens Date: Sun, 26 Nov 2023 02:16:33 +0100 Subject: [PATCH 8/8] Fix crash on ARM during rendering (#1077) * chore: fix crash on ARM during rendering Signed-off-by: Martin d'Allens * chore: remove useless comment Signed-off-by: Martin d'Allens --------- Signed-off-by: Martin d'Allens --- src/serve_rendered.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 0beb383..da5001e 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -1,12 +1,24 @@ 'use strict'; +// SECTION START +// +// The order of the two imports below is important. +// For an unknown reason, if the order is reversed, rendering can crash. +// This happens on ARM: +// > terminate called after throwing an instance of 'std::runtime_error' +// > what(): Cannot read GLX extensions. +import 'canvas'; +import '@maplibre/maplibre-gl-native'; +// +// SECTION END + import advancedPool from 'advanced-pool'; import fs from 'node:fs'; import path from 'path'; import url from 'url'; import util from 'util'; import zlib from 'zlib'; -import sharp from 'sharp'; // sharp has to be required before node-canvas on linux but after it on windows. see https://github.com/lovell/sharp/issues/371 +import sharp from 'sharp'; import clone from 'clone'; import Color from 'color'; import express from 'express';