move tile fetch and add fix verbose logging

Co-Authored-By: Andrew Calcutt <acalcutt@techidiots.net>

src/serve_data.js

@@ -8,7 +8,12 @@ import express from 'express';
 import Pbf from 'pbf';
 import { VectorTile } from '@mapbox/vector-tile';
 
-import { fixTileJSONCenter, getTileUrls, isValidHttpUrl } from './utils.js';
+import {
+  fixTileJSONCenter,
+  getTileUrls,
+  isValidHttpUrl,
+  fetchTileData,
+} from './utils.js';
 import {
   getPMtilesInfo,
   getPMtilesTile,
@@ -61,31 +66,17 @@ export const serve_data = {
         return res.status(404).send('Out of bounds');
       }
 
-      let getTile;
+      const fetchTile = await fetchTileData(
-      if (item.sourceType === 'pmtiles') {
+        item.source,
-        const tileinfo = await getPMtilesTile(item.source, z, x, y);
+        item.sourceType,
-        if (!tileinfo?.data) return res.status(204).send();
+        z,
-        getTile = { data: tileinfo.data, header: tileinfo.header };
+        x,
-      } else if (item.sourceType === 'mbtiles') {
+        y,
-        try {
+      );
-          getTile = await new Promise((resolve, reject) => {
+      if (fetchTile == null) return res.status(204).send();
-            item.source.getTile(z, x, y, (err, tileData, tileHeader) => {
-              if (err) {
-                return /does not exist/.test(err.message)
-                  ? resolve(null)
-                  : reject(err);
-              }
-              resolve({ data: tileData, header: tileHeader });
-            });
-          });
-        } catch (e) {
-          return res.status(500).send(e.message);
-        }
-      }
-      if (getTile == null) return res.status(204).send();
 
-      let data = getTile.data;
+      let data = fetchTile.data;
-      let headers = getTile.header;
+      let headers = fetchTile.headers;
       let isGzipped = data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0;
 
       if (tileJSONFormat === 'pbf') {
@@ -118,7 +109,6 @@ export const serve_data = {
         }
         data = JSON.stringify(geojson);
       }
-      console.log(headers);
       delete headers['ETag']; // do not trust the tile ETag -- regenerate
       headers['Content-Encoding'] = 'gzip';
       res.set(headers);
@@ -159,10 +149,11 @@ export const serve_data = {
    * @param {object} repo Repository object.
    * @param {object} params Parameters object.
    * @param {string} id ID of the data source.
-   * @param {string} publicUrl Public URL of the data.
+   * @param {object} programOpts - An object containing the program options
    * @returns {Promise<void>}
    */
-  add: async function (options, repo, params, id, publicUrl) {
+  add: async function (options, repo, params, id, programOpts) {
+    const { publicUrl } = programOpts;
     let inputFile;
     let inputType;
     if (params.pmtiles) {

src/serve_rendered.js

@@ -33,12 +33,9 @@ import {
   getTileUrls,
   isValidHttpUrl,
   fixTileJSONCenter,
+  fetchTileData,
 } from './utils.js';
-import {
+import { openPMtiles, getPMtilesInfo } from './pmtiles_adapter.js';
-  openPMtiles,
-  getPMtilesInfo,
-  getPMtilesTile,
-} from './pmtiles_adapter.js';
 import { renderOverlay, renderWatermark, renderAttribution } from './render.js';
 import fsp from 'node:fs/promises';
 import { existsP, gunzipP } from './promises.js';
@@ -951,11 +948,11 @@ export const serve_rendered = {
    * @param {object} repo Repository object.
    * @param {object} params Parameters object.
    * @param {string} id ID of the item.
-   * @param {string} publicUrl Public URL.
+   * @param {object} programOpts - An object containing the program options
    * @param {Function} dataResolver Function to resolve data.
    * @returns {Promise<void>}
    */
-  add: async function (options, repo, params, id, publicUrl, dataResolver) {
+  add: async function (options, repo, params, id, programOpts, dataResolver) {
     const map = {
       renderers: [],
       renderersStatic: [],
@@ -963,6 +960,8 @@ export const serve_rendered = {
       sourceTypes: {},
     };
 
+    const { publicUrl, verbose } = programOpts;
+
     let styleJSON;
     /**
      * Creates a pool of renderers.
@@ -1023,88 +1022,57 @@ export const serve_rendered = {
               const y = parts[5].split('.')[0] | 0;
               const format = parts[5].split('.')[1];
 
-              if (sourceType === 'pmtiles') {
+              const fetchTile = await fetchTileData(
-                let tileinfo = await getPMtilesTile(source, z, x, y);
+                source,
-                let data = tileinfo.data;
+                sourceType,
-                let headers = tileinfo.header;
+                z,
-                if (data == undefined) {
+                x,
-                  if (options.verbose)
+                y,
-                    console.log('MBTiles error, serving empty', err);
+              );
-                  createEmptyResponse(
+              if (fetchTile == null) {
-                    sourceInfo.format,
+                if (verbose) {
-                    sourceInfo.color,
+                  console.log(
-                    callback,
+                    'fetchTile error on %s, serving empty response',
+                    req.url,
                   );
-                  return;
-                } else {
-                  const response = {};
-                  response.data = data;
-                  if (headers['Last-Modified']) {
-                    response.modified = new Date(headers['Last-Modified']);
-                  }
-
-                  if (format === 'pbf') {
-                    if (options.dataDecoratorFunc) {
-                      response.data = options.dataDecoratorFunc(
-                        sourceId,
-                        'data',
-                        response.data,
-                        z,
-                        x,
-                        y,
-                      );
-                    }
-                  }
-
-                  callback(null, response);
                 }
-              } else if (sourceType === 'mbtiles') {
+                createEmptyResponse(
-                source.getTile(z, x, y, async (err, data, headers) => {
+                  sourceInfo.format,
-                  if (err) {
+                  sourceInfo.color,
-                    if (options.verbose)
+                  callback,
-                      console.log('MBTiles error, serving empty', err);
+                );
-                    createEmptyResponse(
+                return;
-                      sourceInfo.format,
-                      sourceInfo.color,
-                      callback,
-                    );
-                    return;
-                  }
-
-                  const response = {};
-                  if (headers['Last-Modified']) {
-                    response.modified = new Date(headers['Last-Modified']);
-                  }
-
-                  if (format === 'pbf') {
-                    try {
-                      response.data = await gunzipP(data);
-                    } catch (err) {
-                      console.log(
-                        'Skipping incorrect header for tile mbtiles://%s/%s/%s/%s.pbf',
-                        id,
-                        z,
-                        x,
-                        y,
-                      );
-                    }
-                    if (options.dataDecoratorFunc) {
-                      response.data = options.dataDecoratorFunc(
-                        sourceId,
-                        'data',
-                        response.data,
-                        z,
-                        x,
-                        y,
-                      );
-                    }
-                  } else {
-                    response.data = data;
-                  }
-
-                  callback(null, response);
-                });
               }
+
+              const response = {};
+              response.data = fetchTile.data;
+              let headers = fetchTile.headers;
+              let isGzipped =
+                response.data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) ===
+                0;
+
+              if (headers['Last-Modified']) {
+                response.modified = new Date(headers['Last-Modified']);
+              }
+
+              if (format === 'pbf') {
+                if (isGzipped) {
+                  response.data = await gunzipP(response.data);
+                  isGzipped = false;
+                }
+                if (options.dataDecoratorFunc) {
+                  response.data = options.dataDecoratorFunc(
+                    sourceId,
+                    'data',
+                    response.data,
+                    z,
+                    x,
+                    y,
+                  );
+                }
+              }
+
+              callback(null, response);
             } else if (protocol === 'http' || protocol === 'https') {
               try {
                 const response = await axios.get(req.url, {

src/serve_style.js

@@ -112,7 +112,7 @@ export const serve_style = {
    * @param {object} repo Repository object.
    * @param {object} params Parameters object containing style path
    * @param {string} id ID of the style.
-   * @param {string} publicUrl Public URL of the data.
+   * @param {object} programOpts - An object containing the program options
    * @param {Function} reportTiles Function for reporting tile sources.
    * @param {Function} reportFont Function for reporting font usage
    * @returns {boolean} true if add is succesful
@@ -122,10 +122,11 @@ export const serve_style = {
     repo,
     params,
     id,
-    publicUrl,
+    programOpts,
     reportTiles,
     reportFont,
   ) {
+    const { publicUrl } = programOpts;
     const styleFile = path.resolve(options.paths.styles, params.style);
 
     let styleFileData;

src/server.js

@@ -193,7 +193,7 @@ async function start(opts) {
         serving.styles,
         item,
         id,
-        opts.publicUrl,
+        opts,
         (styleSourceId, protocol) => {
           let dataItemId;
           for (const id of Object.keys(data)) {
@@ -250,7 +250,7 @@ async function start(opts) {
             serving.rendered,
             item,
             id,
-            opts.publicUrl,
+            opts,
             function dataResolver(styleSourceId) {
               let fileType;
               let inputFile;
@@ -301,9 +301,7 @@ async function start(opts) {
       );
       continue;
     }
-    startupPromises.push(
+    startupPromises.push(serve_data.add(options, serving.data, item, id, opts));
-      serve_data.add(options, serving.data, item, id, opts.publicUrl),
-    );
   }
   if (options.serveAllStyles) {
     fs.readdir(options.paths.styles, { withFileTypes: true }, (err, files) => {

src/utils.js

@@ -6,6 +6,7 @@ import fs from 'node:fs';
 import clone from 'clone';
 import { combine } from '@jsse/pbfont';
 import { existsP } from './promises.js';
+import { getPMtilesTile } from './pmtiles_adapter.js';
 
 /**
  * Restrict user input to an allowed set of options.
@@ -310,3 +311,31 @@ export function isValidHttpUrl(string) {
 
   return url.protocol === 'http:' || url.protocol === 'https:';
 }
+
+/**
+ * Fetches tile data from either PMTiles or MBTiles source.
+ * @param {object} source - The source object, which may contain a mbtiles object, or pmtiles object.
+ * @param {string} sourceType - The source type, which should be `pmtiles` or `mbtiles`
+ * @param {number} z - The zoom level.
+ * @param {number} x - The x coordinate of the tile.
+ * @param {number} y - The y coordinate of the tile.
+ * @returns {Promise<object | null>} - A promise that resolves to an object with data and headers or null if no data is found.
+ */
+export async function fetchTileData(source, sourceType, z, x, y) {
+  if (sourceType === 'pmtiles') {
+    return await new Promise(async (resolve) => {
+      const tileinfo = await getPMtilesTile(source, z, x, y);
+      if (!tileinfo?.data) return resolve(null);
+      resolve({ data: tileinfo.data, headers: tileinfo.header });
+    });
+  } else if (sourceType === 'mbtiles') {
+    return await new Promise((resolve) => {
+      source.getTile(z, x, y, (err, tileData, tileHeader) => {
+        if (err) {
+          return resolve(null);
+        }
+        resolve({ data: tileData, headers: tileHeader });
+      });
+    });
+  }
+}