From a679abcab4674c590946a47bcf4575497247459c Mon Sep 17 00:00:00 2001 From: Dimosthenis Kaponis Date: Tue, 6 Mar 2018 16:07:36 +0000 Subject: [PATCH 1/8] Update dependencies to use mapbox-gl-native 3.5.8 instead of 3.5.4 which doesn't compile on llvm/xcode 9. Fix tests. Add tests for markers. --- package.json | 8 +++++--- src/serve_rendered.js | 34 +++++++++++++++++++++++++++++++++- test/static.js | 7 ++++++- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 05bae9b..eb6f22c 100644 --- a/package.json +++ b/package.json @@ -19,13 +19,13 @@ "test": "mocha test/**.js --timeout 10000" }, "dependencies": { - "@mapbox/mapbox-gl-native": "3.5.4", + "@mapbox/mapbox-gl-native": "3.5.8", "@mapbox/mbtiles": "0.9.0", "@mapbox/sphericalmercator": "1.0.5", "@mapbox/vector-tile": "1.3.0", "advanced-pool": "0.3.3", "base64url": "2.0.0", - "canvas": "1.6.8", + "canvas": "^1.6.8", "clone": "2.1.1", "color": "1.0.3", "commander": "2.1.0", @@ -35,10 +35,12 @@ "handlebars": "4.0.11", "http-shutdown": "^1.2.0", "morgan": "1.9.0", + "nomnom": "1.8.1", + "npm": "^5.7.1", "pbf": "3.0.5", "proj4": "2.4.4", "request": "2.83.0", - "sharp": "0.18.2", + "sharp": "^0.20.0", "tileserver-gl-styles": "1.2.0" }, "devDependencies": { diff --git a/src/serve_rendered.js b/src/serve_rendered.js index aca8295..ac339f6 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -23,6 +23,8 @@ var Canvas = require('canvas'), var utils = require('./utils'); +var markerSize = 20; + var FLOAT_PATTERN = '[+-]?(?:\\d+|\\d+\.?\\d+)'; var getScale = function(scale) { @@ -263,6 +265,20 @@ module.exports = function(options, repo, params, id, dataResolver) { styleJSON.glyphs = 'fonts://' + styleJSON.glyphs; } + var markerImages = []; + + var markerImageNames = ['pickup','dropoff']; + + markerImageNames.forEach(function(imageName){ + var imageData = fs.readFileSync("./" + imageName + "-marker.png"); + // TODO: HANDLE ERROR! + var mkrImage = new Canvas.Image(); + mkrImage.src = imageData; + markerImages.push(mkrImage); + }); + + + var tileJSON = { 'tilejson': '2.0.0', 'name': styleJSON.name, @@ -568,6 +584,22 @@ module.exports = function(options, repo, params, id, dataResolver) { ctx.stroke(); } + if (query.showMarkers && query.showMarkers == 1) { + // Add the markers, if requested to do so. + + var markers = [ + [markerImages[0], precisePx(path[0],z).map(function(loc,idx){ return (idx ==1)? loc - markerSize/2: loc - markerSize/2;})], + [markerImages[1], precisePx(path[path.length-1],z).map(function(loc,idx){ return (idx == 1)?loc - markerSize/2: loc - markerSize/2;})] + ]; + console.log(markers); + + markers.forEach(function(imgSpec){ + console.log(imgSpec); + var coordinates = imgSpec[1]; + ctx.drawImage(imgSpec[0], coordinates[0], coordinates[1], markerSize, markerSize); + }); + } + return canvas.toBuffer(); }; @@ -615,7 +647,7 @@ module.exports = function(options, repo, params, id, dataResolver) { format = req.params.format; if (z < 0) { - return res.status(404).send('Invalid zoom'); + return res.status(400).send('Invalid zoom'); } var transformer = raw ? diff --git a/test/static.js b/test/static.js index fc3b59b..ccb5aef 100644 --- a/test/static.js +++ b/test/static.js @@ -49,7 +49,7 @@ describe('Static endpoints', function() { testStatic(prefix, '0,0,0/256x256', 'gif', 400); testStatic(prefix, '0,0,0/256x256', 'png', 404, 1); - testStatic(prefix, '0,0,-1/256x256', 'png', 404); + testStatic(prefix, '0,0,-1/256x256', 'png', 400); testStatic(prefix, '0,0,0/256.5x256.5', 'png', 404); testStatic(prefix, '0,0,0,/256x256', 'png', 404); @@ -91,6 +91,11 @@ describe('Static endpoints', function() { testStatic(prefix, 'auto/20x20', 'png', 200, 2, /image\/png/, '?path=10,10|20,20'); testStatic(prefix, 'auto/200x200', 'png', 200, 3, /image\/png/, '?path=-10,-10|-20,-20'); }); + + describe('with markers', function() { + testStatic(prefix, 'auto/20x20', 'png', 200, 2, /image\/png/, '?path=10,10|20,20&showMarkers=1'); + testStatic(prefix, 'auto/20x20', 'png', 200, 2, /image\/png/, '?path=10,10|20,20&showMarkers=0'); + }) }); describe('invalid requests return 4xx', function() { From bb7d61258fc4afae40136b0bb6174539f52f5649 Mon Sep 17 00:00:00 2001 From: Dimosthenis Kaponis Date: Tue, 6 Mar 2018 17:26:00 +0000 Subject: [PATCH 2/8] Add marker images and updated code to reference those images in the right location. --- public/resources/images/dropoff-marker.png | Bin 0 -> 474 bytes public/resources/images/pickup-marker.png | Bin 0 -> 486 bytes src/serve_rendered.js | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 public/resources/images/dropoff-marker.png create mode 100644 public/resources/images/pickup-marker.png diff --git a/public/resources/images/dropoff-marker.png b/public/resources/images/dropoff-marker.png new file mode 100644 index 0000000000000000000000000000000000000000..4ba2e9131dac7eb13030540cbdc373214f3fb778 GIT binary patch literal 474 zcmV<00VV#4P)Px$lu1NER5%f(Rk2C~K@fd=xs(#n&Q2o%ApyaBfnY09NIJj3#wKEEZE9OTAc6)z zz{WO$7(o!CT}m-nhPcjKGsoVUQ!m+OXXnj(x4SbtXE0@K+HQW>d#wSsW`U|PSZ0{D z7#N@nm`vI>JSM-ENhA@RGuY$tjZ47e%hP|3nb3$W zd8dW)>Uzj*N+y3zK#ZBP;D)FbvNTd+U`J!To!#K$`rd^s>}+81U<)RnOQG0W$T~|w zo!I;NZ3L^mbhsV)e1lqn7OfActvk~-=V?C0sTI|aI_I4Oxy(VD2=6R8ruj66S|Q)2 zoV}z@e`W$W^q86yKgPGIW!4@p>K3-`1GCx&rV+4fuqbC0QJyPo(F)jtBr~(UVO5iv z8WSzRwIF%m9-hq@P)Px$ph-kQR5%f}RliF@Q5623rIF@R87MS488q1v3H=Kelz%~6Lj+-yv`9u(WbPQWspA>Fc4x;v*r|5gnE8(j?xNy&R&iS6_+Rv6M# zZ-jl5)BNcG6!}AxXf)y!l&hhJ$KysOvkzT=G?wgz8;y3HhY~nuz?>>5o!&z-xdXr7 zXRKH(O2J7K3QwwVa@rJo82MnyGpY#rWx?Qr#Z|4oBNSSp2=Y0pk Date: Tue, 6 Mar 2018 17:37:37 +0000 Subject: [PATCH 3/8] Update license and readme. Remove debug logging statmenets. (#3) --- LICENSE.md | 1 + README.md | 42 +----------------------------------------- src/serve_rendered.js | 2 -- 3 files changed, 2 insertions(+), 43 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index 60d6118..1160c6d 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -2,6 +2,7 @@ TileServer GL ============= Copyright (c) 2016, Klokan Technologies GmbH +Copyright (c) 2018, Beat (markers feature) All rights reserved. diff --git a/README.md b/README.md index 044b31e..6130374 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,8 @@ -![tileserver-gl](https://cloud.githubusercontent.com/assets/59284/18173467/fa3aa2ca-7069-11e6-86b1-0f1266befeb6.jpeg) - - # TileServer GL -[![Build Status](https://travis-ci.org/klokantech/tileserver-gl.svg?branch=master)](https://travis-ci.org/klokantech/tileserver-gl) -[![Docker Hub](https://img.shields.io/badge/docker-hub-blue.svg)](https://hub.docker.com/r/klokantech/tileserver-gl/) Vector and raster maps with GL styles. Server side rendering by Mapbox GL Native. Map tile server for Mapbox GL JS, Android, iOS, Leaflet, OpenLayers, GIS via WMTS, etc. -## Get Started - -Make sure you have Node.js version **6** installed (running `node -v` it should output something like `v6.11.3`). - -Install `tileserver-gl` with server-side raster rendering of vector tiles with npm - -```bash -npm install -g tileserver-gl -``` - -Now download vector tiles from [OpenMapTiles](https://openmaptiles.org/downloads/). - -```bash -curl -o zurich_switzerland.mbtiles https://[GET-YOUR-LINK]/extracts/zurich_switzerland.mbtiles -``` - -Start `tileserver-gl` with the downloaded vector tiles. - -```bash -tileserver-gl zurich_switzerland.mbtiles -``` - -Alternatively, you can use the `tileserver-gl-light` package instead, which is pure javascript (does not have any native dependencies) and can run anywhere, but does not contain rasterization on the server side made with MapBox GL Native. - -## Using Docker - -An alternative to npm to start the packed software easier is to install [Docker](http://www.docker.com/) on your computer and then run in the directory with the downloaded MBTiles the command: - -```bash -docker run --rm -it -v $(pwd):/data -p 8080:80 klokantech/tileserver-gl -``` - -This will download and start a ready to use container on your computer and the maps are going to be available in webbrowser on localhost:8080. - -On laptop you can use [Docker Kitematic](https://kitematic.com/) and search "tileserver-gl" and run it, then drop in the 'data' folder the MBTiles. ## Documentation -You can read full documentation of this project at http://tileserver.readthedocs.io/. +You can read full documentation of the upstream project at http://tileserver.readthedocs.io/. diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 9998707..8a7fc4b 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -591,10 +591,8 @@ module.exports = function(options, repo, params, id, dataResolver) { [markerImages[0], precisePx(path[0],z).map(function(loc,idx){ return (idx ==1)? loc - markerSize/2: loc - markerSize/2;})], [markerImages[1], precisePx(path[path.length-1],z).map(function(loc,idx){ return (idx == 1)?loc - markerSize/2: loc - markerSize/2;})] ]; - console.log(markers); markers.forEach(function(imgSpec){ - console.log(imgSpec); var coordinates = imgSpec[1]; ctx.drawImage(imgSpec[0], coordinates[0], coordinates[1], markerSize, markerSize); }); From fede206b1406d24b92880516e9b1ea9ed9f5b5c3 Mon Sep 17 00:00:00 2001 From: cosmix Date: Tue, 6 Mar 2018 17:40:00 +0000 Subject: [PATCH 4/8] Delete Dockerfile_light (#4) --- Dockerfile_light | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 Dockerfile_light diff --git a/Dockerfile_light b/Dockerfile_light deleted file mode 100644 index b3926ee..0000000 --- a/Dockerfile_light +++ /dev/null @@ -1,12 +0,0 @@ -FROM node:6 -MAINTAINER Petr Sloup - -ENV NODE_ENV="production" -EXPOSE 80 -VOLUME /data -WORKDIR /data -ENTRYPOINT ["node", "/usr/src/app/", "-p", "80"] - -RUN mkdir -p /usr/src/app -COPY / /usr/src/app -RUN cd /usr/src/app && npm install --production From 5b293db64f5de157ab615cbc8971f1c2cc6c0842 Mon Sep 17 00:00:00 2001 From: cosmix Date: Sat, 10 Mar 2018 17:07:36 +0000 Subject: [PATCH 5/8] Remove 'light' README. Make marker image loading async. (#5) --- README_light.md | 17 ----------------- src/serve_rendered.js | 27 +++++++++++++++++---------- 2 files changed, 17 insertions(+), 27 deletions(-) delete mode 100644 README_light.md diff --git a/README_light.md b/README_light.md deleted file mode 100644 index f31745d..0000000 --- a/README_light.md +++ /dev/null @@ -1,17 +0,0 @@ -# TileServer GL light -[![Build Status](https://travis-ci.org/klokantech/tileserver-gl.svg?branch=master)](https://travis-ci.org/klokantech/tileserver-gl) -[![Docker Hub](https://img.shields.io/badge/docker-hub-blue.svg)](https://hub.docker.com/r/klokantech/tileserver-gl/) - -Vector maps with GL styles. Map tile server for Mapbox Android, iOS, GL JS, Leaflet, OpenLayers, etc. without server side rendering. - -## Quickstart -Use `npm install -g tileserver-gl-light` to install the package from npm. - -Then you can simply run `tileserver-gl-light zurich_switzerland.mbtiles` to start the server for the given mbtiles. - -See also `tileserver-gl` which contains server side rendering. - -Prepared vector tiles can be downloaded from [OSM2VectorTiles](http://osm2vectortiles.org/). - -## Documentation -You can read full documentation of this project at http://tileserver.readthedocs.io/. \ No newline at end of file diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 8a7fc4b..90de152 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -266,19 +266,26 @@ module.exports = function(options, repo, params, id, dataResolver) { } var markerImages = []; - var markerImageNames = ['pickup','dropoff']; - markerImageNames.forEach(function(imageName){ - var imageData = fs.readFileSync(path.join(__dirname, "../public/resources/images/") + imageName + "-marker.png"); - // TODO: HANDLE ERROR! - var mkrImage = new Canvas.Image(); - mkrImage.src = imageData; - markerImages.push(mkrImage); + + var markerLoadPromise = new Promise(function(resolveCallback, rejectCallback) { + + markerImageNames.forEach(function(imageName){ + fs.readFile(path.join(__dirname, "../public/resources/images/") + imageName + '-marker.png', function(err, fileData) { + + if (err) { + rejectCallback(err); + } + + var mkrImage = new Canvas.Image(); + mkrImage.src = fileData; + markerImages.push(mkrImage); + }); + }); + resolveCallback(); }); - - var tileJSON = { 'tilejson': '2.0.0', 'name': styleJSON.name, @@ -781,7 +788,7 @@ module.exports = function(options, repo, params, id, dataResolver) { return res.send(info); }); - return Promise.all([fontListingPromise, renderersReadyPromise]).then(function() { + return Promise.all([markerLoadPromise, fontListingPromise, renderersReadyPromise]).then(function() { return app; }); From 8a6bf05198dba86fe942b8b81701056112c4cc0a Mon Sep 17 00:00:00 2001 From: cosmix Date: Wed, 28 Mar 2018 01:13:03 +0200 Subject: [PATCH 6/8] Improve health endpoint. Add average response middleware and hook to metrics endpoint. (#6) * WiP. Add average response middleware and hook to health endpoint. * Add standard health formatting, average response time for last 50 requests. * Replace homebrew metrics with prometheus to placate the Vrachnis/Panagiotou duo * Add unit suffix on prometheus metric * Remove dangling sprintf-js dependency and require statement. --- package.json | 3 ++- src/avgresp.js | 42 ++++++++++++++++++++++++++++++++++++++++++ src/server.js | 30 +++++++++++++++++++++++++++--- 3 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 src/avgresp.js diff --git a/package.json b/package.json index eb6f22c..3d07070 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,8 @@ "proj4": "2.4.4", "request": "2.83.0", "sharp": "^0.20.0", - "tileserver-gl-styles": "1.2.0" + "tileserver-gl-styles": "1.2.0", + "prom-client": "11.0.0" }, "devDependencies": { "should": "^11.2.0", diff --git a/src/avgresp.js b/src/avgresp.js new file mode 100644 index 0000000..7ba796d --- /dev/null +++ b/src/avgresp.js @@ -0,0 +1,42 @@ +#!/usr/bin/env node +'use strict'; + +var prometheus = require('prom-client'); + +module.exports = { + avgresp: avgresp +}; + + +var history; +var samples; +var fullhistory = false; + +/** + * Create middleware to record and output average response times + * @param {Object} options + * @return {function} + */ +function avgresp(options) { + + var opts = options || {}; + + history = 50; + samples = new Array(history); + var currentIndex = 0; + + const respSummary = new prometheus.Summary({ + name: "tileserver_static_latency_seconds", + help: "The tileserver response time in seconds" + }); + + return function avgresp(req, res, next) { + var end = respSummary.startTimer(); + + res.on('finish', function() { + end(); + }) + + next(); + } +} diff --git a/src/server.js b/src/server.js index db67be4..64cf3dc 100644 --- a/src/server.js +++ b/src/server.js @@ -21,7 +21,11 @@ var packageJson = require('../package'), serve_rendered = null, serve_style = require('./serve_style'), serve_data = require('./serve_data'), - utils = require('./utils'); + utils = require('./utils'), + avgresp = require('./avgresp'), + prometheus = require('prom-client'); + +prometheus.collectDefaultMetrics({ timeout: 5000 }); var isLight = packageJson.name.slice(-6) == '-light'; if (!isLight) { @@ -52,6 +56,8 @@ function start(opts) { })); } + app.use(/\/styles\/.*\/static\/.*$/, avgresp.avgresp()); + var config = opts.config || null; var configPath = null; if (opts.configPath) { @@ -391,12 +397,30 @@ function start(opts) { console.log('Startup complete'); startupComplete = true; }); + app.get('/health', function(req, res, next) { + + var healthTemplate = { + "service": "tileserver", + "status": "200", + "message": "OK" + }; + + var statusCode = 200; + if (startupComplete) { - return res.status(200).send('OK'); + statusCode = 200; + healthTemplate.message = "OK"; } else { - return res.status(503).send('Starting'); + statusCode = 503; + healthTemplate.message = "Starting"; } + healthTemplate.status = statusCode; + return res.status(statusCode).send(healthTemplate); + }); + + app.get('/metrics', function(req, res, next){ + res.end(prometheus.register.metrics()); }); var server = app.listen(process.env.PORT || opts.port, process.env.BIND || opts.bind, function() { From 4324b009f8676223217fed223482791680a1b15a Mon Sep 17 00:00:00 2001 From: cosmix Date: Wed, 28 Mar 2018 01:58:15 +0200 Subject: [PATCH 7/8] Update README.md (#7) --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 6130374..ef312ad 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,12 @@ Vector and raster maps with GL styles. Server side rendering by Mapbox GL Native. Map tile server for Mapbox GL JS, Android, iOS, Leaflet, OpenLayers, GIS via WMTS, etc. +This fork adds features used by BEAT, including: + +* Marker support in the static (rendered) maps +* Prometheus compatible `/metrics` endpoint +* Improved `/health/` endpoint + ## Documentation From 53dada5a568ada8a1a1aa7d9c592eca50080214d Mon Sep 17 00:00:00 2001 From: Panagiotis Atmatzidis Date: Fri, 30 Mar 2018 12:47:42 +0300 Subject: [PATCH 8/8] add Jenkinsfile --- Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..655f05b --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,3 @@ +@Library("jenkins-devops-scripts") _ +tileserver-gl { +}