Add pmtiles source support (#39)

* try to add pmtiles support * simplify tile type * fix comments * use isOnlineURL where possible * remove unused return * fix local pmtiles extension * simplify * update comment for resolvePMTilesURL * better default bounds * remove unneeded metadata * close file to prevent 'too many open files' error * use 'jpg' instead of 'jpeg' in pmtiles types * simplify bounds and center * use default tilejson bounds * Remove unneeded PMtilesTypes * fix local sprite callback * make open and close pmtiles functions * update comments * remove comment since this is fixed * add test style and assets * add pmtiles test * fix style name * add jest pmtiles exclusion
ConservationMetrics · Feb 19, 2024 · d6b9884 · d6b9884
1 parent bbed6d2
commit d6b9884
Show file tree

Hide file tree

Showing 7 changed files with 384 additions and 19 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -22,6 +22,7 @@
     "osmtogeojson": "^3.0.0-beta.5",
     "p-limit": "^5.0.0",
     "path": "^0.12.7",
+    "pmtiles": "3.0.3",
     "sharp": "^0.33.2",
     "xmldom": "^0.6.0"
   },
@@ -42,6 +43,11 @@
       "pre-commit": "lint-staged"
     }
   },
+  "jest": {
+    "transformIgnorePatterns": [
+      "node_modules/(?!(pmtiles)/)"
+    ]
+  },
   "lint-staged": {
     "src/**/*.{js,jsx,ts,tsx,json,css}": [
       "prettier --write ./src",

diff --git a/src/request_resources.js b/src/request_resources.js
@@ -2,19 +2,44 @@ import fs from "fs";
 import path from "path";
 import zlib from "zlib";
 import MBTiles from "@mapbox/mbtiles";
+import { PMTiles, FetchSource } from "pmtiles";
 import https from "https";
 
 // Validations for source URLs
-const TILE_REGEXP = RegExp("mbtiles://([^/]+)/(\\d+)/(\\d+)/(\\d+)");
+const MBTILES_REGEXP = RegExp("mbtiles://([^/]+)/(\\d+)/(\\d+)/(\\d+)");
+const PMTILES_REGEXP = RegExp("pmtiles://([^/]+)(?:/)?.*/(\\d+)/(\\d+)/(\\d+)");
+const HTTP_REGEX = RegExp("^http(s)?://");
 const XYZ_REGEXP = /(\d+)\/(\d+)\/(\d+)\.(jpg|png|pbf|mvt)$/;
 const isMBTilesURL = (url) => url.startsWith("mbtiles://");
+const isPMTilesURL = (url) => url.startsWith("pmtiles://");
 const isGeoJSONURL = (url) => url.endsWith(".geojson");
 const isProtomapsTileJSONURL = (url) => url.endsWith("protomaps-tiles.json");
 const isXYZDirURL = (url) => /\/?\d+\/\d+\/\d+(\.\w+)?$/.test(url);
-const isOnlineURL = (url) => url.startsWith("http");
+const isOnlineURL = (url) => url.match(HTTP_REGEX);
 
 // Split out mbtiles service name from the URL
 const resolveNamefromURL = (url) => url.split("://")[1].split("/")[0];
+const resolveNamefromPMtilesURL = (url) => url.split("pmtiles://")[1];
+
+// Resolve a URL of a local pmtiles file to a file path or return the url for a http(s) pmtiles file
+// Expected to follow this format "pmtiles://<service_name>/*" for local files or "pmtiles://https://foo.lan/filename.pmtiles/*" for a url file
+const resolvePMTilesURL = (sourceDir, url) => {
+  /*
+   * @param {String} sourceDir - path containing pmtiles files
+   * @param {String} url - url of a data source in style.json file.
+   */
+
+  const pmtilesFile = resolveNamefromPMtilesURL(url);
+  if (isOnlineURL(pmtilesFile)) {
+    return pmtilesFile;
+  } else {
+    return path.format({
+      dir: sourceDir,
+      name: resolveNamefromURL(url),
+      ext: ".pmtiles",
+    });
+  }
+};
 
 // Resolve a URL of a local mbtiles file to a file path
 // Expected to follow this format "mbtiles://<service_name>/*"
@@ -38,20 +63,14 @@ const getLocalSpriteImage = (styleDir, url, callback) => {
       callback(err);
       return null;
     }
-    callback(null, data);
+    callback(null, { data });
     return null;
   });
 };
 
 // Given a URL to a local sprite JSON, get the JSON data.
 const getLocalSpriteJSON = (styleDir, url, callback) => {
   const spriteJsonPath = path.join(styleDir, `${url}`);
-
-  // TODO: currently, any styles with sprites defined will
-  // fail to render. The callback in this function does
-  // correctly return a buffer of the sprite JSON, but
-  // Maplibre just hangs when trying to render it.
-  // No errors are thrown, and the process never exits.
   fs.readFile(spriteJsonPath, (err, data) => {
     if (err) {
       callback(err);
@@ -62,8 +81,8 @@ const getLocalSpriteJSON = (styleDir, url, callback) => {
   });
 };
 
-// Given a URL to an online glyph, get the glyph data.
-const getOnlineGlyph = (url, callback) => {
+// Given a URL to an online data, get the data.
+const getOnlineData = (url, callback) => {
   https
     .get(url, (res) => {
       let data = [];
@@ -96,6 +115,60 @@ const getLocalGlyph = (styleDir, url, callback) => {
   });
 };
 
+// Given a URL to a pmtiles file, get the TileJSON for that to load correct tiles.
+const getPMTilesTileJSON = async (sourceDir, url, callback) => {
+  /*
+   * @param {String} sourceDir - path containing pmtiles files.
+   * @param {String} url - url of a data source in style.json file.
+   * @param {function} callback - function to call with (err, {data}).
+   */
+  const pmtilesFile = resolvePMTilesURL(sourceDir, url);
+  const service = resolveNamefromPMtilesURL(url);
+
+  //Open the pmtiles file
+  const pmtiles = openPMtiles(pmtilesFile);
+
+  //Get PMtiles header information
+  const header = await pmtiles.getHeader();
+
+  // If the tileType is 1(mvt/pbf), add a .pbf extension
+  const ext = header.tileType === 1 ? ".pbf" : "";
+
+  //Close the pmtiles file to prevent too many open files
+  closePMtiles(pmtiles);
+
+  //Add missing metadata from header
+  if (
+    header.minLon == 0 &&
+    header.minLat == 0 &&
+    header.maxLon == 0 &&
+    header.maxLat == 0
+  ) {
+    header["bounds"] = [-180, -85.05112877980659, 180, 85.0511287798066];
+  } else {
+    header["bounds"] = [
+      header.minLon,
+      header.minLat,
+      header.maxLon,
+      header.maxLat,
+    ];
+  }
+  header["center"] = [header.centerLon, header.centerLat, header.centerZoom];
+
+  //Create TileJSON
+  const tileJSON = {
+    tilejson: "1.0.0",
+    tiles: [`pmtiles://${service}/{z}/{x}/{y}${ext}`],
+    minzoom: header.minZoom,
+    maxzoom: header.maxZoom,
+    center: header.center,
+    bounds: header.bounds,
+  };
+
+  callback(null, { data: Buffer.from(JSON.stringify(tileJSON)) });
+  return null;
+};
+
 // Given a URL to a local mbtiles file, get the TileJSON for that to load correct tiles.
 const getLocalMBTileJSON = (sourceDir, url, callback) => {
   /*
@@ -139,14 +212,47 @@ const getLocalMBTileJSON = (sourceDir, url, callback) => {
   });
 };
 
+// Fetch a tile from a pmtiles file.
+const getPMTiles = async (sourceDir, url, callback) => {
+  /*
+   * @param {String} sourceDir - path containing pmtiles files.
+   * @param {String} url - url of a data source in style.json file.
+   * @param {function} callback - function to call with (err, {data}).
+   */
+  const matches = url.match(PMTILES_REGEXP);
+  const [z, x, y] = matches.slice(matches.length - 3, matches.length);
+  const pmtilesFile = resolvePMTilesURL(
+    sourceDir,
+    url.split("/").slice(0, -3).join("/"),
+  );
+
+  //Open the pmtiles file
+  const pmtiles = openPMtiles(pmtilesFile);
+
+  //Get the requested tile
+  let zxyTile = await pmtiles.getZxy(z, x, y);
+
+  //Close the pmtiles file to prevent too many open files
+  closePMtiles(pmtiles);
+
+  //Return the tile data
+  if (zxyTile && zxyTile.data) {
+    const data = Buffer.from(zxyTile.data);
+    callback(null, { data });
+  } else {
+    callback(null, {});
+    return null;
+  }
+};
+
 // Fetch a tile from a local mbtiles file.
 const getLocalMBTile = (sourceDir, url, callback) => {
   /*
    * @param {String} sourceDir - path containing mbtiles files.
    * @param {String} url - url of a data source in style.json file.
    * @param {function} callback - function to call with (err, {data}).
    */
-  const matches = url.match(TILE_REGEXP);
+  const matches = url.match(MBTILES_REGEXP);
   const [z, x, y] = matches.slice(matches.length - 3, matches.length);
   const isVector = path.extname(url) === ".pbf";
   const mbtilesFile = resolveMBTilesURL(sourceDir, url);
@@ -261,6 +367,8 @@ export const requestHandler =
           // source
           if (isMBTilesURL(url)) {
             getLocalMBTileJSON(sourceDir, url, callback);
+          } else if (isPMTilesURL(url)) {
+            getPMTilesTileJSON(sourceDir, url, callback);
           } else if (isXYZDirURL(url)) {
             getLocalXYZTile(sourceDir, url, callback);
           } else if (isGeoJSONURL(url)) {
@@ -277,6 +385,8 @@ export const requestHandler =
           // tile
           if (isMBTilesURL(url)) {
             getLocalMBTile(sourceDir, url, callback);
+          } else if (isPMTilesURL(url)) {
+            getPMTiles(sourceDir, url, callback);
           } else if (isXYZDirURL(url)) {
             getLocalXYZTile(sourceDir, url, callback);
           } else if (isGeoJSONURL(url)) {
@@ -290,20 +400,28 @@ export const requestHandler =
         case 4: {
           // glyph
           if (isOnlineURL(url)) {
-            getOnlineGlyph(url, callback);
+            getOnlineData(url, callback);
           } else {
             getLocalGlyph(styleDir, url, callback);
           }
           break;
         }
         case 5: {
           // sprite image
-          getLocalSpriteImage(styleDir, url, callback);
+          if (isOnlineURL(url)) {
+            getOnlineData(url, callback);
+          } else {
+            getLocalSpriteImage(styleDir, url, callback);
+          }
           break;
         }
         case 6: {
           // sprite json
-          getLocalSpriteJSON(styleDir, url, callback);
+          if (isOnlineURL(url)) {
+            getOnlineData(url, callback);
+          } else {
+            getLocalSpriteJSON(styleDir, url, callback);
+          }
           break;
         }
         default: {
@@ -316,3 +434,55 @@ export const requestHandler =
       callback(msg);
     }
   };
+
+// pmtiles class to allow reading from a local file
+class PMTilesFileSource {
+  constructor(fd) {
+    this.fd = fd;
+  }
+  getKey() {
+    return this.fd;
+  }
+  async getBytes(offset, length) {
+    const buffer = Buffer.alloc(length);
+    await readFileBytes(this.fd, buffer, offset);
+    const ab = buffer.buffer.slice(
+      buffer.byteOffset,
+      buffer.byteOffset + buffer.byteLength,
+    );
+    return { data: ab };
+  }
+}
+
+// reads specified bytes from a file
+async function readFileBytes(fd, buffer, offset) {
+  return new Promise((resolve, reject) => {
+    fs.read(fd, buffer, 0, buffer.length, offset, (err) => {
+      if (err) {
+        return reject(err);
+      }
+      resolve();
+    });
+  });
+}
+
+// open the pmtiles file or url
+function openPMtiles(pmtilesFile) {
+  let pmtiles;
+  if (isOnlineURL(pmtilesFile)) {
+    const source = new FetchSource(pmtilesFile);
+    pmtiles = new PMTiles(source);
+  } else {
+    const fd = fs.openSync(pmtilesFile, "r");
+    const source = new PMTilesFileSource(fd);
+    pmtiles = new PMTiles(source);
+  }
+  return pmtiles;
+}
+
+// close the pmtiles file
+function closePMtiles(pmtiles) {
+  if (pmtiles.source.fd) {
+    fs.closeSync(pmtiles.source.fd);
+  }
+}
diff --git a/tests/fixtures/alert/sources/output-raster.pmtiles b/tests/fixtures/alert/sources/output-raster.pmtiles
diff --git a/tests/fixtures/alert/sources/output-vector.pmtiles b/tests/fixtures/alert/sources/output-vector.pmtiles