Merge pull request #11 from trailbehind/put-tile-callback

Put tile callback
trailbehind · Jan 24, 2018 · f469d99 · f469d99
2 parents 0aef253 + 2a44079
commit f469d99
Show file tree

Hide file tree

Showing 11 changed files with 8,236 additions and 199 deletions.
diff --git a/.prettierrc b/.prettierrc
@@ -0,0 +1,3 @@
+{
+  "printWidth": 100
+}
diff --git a/bin/add-bbox.js b/bin/add-bbox.js
@@ -1,10 +1,11 @@
 #!/usr/bin/env node
 
 "use strict";
-var debug = debug = require("debug")("tile-squirrel-add-bbox"),
+var async = require("async"),
+  debug = require("debug")("tile-squirrel-add-bbox"),
   path = require("path"),
   SphericalMercator = require("sphericalmercator"),
-  QueueWriter =  require("../lib/queueWriter");
+  QueueWriter = require("../lib/queueWriter");
 
 var nomnom = require("nomnom")
   .options({
@@ -16,13 +17,14 @@ var nomnom = require("nomnom")
     config: {
       abbr: "c",
       metavar: "CONFIG",
-      help: "Provide a configuration file. Configuration file is not needed," + 
-      " but if it is provided sources will be verified to exist in config."
+      help:
+        "Provide a configuration file. Configuration file is not needed," +
+        " but if it is provided sources will be verified to exist in config."
     },
     dryRun: {
       abbr: "d",
       flag: true,
-      help: "Dry run. Don't actually add messages to queue.",
+      help: "Dry run. Don't actually add messages to queue."
     },
     zoom: {
       abbr: "z",
@@ -59,21 +61,21 @@ var nomnom = require("nomnom")
 
 var opts = nomnom.parse();
 
-if(!opts.zoom) {
-  console.log("Zoom paramater is required.")
+if (!opts.zoom) {
+  console.log("Zoom paramater is required.");
   process.exit(-1);
 }
 
-if(!opts.sources || opts.sources.length == 0) {
+if (!opts.sources || opts.sources.length == 0) {
   console.log("At least 1 source name is required.");
   process.exit(-1);
 }
 
 //Check if source exists if a config was specified
-if(opts.config) {
+if (opts.config) {
   var config = require(path.resolve(opts.config));
   for (var i = 0; i < opts.sources.length; i++) {
-    if(!config[opts.sources[i]]) {
+    if (!config[opts.sources[i]]) {
       console.log("Source " + opts.sources[i] + " not found in config");
       process.exit(-1);
     }
@@ -83,13 +85,13 @@ if(opts.config) {
 var minZoom = 0,
   maxZoom = 0;
 
-if((typeof opts.zoom) === "number") {
+if (typeof opts.zoom === "number") {
   minZoom = maxZoom = opts.zoom;
-} else if(opts.zoom.toString().search("-") == -1) {
+} else if (opts.zoom.toString().search("-") == -1) {
   minZoom = maxZoom = parseInt(opts.zoom.toString());
 } else {
   var zoomComponents = opts.zoom.toString().split("-");
-  if(zoomComponents.length != 2) {
+  if (zoomComponents.length != 2) {
     console.log("Error: zoom range must be 2 numbers seperated by a -");
     process.exit(-1);
   }
@@ -101,49 +103,87 @@ if((typeof opts.zoom) === "number") {
 debug("Zoom range: " + minZoom + "-" + maxZoom);
 
 var bbox = opts.bbox.split(",");
-if(bbox.length != 4) {
+if (bbox.length != 4) {
   console.log("BBOX must have 4 numbers seperated by ,");
   process.exit(-1);
 }
 
-for(var i = 0; i < bbox.length; i++) {
+for (var i = 0; i < bbox.length; i++) {
   bbox[i] = parseFloat(bbox[i]);
 }
 
 var merc = new SphericalMercator({
-    size: 256
+  size: 256
 });
 
 function iterateTiles(queueWriter) {
-  for(var zoom = minZoom; zoom <= maxZoom; zoom++) {
-    var tileBounds = merc.xyz(bbox, zoom);
-    debug(zoom, tileBounds);
-    var zoomString = zoom.toString();
-    for(var x = tileBounds.minX; x <= tileBounds.maxX; x += opts.xSize) {
-      var maxX = Math.min(x + opts.xSize - 1, tileBounds.maxX);
-      var xRangeString;
-      if(x == maxX) {
-        xRangeString = x.toString();
-      } else {
-        xRangeString = x.toString() + "-" + maxX.toString();
+  var zooms = [];
+  for (var zoom = minZoom; zoom <= maxZoom; zoom++) {
+    zooms.push(zoom);
+  }
+  async.eachSeries(
+    zooms,
+    function(zoom, zoomCallback) {
+      var tileBounds = merc.xyz(bbox, zoom);
+      debug(zoom, tileBounds);
+      var zoomString = zoom.toString();
+      var xValues = [];
+      for (var x = tileBounds.minX; x <= tileBounds.maxX; x += opts.xSize) {
+        xValues.push(x);
       }
-      for(var y = tileBounds.minY; y <= tileBounds.maxY; y += opts.ySize) {
-        var maxY = Math.min(y + opts.ySize - 1, tileBounds.maxY);
-        var yRangeString;
-        if(y == maxY) {
-          yRangeString = y.toString();
-        } else {
-          yRangeString = y.toString() + "-" + maxY.toString();
+      async.eachSeries(
+        xValues,
+        function(x, xCallback) {
+          var maxX = Math.min(x + opts.xSize - 1, tileBounds.maxX);
+          var xRangeString;
+          if (x == maxX) {
+            xRangeString = x.toString();
+          } else {
+            xRangeString = x.toString() + "-" + maxX.toString();
+          }
+          var yValues = [];
+          for (var y = tileBounds.minY; y <= tileBounds.maxY; y += opts.ySize) {
+            yValues.push(y);
+          }
+          async.eachSeries(
+            yValues,
+            function(y, yCallback) {
+              var maxY = Math.min(y + opts.ySize - 1, tileBounds.maxY);
+              var yRangeString;
+              if (y == maxY) {
+                yRangeString = y.toString();
+              } else {
+                yRangeString = y.toString() + "-" + maxY.toString();
+              }
+              var tileRangeName =
+                zoomString + "/" + xRangeString + "/" + yRangeString;
+              debug(tileRangeName);
+              queueWriter.putTile(tileRangeName, yCallback);
+            },
+            function(err) {
+              xCallback(err);
+            }
+          );
+        },
+        function(err) {
+          debug("finished with zoom: " + zoom);
+          zoomCallback(err);
         }
-        var tileRangeName = zoomString + "/" + xRangeString + "/" +  yRangeString;
-        debug(tileRangeName);
-        queueWriter.putTile(tileRangeName);
+      );
+    },
+    function(err) {
+      if (err) {
+        console.log(err);
       }
+      debug("calling tileStream.end");
+      queueWriter.tileStream.end();
     }
-  }
-  queueWriter.tileStream.end();
-};
+  );
+}
 
-new QueueWriter(opts.sources, {dryRun:opts.dryRun}, function(err, queueWriter) {
+new QueueWriter(opts.sources, { dryRun: opts.dryRun }, function(
+  err,
+  queueWriter
+) {
   iterateTiles(queueWriter);
 });
diff --git a/bin/add-tile-list.js b/bin/add-tile-list.js
@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 
 "use strict";
-var debug =  require("debug")("tile-squirrel-add-tile-list"),
-  QueueWriter =  require("../lib/queueWriter");
+var debug = require("debug")("tile-squirrel-add-tile-list"),
+  QueueWriter = require("../lib/queueWriter");
 
 var nomnom = require("nomnom")
   .options({
@@ -29,19 +29,22 @@ var nomnom = require("nomnom")
 
 var opts = nomnom.parse();
 
-if(!opts.sources || opts.sources.length == 0) {
+if (!opts.sources || opts.sources.length == 0) {
   console.log("At least 1 source name is required.");
   process.exit(-1);
 }
 
 new QueueWriter(opts.sources, {}, function(err, queueWriter) {
-  var pipe = process.stdin.pipe(require('split')());
-  pipe.on('data', function processLine (line) {
-    if(line.length) {
-      queueWriter.putTile(line);
+  var pipe = process.stdin.pipe(require("split")());
+  pipe.on("data", function(line) {
+    if (line.length) {
+      pipe.pause();
+      queueWriter.putTile(line, function() {
+        pipe.resume();
+      });
     }
   });
-  pipe.on('end', function() {
+  pipe.on("end", function() {
     queueWriter.tileStream.end();
   });
 });
diff --git a/bin/add-tile.js b/bin/add-tile.js
@@ -2,26 +2,26 @@
 
 "use strict";
 
-var debug =  require("debug")("tile-squirrel-add-tile"),
-  QueueWriter =  require("../lib/queueWriter"),
+var debug = require("debug")("tile-squirrel-add-tile"),
+  QueueWriter = require("../lib/queueWriter"),
   path = require("path");
 
-
 var nomnom = require("nomnom")
   .options({
     source: {
       position: 0,
-      help: "Source to queue",
+      help: "Source to queue"
     },
     tile: {
       position: 1,
-      help: "Tile to queue",
+      help: "Tile to queue"
     },
     config: {
       abbr: "c",
       metavar: "CONFIG",
-      help: "Provide a configuration file. Configuration file is not needed," + 
-      " but if it is provided sources will be verified to exist in config."
+      help:
+        "Provide a configuration file. Configuration file is not needed," +
+        " but if it is provided sources will be verified to exist in config."
     },
     version: {
       abbr: "v",
@@ -36,20 +36,21 @@ var nomnom = require("nomnom")
 
 var opts = nomnom.parse();
 
-if(!opts.source || !opts.tile) {
-    nomnom.print(nomnom.getUsage());
+if (!opts.source || !opts.tile) {
+  nomnom.print(nomnom.getUsage());
 }
 
 //Check if source exists if a config was specified
-if(opts.config) {
+if (opts.config) {
   var config = require(path.resolve(opts.config));
-  if(!config[opts.source]) {
+  if (!config[opts.source]) {
     console.log("Source " + opts.source + " not found in config");
     process.exit(-1);
   }
 }
 
 new QueueWriter([opts.source], {}, function(err, queueWriter) {
-  queueWriter.putTile(opts.tile);
-  queueWriter.tileStream.end();
+  queueWriter.putTile(opts.tile, function() {
+    queueWriter.tileStream.end();
+  });
 });
diff --git a/bin/copy-info.js b/bin/copy-info.js
@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 "use strict";
-var debug = debug = require("debug")("tile-squirrel-add-bbox"),
+var debug = (debug = require("debug")("tile-squirrel-add-bbox")),
   path = require("path"),
   Q = require("q");
 
@@ -30,12 +30,12 @@ var nomnom = require("nomnom")
 
 var opts = nomnom.parse();
 
-if(!opts.sources || opts.sources.length == 0) {
+if (!opts.sources || opts.sources.length == 0) {
   console.log("At least 1 source name is required.");
   process.exit(-1);
 }
 
-if(!opts.config) {
+if (!opts.config) {
   console.log("Config option is required.");
   process.exit(-1);
 }
@@ -52,31 +52,31 @@ for (var i = 0; i < opts.sources.length; i++) {
 
   var deferred = Q.defer();
   sourceLoadingPromises.push(deferred.promise);
-  tilelive.load(config[sourceKey]['source'], function(err, src){
-    if(err) {
-      console.log("Error loading source " + config[sourceKey]['source']);
+  tilelive.load(config[sourceKey]["source"], function(err, src) {
+    if (err) {
+      console.log("Error loading source " + config[sourceKey]["source"]);
       return deferred.reject(err);
     }
-    tilelive.load(config[sourceKey]['destination'], function(err, dest){
-      if(err) {
-        console.log("Error loading destination " + config[sourceKey]['destination']);
+    tilelive.load(config[sourceKey]["destination"], function(err, dest) {
+      if (err) {
+        console.log("Error loading destination " + config[sourceKey]["destination"]);
         return deferred.reject(err);
       }
-      src.getInfo(function(err, data){
-        if(err){
+      src.getInfo(function(err, data) {
+        if (err) {
           return deferred.reject(err);
         }
-        if(data) {
-          dest.startWriting(function(err){
-            if(err) {
+        if (data) {
+          dest.startWriting(function(err) {
+            if (err) {
               return deferred.reject(err);
             }
-            dest.putInfo(data, function(err){
-              if(err) {
+            dest.putInfo(data, function(err) {
+              if (err) {
                 return deferred.reject(err);
               }
-              dest.stopWriting(function(err){
-                if(err) {
+              dest.stopWriting(function(err) {
+                if (err) {
                   deferred.reject(err);
                 } else {
                   deferred.resolve();
@@ -89,11 +89,14 @@ for (var i = 0; i < opts.sources.length; i++) {
     });
   });
 }
-
-Q.all(sourceLoadingPromises).then(function() {
-  process.exit(0);
-}, function(err) {
-  debug("Error loading sources");
-  console.log(err);
-  process.exit(-1);
-});
+
+Q.all(sourceLoadingPromises).then(
+  function() {
+    process.exit(0);
+  },
+  function(err) {
+    debug("Error loading sources");
+    console.log(err);
+    process.exit(-1);
+  }
+);