Refactor of Mbtiles into generic TileArchive. [onthegomap#98]

* add package com.onthegomap.planetiler.writer * create new interface TileArchive * rename MbtilesMetadata to TileArchiveMetdata, make MbtilesWriter into general TileArchiveWriter
bdon · Jan 13, 2023 · f92ed46 · f92ed46
1 parent 9e4af5b
commit f92ed46
Show file tree

Hide file tree

Showing 11 changed files with 160 additions and 91 deletions.
diff --git a/...benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java b/...benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java
@@ -5,8 +5,8 @@
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.mbtiles.Mbtiles;
-import com.onthegomap.planetiler.mbtiles.Mbtiles.BatchedTileWriter;
-import com.onthegomap.planetiler.mbtiles.TileEncodingResult;
+import com.onthegomap.planetiler.writer.TileArchive.TileWriter;
+import com.onthegomap.planetiler.writer.TileEncodingResult;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
@@ -74,7 +74,7 @@ public static void main(String[] args) throws IOException {
           mbtiles.createTablesWithIndexes();
         }
 
-        try (var writer = mbtiles.newBatchedTileWriter()) {
+        try (var writer = mbtiles.newTileWriter()) {
           Stopwatch sw = Stopwatch.createStarted();
           writeTiles(writer, tilesToWrite, distinctTilesInPercent, distinctTileData, dupeTileData, dupeSpreadInPercent);
           sw.stop();
@@ -92,7 +92,7 @@ public static void main(String[] args) throws IOException {
   }
 
 
-  private static void writeTiles(BatchedTileWriter writer, int tilesToWrite, int distinctTilesInPercent,
+  private static void writeTiles(TileWriter writer, int tilesToWrite, int distinctTilesInPercent,
     byte[] distinctTileData, byte[] dupeTileData, int dupeSpreadInPercent) {
 
     int dupesToWrite = (int) Math.round(tilesToWrite * (100 - distinctTilesInPercent) / 100.0);

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java
@@ -4,9 +4,8 @@
 import com.onthegomap.planetiler.collection.LongLongMap;
 import com.onthegomap.planetiler.collection.LongLongMultimap;
 import com.onthegomap.planetiler.config.Arguments;
-import com.onthegomap.planetiler.config.MbtilesMetadata;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
-import com.onthegomap.planetiler.mbtiles.MbtilesWriter;
+import com.onthegomap.planetiler.mbtiles.Mbtiles;
 import com.onthegomap.planetiler.reader.GeoPackageReader;
 import com.onthegomap.planetiler.reader.NaturalEarthReader;
 import com.onthegomap.planetiler.reader.ShapefileReader;
@@ -27,6 +26,9 @@
 import com.onthegomap.planetiler.util.Translations;
 import com.onthegomap.planetiler.util.Wikidata;
 import com.onthegomap.planetiler.worker.RunnableThatThrows;
+import com.onthegomap.planetiler.writer.TileArchiveMetadata;
+import com.onthegomap.planetiler.writer.TileArchiveWriter;
+import java.io.IOException;
 import java.nio.file.FileSystem;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -551,6 +553,7 @@ public void run() throws Exception {
     }
     ran = true;
     mbtilesMetadata = new MbtilesMetadata(profile, config.arguments());
+    TileArchiveMetadata tileArchiveMetadata = new TileArchiveMetadata(profile, config.arguments());
 
     if (arguments.getBoolean("help", "show arguments then exit", false)) {
       System.exit(0);
@@ -633,7 +636,12 @@ public void run() throws Exception {
 
     featureGroup.prepare();
 
-    MbtilesWriter.writeOutput(featureGroup, output, mbtilesMetadata, config, stats);
+    try (Mbtiles db = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) {
+      TileArchiveWriter.writeOutput(featureGroup, db, () -> FileUtils.fileSize(output), tileArchiveMetadata, config,
+        stats);
+    } catch (IOException e) {
+      throw new IllegalStateException("Unable to write to " + output, e);
+    }
 
     overallTimer.stop();
     LOGGER.info("FINISHED!");

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
@@ -7,9 +7,14 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
+import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.geo.GeoUtils;
 import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.util.Format;
+import com.onthegomap.planetiler.util.LayerStats;
+import com.onthegomap.planetiler.writer.TileArchive;
+import com.onthegomap.planetiler.writer.TileArchiveMetadata;
+import com.onthegomap.planetiler.writer.TileEncodingResult;
 import java.io.Closeable;
 import java.io.IOException;
 import java.nio.file.Path;
@@ -46,7 +51,7 @@
  *
  * @see <a href="https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md">MBTiles Specification</a>
  */
-public final class Mbtiles implements Closeable {
+public final class Mbtiles implements Closeable, TileArchive {
 
   // https://www.sqlite.org/src/artifact?ci=trunk&filename=magic.txt
   private static final int MBTILES_APPLICATION_ID = 0x4d504258;
@@ -145,6 +150,36 @@ public static Mbtiles newReadOnlyDatabase(Path path) {
     }
   }
 
+  public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) {
+    if (config.skipIndexCreation()) {
+      createTablesWithoutIndexes();
+      if (LOGGER.isInfoEnabled()) {
+        LOGGER.info("Skipping index creation. Add later by executing: {}",
+          String.join(" ; ", getManualIndexCreationStatements()));
+      }
+    } else {
+      createTablesWithIndexes();
+    }
+
+    metadata()
+      .setName(tileArchiveMetadata.name())
+      .setFormat("pbf")
+      .setDescription(tileArchiveMetadata.description())
+      .setAttribution(tileArchiveMetadata.attribution())
+      .setVersion(tileArchiveMetadata.version())
+      .setType(tileArchiveMetadata.type())
+      .setBoundsAndCenter(config.bounds().latLon())
+      .setMinzoom(config.minzoom())
+      .setMaxzoom(config.maxzoom())
+      .setJson(layerStats.getTileStats());
+  }
+
+  public void finalize(PlanetilerConfig config) {
+    if (config.optimizeDb()) {
+      vacuumAnalyze();
+    }
+  }
+
   @Override
   public void close() throws IOException {
     try {
@@ -281,7 +316,7 @@ public Mbtiles vacuumAnalyze() {
   }
 
   /** Returns a writer that queues up inserts into the tile database(s) into large batches before executing them. */
-  public BatchedTileWriter newBatchedTileWriter() {
+  public TileArchive.TileWriter newTileWriter() {
     if (compactDb) {
       return new BatchedCompactTileWriter();
     } else {
@@ -659,20 +694,7 @@ protected int setParamsInStatementForItem(int positionOffset, PreparedStatement
     }
   }
 
-
-  /**
-   * A high-throughput writer that accepts new tiles and queues up the writes to execute them in fewer large-batches.
-   */
-  public interface BatchedTileWriter extends AutoCloseable {
-    void write(TileEncodingResult encodingResult);
-
-    @Override
-    void close();
-
-    default void printStats() {}
-  }
-
-  private class BatchedNonCompactTileWriter implements BatchedTileWriter {
+  private class BatchedNonCompactTileWriter implements TileWriter {
 
     private final BatchedTileTableWriter tableWriter = new BatchedTileTableWriter();
 
@@ -688,7 +710,7 @@ public void close() {
 
   }
 
-  private class BatchedCompactTileWriter implements BatchedTileWriter {
+  private class BatchedCompactTileWriter implements TileWriter {
 
     private final BatchedTileShallowTableWriter batchedTileShallowTableWriter = new BatchedTileShallowTableWriter();
     private final BatchedTileDataTableWriter batchedTileDataTableWriter = new BatchedTileDataTableWriter();

diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java
@@ -0,0 +1,28 @@
+package com.onthegomap.planetiler.writer;
+
+import com.onthegomap.planetiler.config.PlanetilerConfig;
+import com.onthegomap.planetiler.util.LayerStats;
+
+/**
+ * A TileArchive is a portable on-disk stored tileset, e.g. MBTiles
+ */
+public interface TileArchive {
+
+  /**
+   * A high-throughput writer that accepts new tiles and queues up the writes to execute them in fewer large-batches.
+   */
+  public interface TileWriter extends AutoCloseable {
+    void write(TileEncodingResult encodingResult);
+
+    @Override
+    void close();
+
+    default void printStats() {}
+  }
+
+  void initialize(PlanetilerConfig config, TileArchiveMetadata metadata, LayerStats layerStats);
+
+  TileWriter newTileWriter();
+
+  void finalize(PlanetilerConfig config);
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveMetadata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveMetadata.java
@@ -0,0 +1,35 @@
+package com.onthegomap.planetiler.writer;
+
+import com.onthegomap.planetiler.Profile;
+import com.onthegomap.planetiler.config.Arguments;
+
+/** Controls information that tile writers will write to the archive */
+public record TileArchiveMetadata(
+  String name,
+  String description,
+  String attribution,
+  String version,
+  String type
+) {
+
+  public TileArchiveMetadata(Profile profile) {
+    this(
+      profile.name(),
+      profile.description(),
+      profile.attribution(),
+      profile.version(),
+      profile.isOverlay() ? "overlay" : "baselayer"
+    );
+  }
+
+  public TileArchiveMetadata(Profile profile, Arguments args) {
+    this(
+      args.getString("tileset_name", "'name' attribute for tileset metadata", profile.name()),
+      args.getString("tileset_description", "'description' attribute for tileset metadata", profile.description()),
+      args.getString("tileset_attribution", "'attribution' attribute for tileset metadata", profile.attribution()),
+      args.getString("tileset_version", "'version' attribute for tileset metadata", profile.version()),
+      args.getString("tileset_type", "'type' attribute for tileset metadata",
+        profile.isOverlay() ? "overlay" : "baselayer")
+    );
+  }
+}