Convert get all tile coords to iterator

ARCHITECTURE.md

@@ -96,15 +96,17 @@ of the intermediate features using a worker thread per core:
 
 ## 3) Emit Vector Tiles
 
-[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) is the main driver.
+[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java) is the main
+driver.
 First, a single-threaded reader reads features from disk:
 
 - [ExternalMergeSort](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java) emits
   sorted features by doing a k-way merge using a priority queue of iterators reading from each sorted chunk
 - [FeatureGroup](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java) collects
   consecutive features in the same tile into a `TileFeatures` instance, dropping features in the same group over the
   grouping limit to limit point label density
-- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) groups tiles
+- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java)
+  groups tiles
   into variable-sized batches for workers to process (complex tiles get their own batch to ensure workers stay busy
   while the writer thread waits for finished tiles in order)
 

planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java

@@ -1,12 +1,12 @@
 package com.onthegomap.planetiler.benchmarks;
 
 import com.google.common.base.Stopwatch;
+import com.onthegomap.planetiler.archive.TileEncodingResult;
+import com.onthegomap.planetiler.archive.WriteableTileArchive.TileWriter;
 import com.onthegomap.planetiler.config.Arguments;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.mbtiles.Mbtiles;
-import com.onthegomap.planetiler.writer.TileArchive.TileWriter;
-import com.onthegomap.planetiler.writer.TileEncodingResult;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;

planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java

@@ -1,5 +1,8 @@
 package com.onthegomap.planetiler;
 
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.archive.TileArchiveWriter;
+import com.onthegomap.planetiler.archive.WriteableTileArchive;
 import com.onthegomap.planetiler.collection.FeatureGroup;
 import com.onthegomap.planetiler.collection.LongLongMap;
 import com.onthegomap.planetiler.collection.LongLongMultimap;
@@ -26,9 +29,6 @@
 import com.onthegomap.planetiler.util.Translations;
 import com.onthegomap.planetiler.util.Wikidata;
 import com.onthegomap.planetiler.worker.RunnableThatThrows;
-import com.onthegomap.planetiler.writer.TileArchive;
-import com.onthegomap.planetiler.writer.TileArchiveMetadata;
-import com.onthegomap.planetiler.writer.TileArchiveWriter;
 import java.io.IOException;
 import java.nio.file.FileSystem;
 import java.nio.file.Files;
@@ -658,7 +658,7 @@ public void run() throws Exception {
       bounds.addFallbackProvider(new OsmNodeBoundsProvider(osmInputFile, config, stats));
     }
 
-    try (TileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) {
+    try (WriteableTileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) {
       featureGroup =
         FeatureGroup.newDiskBackedFeatureGroup(archive.tileOrder(), featureDbPath, profile, config, stats);
       stats.monitorFile("nodes", nodeDbPath);

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java

@@ -0,0 +1,44 @@
+package com.onthegomap.planetiler.archive;
+
+import com.onthegomap.planetiler.geo.TileCoord;
+import com.onthegomap.planetiler.util.CloseableIterator;
+import java.io.Closeable;
+
+/**
+ * Read API for on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive
+ * format.
+ * <p>
+ * See {@link WriteableTileArchive} for the write API.
+ */
+public interface ReadableTileArchive extends Closeable {
+
+  /** Returns the raw tile data associated with the tile at {@code coord}. */
+  default byte[] getTile(TileCoord coord) {
+    return getTile(coord.x(), coord.y(), coord.z());
+  }
+
+  /** Returns the raw tile data associated with the tile at coordinate {@code x, y, z}. */
+  byte[] getTile(int x, int y, int z);
+
+  /**
+   * Returns an iterator over the coordinates of tiles in this archive.
+   * <p>
+   * The order should respect {@link WriteableTileArchive#tileOrder()} of the corresponding writer.
+   * <p>
+   * Clients should be sure to close the iterator after iterating through it, for example:
+   *
+   * <pre>
+   * {@code
+   * try (var iter = archive.getAllTileCoords()) {
+   *   while (iter.hasNext()) {
+   *     var coord = iter.next();
+   *     ...
+   *   }
+   * }
+   * }
+   * </pre>
+   */
+  CloseableIterator<TileCoord> getAllTileCoords();
+
+  // TODO access archive metadata
+}

planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveMetadata.java → planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java

@@ -1,4 +1,4 @@
-package com.onthegomap.planetiler.writer;
+package com.onthegomap.planetiler.archive;
 
 import com.onthegomap.planetiler.Profile;
 import com.onthegomap.planetiler.config.Arguments;

planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java → planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java

@@ -1,4 +1,4 @@
-package com.onthegomap.planetiler.writer;
+package com.onthegomap.planetiler.archive;
 
 import static com.onthegomap.planetiler.util.Gzip.gzip;
 import static com.onthegomap.planetiler.worker.Worker.joinFutures;
@@ -40,7 +40,7 @@
 
 /**
  * Final stage of the map generation process that encodes vector tiles using {@link VectorTile} and writes them to a
- * {@link TileArchive}.
+ * {@link WriteableTileArchive}.
  */
 public class TileArchiveWriter {
 
@@ -49,7 +49,7 @@ public class TileArchiveWriter {
   private static final long MAX_TILES_PER_BATCH = 1_000;
   private final Counter.Readable featuresProcessed;
   private final Counter memoizedTiles;
-  private final TileArchive archive;
+  private final WriteableTileArchive archive;
   private final PlanetilerConfig config;
   private final Stats stats;
   private final LayerStats layerStats;
@@ -60,7 +60,7 @@ public class TileArchiveWriter {
   private final AtomicReference<TileCoord> lastTileWritten = new AtomicReference<>();
   private final TileArchiveMetadata tileArchiveMetadata;
 
-  private TileArchiveWriter(Iterable<FeatureGroup.TileFeatures> inputTiles, TileArchive archive,
+  private TileArchiveWriter(Iterable<FeatureGroup.TileFeatures> inputTiles, WriteableTileArchive archive,
     PlanetilerConfig config,
     TileArchiveMetadata tileArchiveMetadata, Stats stats, LayerStats layerStats) {
     this.inputTiles = inputTiles;
@@ -88,7 +88,7 @@ private TileArchiveWriter(Iterable<FeatureGroup.TileFeatures> inputTiles, TileAr
   }
 
   /** Reads all {@code features}, encodes them in parallel, and writes to {@code output}. */
-  public static void writeOutput(FeatureGroup features, TileArchive output, DiskBacked fileSize,
+  public static void writeOutput(FeatureGroup features, WriteableTileArchive output, DiskBacked fileSize,
     TileArchiveMetadata tileArchiveMetadata, PlanetilerConfig config, Stats stats) {
     var timer = stats.startStage("archive");
 
@@ -335,28 +335,30 @@ private void tileWriter(Iterable<TileBatch> tileBatches) throws ExecutionExcepti
   }
 
   private void printTileStats() {
-    Format format = Format.defaultInstance();
-    LOGGER.debug("Tile stats:");
-    long sumSize = 0;
-    long sumCount = 0;
-    long maxMax = 0;
-    for (int z = config.minzoom(); z <= config.maxzoom(); z++) {
-      long totalCount = tilesByZoom[z].get();
-      long totalSize = totalTileSizesByZoom[z].get();
-      sumSize += totalSize;
-      sumCount += totalCount;
-      long maxSize = maxTileSizesByZoom[z].get();
-      maxMax = Math.max(maxMax, maxSize);
-      LOGGER.debug("z{} avg:{} max:{}",
-        z,
-        format.storage(totalCount == 0 ? 0 : (totalSize / totalCount), false),
-        format.storage(maxSize, false));
+    if (LOGGER.isDebugEnabled()) {
+      Format format = Format.defaultInstance();
+      LOGGER.debug("Tile stats:");
+      long sumSize = 0;
+      long sumCount = 0;
+      long maxMax = 0;
+      for (int z = config.minzoom(); z <= config.maxzoom(); z++) {
+        long totalCount = tilesByZoom[z].get();
+        long totalSize = totalTileSizesByZoom[z].get();
+        sumSize += totalSize;
+        sumCount += totalCount;
+        long maxSize = maxTileSizesByZoom[z].get();
+        maxMax = Math.max(maxMax, maxSize);
+        LOGGER.debug("z{} avg:{} max:{}",
+          z,
+          format.storage(totalCount == 0 ? 0 : (totalSize / totalCount), false),
+          format.storage(maxSize, false));
+      }
+      LOGGER.debug("all avg:{} max:{}",
+        format.storage(sumCount == 0 ? 0 : (sumSize / sumCount), false),
+        format.storage(maxMax, false));
+      LOGGER.debug(" # features: {}", format.integer(featuresProcessed.get()));
+      LOGGER.debug("    # tiles: {}", format.integer(this.tilesEmitted()));
     }
-    LOGGER.debug("all avg:{} max:{}",
-      format.storage(sumCount == 0 ? 0 : (sumSize / sumCount), false),
-      format.storage(maxMax, false));
-    LOGGER.debug(" # features: {}", format.integer(featuresProcessed.get()));
-    LOGGER.debug("    # tiles: {}", format.integer(this.tilesEmitted()));
   }
 
   private long tilesEmitted() {

planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileEncodingResult.java → planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java

@@ -1,4 +1,4 @@
-package com.onthegomap.planetiler.writer;
+package com.onthegomap.planetiler.archive;
 
 import com.onthegomap.planetiler.geo.TileCoord;
 import java.util.Arrays;

planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java → planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java

@@ -1,4 +1,4 @@
-package com.onthegomap.planetiler.writer;
+package com.onthegomap.planetiler.archive;
 
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.geo.TileOrder;
@@ -7,16 +7,22 @@
 import javax.annotation.concurrent.NotThreadSafe;
 
 /**
- * A TileArchive is a on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive
+ * Write API for an on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive
  * format.
+ * <p>
+ * See {@link ReadableTileArchive} for the read API.
  */
 @NotThreadSafe
-public interface TileArchive extends Closeable {
+public interface WriteableTileArchive extends Closeable {
+
   interface TileWriter extends Closeable {
+
     void write(TileEncodingResult encodingResult);
 
     // TODO: exists for compatibility reasons
-    void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult);
+    default void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) {
+      write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash()));
+    }
 
     @Override
     void close();
@@ -47,4 +53,6 @@ default void printStats() {}
    * disk.
    */
   void finish(PlanetilerConfig config);
+
+  // TODO update archive metadata
 }

planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java

@@ -1,13 +1,14 @@
 package com.onthegomap.planetiler.geo;
 
+import com.onthegomap.planetiler.archive.WriteableTileArchive;
 import java.util.function.IntFunction;
 import java.util.function.ToDoubleBiFunction;
 import java.util.function.ToIntFunction;
 
 /**
  * Controls the sort order of {@link com.onthegomap.planetiler.collection.FeatureGroup}, which determines the ordering
- * of {@link com.onthegomap.planetiler.writer.TileEncodingResult}s when written to
- * {@link com.onthegomap.planetiler.writer.TileArchive.TileWriter}.
+ * of {@link com.onthegomap.planetiler.archive.TileEncodingResult}s when written to
+ * {@link WriteableTileArchive.TileWriter}.
  */
 public enum TileOrder {
   TMS(TileCoord::encoded, TileCoord::decode, TileCoord::progressOnLevel),