From bfafce104e407fd4452384a2c155436c5f94790e Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sat, 28 Jan 2023 06:51:28 -0500 Subject: [PATCH 1/7] convert get all tile coords to iterator --- .../planetiler/mbtiles/Mbtiles.java | 98 +++++++++++++------ .../onthegomap/planetiler/mbtiles/Verify.java | 25 ++--- .../planetiler/util/CloseableIterator.java | 17 ++++ .../planetiler/writer/TileArchive.java | 17 +++- .../planetiler/mbtiles/MbtilesTest.java | 3 +- 5 files changed, 117 insertions(+), 43 deletions(-) create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java index 5a31492254..aa54ee7dc6 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java @@ -11,6 +11,8 @@ import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.geo.TileOrder; +import com.onthegomap.planetiler.reader.FileFormatException; +import com.onthegomap.planetiler.util.CloseableIterator; import com.onthegomap.planetiler.util.Format; import com.onthegomap.planetiler.util.LayerStats; import com.onthegomap.planetiler.writer.TileArchive; @@ -31,6 +33,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.OptionalLong; import java.util.TreeMap; @@ -88,13 +91,8 @@ public final class Mbtiles implements TileArchive { } private final Connection connection; - private PreparedStatement getTileStatement = null; private final boolean compactDb; - - @Override - public TileOrder tileOrder() { - return TileOrder.TMS; - } + private PreparedStatement getTileStatement = null; private Mbtiles(Connection connection, boolean compactDb) { this.connection = connection; @@ -152,6 +150,11 @@ public static Mbtiles newReadOnlyDatabase(Path path) { } } + @Override + public TileOrder tileOrder() { + return TileOrder.TMS; + } + @Override public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) { if (config.skipIndexCreation()) { @@ -356,10 +359,7 @@ private PreparedStatement getTileStatement() { return getTileStatement; } - public byte[] getTile(TileCoord coord) { - return getTile(coord.x(), coord.y(), coord.z()); - } - + @Override public byte[] getTile(int x, int y, int z) { try { PreparedStatement stmt = getTileStatement(); @@ -374,22 +374,9 @@ public byte[] getTile(int x, int y, int z) { } } - public List getAllTileCoords() { - List result = new ArrayList<>(); - try (Statement statement = connection.createStatement()) { - ResultSet rs = statement.executeQuery( - "select %s, %s, %s, %s from %s".formatted(TILES_COL_Z, TILES_COL_X, TILES_COL_Y, TILES_COL_DATA, TILES_TABLE) - ); - while (rs.next()) { - int z = rs.getInt(TILES_COL_Z); - int rawy = rs.getInt(TILES_COL_Y); - int x = rs.getInt(TILES_COL_X); - result.add(TileCoord.ofXYZ(x, (1 << z) - 1 - rawy, z)); - } - } catch (SQLException throwables) { - throw new IllegalStateException("Could not get all tile coordinates", throwables); - } - return result; + @Override + public CloseableIterator getAllTileCoords() { + return new TileCoordIterator(); } public Connection connection() { @@ -492,14 +479,69 @@ public boolean equals(Object obj) { if (this == obj) { return true; } - if (!(obj instanceof TileDataEntry)) { + if (!(obj instanceof TileDataEntry other)) { return false; } - TileDataEntry other = (TileDataEntry) obj; return Arrays.equals(tileData, other.tileData) && tileDataId == other.tileDataId; } } + /** Iterates through tile coordinates one at a time without materializing the entire list in memory. */ + private class TileCoordIterator implements CloseableIterator { + private final Statement statement; + private final ResultSet rs; + private boolean hasNext; + + private TileCoordIterator() { + try { + this.statement = connection.createStatement(); + this.rs = statement.executeQuery( + "select %s, %s, %s, %s from %s".formatted(TILES_COL_Z, TILES_COL_X, TILES_COL_Y, TILES_COL_DATA, TILES_TABLE) + ); + hasNext = rs.next(); + if (!hasNext) { + close(); + } + } catch (SQLException e) { + throw new FileFormatException("Could not read tile coordinates from mbtiles file", e); + } + } + + @Override + public void close() { + try { + statement.close(); + } catch (SQLException e) { + throw new IllegalStateException("Could not close mbtiles file", e); + } + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public TileCoord next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + try { + int z = rs.getInt(TILES_COL_Z); + int rawy = rs.getInt(TILES_COL_Y); + int x = rs.getInt(TILES_COL_X); + var result = TileCoord.ofXYZ(x, (1 << z) - 1 - rawy, z); + hasNext = rs.next(); + if (!hasNext) { + close(); + } + return result; + } catch (SQLException e) { + throw new IllegalStateException("Could not read close mbtiles file", e); + } + } + } + private abstract class BatchedTableWriterBase implements AutoCloseable { private static final int MAX_PARAMETERS_IN_PREPARED_STATEMENT = 999; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java index 093ceea2b1..8fa73fd630 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java @@ -59,16 +59,19 @@ public static void main(String[] args) throws IOException { public static int getNumFeatures(Mbtiles db, String layer, int zoom, Map attrs, Envelope envelope, Class clazz) throws GeometryException { int num = 0; - for (var tileCoord : db.getAllTileCoords()) { - Envelope tileEnv = new Envelope(); - tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMinX(), envelope.getMinY())); - tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMaxX(), envelope.getMaxY())); - if (tileCoord.z() == zoom) { - byte[] data = db.getTile(tileCoord); - for (var feature : decode(data)) { - if (layer.equals(feature.layer()) && feature.attrs().entrySet().containsAll(attrs.entrySet())) { - Geometry geometry = feature.geometry().decode(); - num += getGeometryCounts(geometry, clazz); + try (var tileCoords = db.getAllTileCoords()) { + while (tileCoords.hasNext()) { + var tileCoord = tileCoords.next(); + Envelope tileEnv = new Envelope(); + tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMinX(), envelope.getMinY())); + tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMaxX(), envelope.getMaxY())); + if (tileCoord.z() == zoom) { + byte[] data = db.getTile(tileCoord); + for (var feature : decode(data)) { + if (layer.equals(feature.layer()) && feature.attrs().entrySet().containsAll(attrs.entrySet())) { + Geometry geometry = feature.geometry().decode(); + num += getGeometryCounts(geometry, clazz); + } } } } @@ -187,7 +190,7 @@ public void failIfErrors() { private void checkBasicStructure() { check("contains name attribute", () -> mbtiles.metadata().getAll().containsKey("name")); - check("contains at least one tile", () -> !mbtiles.getAllTileCoords().isEmpty()); + check("contains at least one tile", () -> mbtiles.getAllTileCoords().stream().findAny().isPresent()); checkWithMessage("all tiles are valid", () -> { List invalidTiles = mbtiles.getAllTileCoords().stream() .flatMap(coord -> checkValidity(coord, decode(mbtiles.getTile(coord))).stream()) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java new file mode 100644 index 0000000000..151ca9c14a --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java @@ -0,0 +1,17 @@ +package com.onthegomap.planetiler.util; + +import java.io.Closeable; +import java.util.Iterator; +import java.util.Spliterators; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public interface CloseableIterator extends Closeable, Iterator { + + @Override + void close(); + + default Stream stream() { + return StreamSupport.stream(Spliterators.spliteratorUnknownSize(this, 0), false).onClose(this::close); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java index 30126489cd..6950fb91f0 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java @@ -1,14 +1,18 @@ package com.onthegomap.planetiler.writer; import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.geo.TileOrder; +import com.onthegomap.planetiler.util.CloseableIterator; import com.onthegomap.planetiler.util.LayerStats; import java.io.Closeable; import javax.annotation.concurrent.NotThreadSafe; /** - * A TileArchive is a on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive - * format. + * A TileArchive is an on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based + * archive format. + *

+ * Implementing classes should be thread-safe for reading, but can assume writing happens from a single thread. */ @NotThreadSafe public interface TileArchive extends Closeable { @@ -47,4 +51,13 @@ default void printStats() {} * disk. */ void finish(PlanetilerConfig config); + + + default byte[] getTile(TileCoord coord) { + return getTile(coord.x(), coord.y(), coord.z()); + } + + byte[] getTile(int x, int y, int z); + + CloseableIterator getAllTileCoords(); } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java index 42d5e8a2c9..8d20194df2 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java @@ -13,7 +13,6 @@ import java.math.RoundingMode; import java.sql.SQLException; import java.sql.Statement; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.OptionalLong; @@ -69,7 +68,7 @@ void testWriteTiles(int howMany, boolean skipIndexCreation, boolean optimize, bo assertEquals(howMany, all.size()); assertEquals(expected, all); assertEquals(expected.stream().map(Mbtiles.TileEntry::tile).collect(Collectors.toSet()), - new HashSet<>(db.getAllTileCoords())); + db.getAllTileCoords().stream().collect(Collectors.toSet())); for (var expectedEntry : expected) { var tile = expectedEntry.tile(); byte[] data = db.getTile(tile.x(), tile.y(), tile.z()); From 1940e024b8bec8e4d473bd0529f29388d9c6f669 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sat, 28 Jan 2023 07:03:45 -0500 Subject: [PATCH 2/7] typo --- .../main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java index aa54ee7dc6..6731b9d669 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java @@ -537,7 +537,7 @@ public TileCoord next() { } return result; } catch (SQLException e) { - throw new IllegalStateException("Could not read close mbtiles file", e); + throw new IllegalStateException("Could not read mbtiles file", e); } } } From f1781db71875ec032c3b7cc6cd5648225c5ee64d Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sat, 28 Jan 2023 07:08:06 -0500 Subject: [PATCH 3/7] pull up backwards-compatible method --- .../com/onthegomap/planetiler/mbtiles/Mbtiles.java | 12 ------------ .../onthegomap/planetiler/writer/TileArchive.java | 4 +++- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java index 6731b9d669..55c7260c58 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java @@ -709,12 +709,6 @@ public void write(TileEncodingResult encodingResult) { tableWriter.write(new TileEntry(encodingResult.coord(), encodingResult.tileData())); } - // TODO: exists for compatibility purposes - @Override - public void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) { - tableWriter.write(new TileEntry(encodingResult.coord(), encodingResult.tileData())); - } - @Override public void close() { tableWriter.close(); @@ -756,12 +750,6 @@ public void write(TileEncodingResult encodingResult) { batchedTileShallowTableWriter.write(new TileShallowEntry(encodingResult.coord(), tileDataId)); } - // TODO: exists for compatibility purposes - @Override - public void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) { - write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash())); - } - @Override public void close() { batchedTileShallowTableWriter.close(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java index 6950fb91f0..129f821fb2 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java @@ -20,7 +20,9 @@ interface TileWriter extends Closeable { void write(TileEncodingResult encodingResult); // TODO: exists for compatibility reasons - void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult); + default void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) { + write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash())); + } @Override void close(); From f66154a6b069709a0df1db55ec93186e59c56ce5 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sat, 28 Jan 2023 14:03:46 -0500 Subject: [PATCH 4/7] a little safer --- .../java/com/onthegomap/planetiler/mbtiles/Mbtiles.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java index 55c7260c58..85bcd91072 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java @@ -490,7 +490,7 @@ public boolean equals(Object obj) { private class TileCoordIterator implements CloseableIterator { private final Statement statement; private final ResultSet rs; - private boolean hasNext; + private boolean hasNext = false; private TileCoordIterator() { try { @@ -499,11 +499,12 @@ private TileCoordIterator() { "select %s, %s, %s, %s from %s".formatted(TILES_COL_Z, TILES_COL_X, TILES_COL_Y, TILES_COL_DATA, TILES_TABLE) ); hasNext = rs.next(); + } catch (SQLException e) { + throw new FileFormatException("Could not read tile coordinates from mbtiles file", e); + } finally { if (!hasNext) { close(); } - } catch (SQLException e) { - throw new FileFormatException("Could not read tile coordinates from mbtiles file", e); } } From 4159f2444c50273526eacc7a0dcbf3213dde17fc Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sat, 4 Feb 2023 14:57:47 -0500 Subject: [PATCH 5/7] Extract readable tile archive interface --- ARCHITECTURE.md | 6 ++- .../benchmarks/BenchmarkMbtilesWriter.java | 4 +- .../com/onthegomap/planetiler/Planetiler.java | 8 ++-- .../archive/ReadableTileArchive.java | 44 +++++++++++++++++++ .../TileArchiveMetadata.java | 2 +- .../TileArchiveWriter.java | 10 ++--- .../TileEncodingResult.java | 2 +- .../WriteableTileArchive.java} | 23 ++++------ .../onthegomap/planetiler/geo/TileOrder.java | 5 ++- .../planetiler/mbtiles/Mbtiles.java | 13 +++--- .../planetiler/util/LayerStats.java | 6 +-- .../planetiler/PlanetilerTests.java | 4 +- .../collection/FeatureGroupTest.java | 2 +- .../planetiler/mbtiles/MbtilesTest.java | 2 +- .../planetiler/mbtiles/VerifyTest.java | 2 +- .../examples/ToiletsOverlayLowLevelApi.java | 4 +- 16 files changed, 89 insertions(+), 48 deletions(-) create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java rename planetiler-core/src/main/java/com/onthegomap/planetiler/{writer => archive}/TileArchiveMetadata.java (98%) rename planetiler-core/src/main/java/com/onthegomap/planetiler/{writer => archive}/TileArchiveWriter.java (98%) rename planetiler-core/src/main/java/com/onthegomap/planetiler/{writer => archive}/TileEncodingResult.java (96%) rename planetiler-core/src/main/java/com/onthegomap/planetiler/{writer/TileArchive.java => archive/WriteableTileArchive.java} (70%) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f10f4fec17..ddd7ba9f68 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -96,7 +96,8 @@ of the intermediate features using a worker thread per core: ## 3) Emit Vector Tiles -[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) is the main driver. +[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java) is the main +driver. First, a single-threaded reader reads features from disk: - [ExternalMergeSort](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java) emits @@ -104,7 +105,8 @@ First, a single-threaded reader reads features from disk: - [FeatureGroup](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java) collects consecutive features in the same tile into a `TileFeatures` instance, dropping features in the same group over the grouping limit to limit point label density -- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) groups tiles +- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java) + groups tiles into variable-sized batches for workers to process (complex tiles get their own batch to ensure workers stay busy while the writer thread waits for finished tiles in order) diff --git a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java index b53777b127..47379b9f7f 100644 --- a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java +++ b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java @@ -1,12 +1,12 @@ package com.onthegomap.planetiler.benchmarks; import com.google.common.base.Stopwatch; +import com.onthegomap.planetiler.archive.TileEncodingResult; +import com.onthegomap.planetiler.archive.WriteableTileArchive.TileWriter; import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.mbtiles.Mbtiles; -import com.onthegomap.planetiler.writer.TileArchive.TileWriter; -import com.onthegomap.planetiler.writer.TileEncodingResult; import java.io.File; import java.io.IOException; import java.nio.file.Files; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index cd30e956ca..718e0c1255 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -1,5 +1,8 @@ package com.onthegomap.planetiler; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileArchiveWriter; +import com.onthegomap.planetiler.archive.WriteableTileArchive; import com.onthegomap.planetiler.collection.FeatureGroup; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.LongLongMultimap; @@ -26,9 +29,6 @@ import com.onthegomap.planetiler.util.Translations; import com.onthegomap.planetiler.util.Wikidata; import com.onthegomap.planetiler.worker.RunnableThatThrows; -import com.onthegomap.planetiler.writer.TileArchive; -import com.onthegomap.planetiler.writer.TileArchiveMetadata; -import com.onthegomap.planetiler.writer.TileArchiveWriter; import java.io.IOException; import java.nio.file.FileSystem; import java.nio.file.Files; @@ -658,7 +658,7 @@ public void run() throws Exception { bounds.addFallbackProvider(new OsmNodeBoundsProvider(osmInputFile, config, stats)); } - try (TileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) { + try (WriteableTileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) { featureGroup = FeatureGroup.newDiskBackedFeatureGroup(archive.tileOrder(), featureDbPath, profile, config, stats); stats.monitorFile("nodes", nodeDbPath); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java new file mode 100644 index 0000000000..48a59ba224 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java @@ -0,0 +1,44 @@ +package com.onthegomap.planetiler.archive; + +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.util.CloseableIterator; +import java.io.Closeable; + +/** + * Read API for on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive + * format. + *

+ * See {@link WriteableTileArchive} for the write API. + */ +public interface ReadableTileArchive extends Closeable { + + /** Returns the raw tile data associated with the tile at {@code coord}. */ + default byte[] getTile(TileCoord coord) { + return getTile(coord.x(), coord.y(), coord.z()); + } + + /** Returns the raw tile data associated with the tile at coordinate {@code x, y, z}. */ + byte[] getTile(int x, int y, int z); + + /** + * Returns an iterator over the coordinates of tiles in this archive. + *

+ * The order should respect {@link WriteableTileArchive#tileOrder()} of the corresponding writer. + *

+ * Clients should be sure to close the iterator after iterating through it, for example: + * + * 

+   * {@code
+   * try (var iter = archive.getAllTileCoord()) {
+   *   while (iter.hasNext()) {
+   *     var coord = iter.next();
+   *     ...
+   *   }
+   * }
+   * }
+   *
+ */ + CloseableIterator getAllTileCoords(); + + // TODO access archive metadata +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveMetadata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java similarity index 98% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveMetadata.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java index 271ce5a937..053dc3c1bb 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveMetadata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java @@ -1,4 +1,4 @@ -package com.onthegomap.planetiler.writer; +package com.onthegomap.planetiler.archive; import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.config.Arguments; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java similarity index 98% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java index fad9f06b86..c23fe2818a 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java @@ -1,4 +1,4 @@ -package com.onthegomap.planetiler.writer; +package com.onthegomap.planetiler.archive; import static com.onthegomap.planetiler.util.Gzip.gzip; import static com.onthegomap.planetiler.worker.Worker.joinFutures; @@ -40,7 +40,7 @@ /** * Final stage of the map generation process that encodes vector tiles using {@link VectorTile} and writes them to a - * {@link TileArchive}. + * {@link WriteableTileArchive}. */ public class TileArchiveWriter { @@ -49,7 +49,7 @@ public class TileArchiveWriter { private static final long MAX_TILES_PER_BATCH = 1_000; private final Counter.Readable featuresProcessed; private final Counter memoizedTiles; - private final TileArchive archive; + private final WriteableTileArchive archive; private final PlanetilerConfig config; private final Stats stats; private final LayerStats layerStats; @@ -60,7 +60,7 @@ public class TileArchiveWriter { private final AtomicReference lastTileWritten = new AtomicReference<>(); private final TileArchiveMetadata tileArchiveMetadata; - private TileArchiveWriter(Iterable inputTiles, TileArchive archive, + private TileArchiveWriter(Iterable inputTiles, WriteableTileArchive archive, PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, Stats stats, LayerStats layerStats) { this.inputTiles = inputTiles; @@ -88,7 +88,7 @@ private TileArchiveWriter(Iterable inputTiles, TileAr } /** Reads all {@code features}, encodes them in parallel, and writes to {@code output}. */ - public static void writeOutput(FeatureGroup features, TileArchive output, DiskBacked fileSize, + public static void writeOutput(FeatureGroup features, WriteableTileArchive output, DiskBacked fileSize, TileArchiveMetadata tileArchiveMetadata, PlanetilerConfig config, Stats stats) { var timer = stats.startStage("archive"); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileEncodingResult.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java similarity index 96% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileEncodingResult.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java index 39b9a08de2..5df7c0a4c4 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileEncodingResult.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java @@ -1,4 +1,4 @@ -package com.onthegomap.planetiler.writer; +package com.onthegomap.planetiler.archive; import com.onthegomap.planetiler.geo.TileCoord; import java.util.Arrays; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java similarity index 70% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java index 129f821fb2..7449c0b1ae 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java @@ -1,22 +1,22 @@ -package com.onthegomap.planetiler.writer; +package com.onthegomap.planetiler.archive; import com.onthegomap.planetiler.config.PlanetilerConfig; -import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.geo.TileOrder; -import com.onthegomap.planetiler.util.CloseableIterator; import com.onthegomap.planetiler.util.LayerStats; import java.io.Closeable; import javax.annotation.concurrent.NotThreadSafe; /** - * A TileArchive is an on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based - * archive format. + * Write API for an on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive + * format. *

- * Implementing classes should be thread-safe for reading, but can assume writing happens from a single thread. + * See {@link ReadableTileArchive} for the read API. */ @NotThreadSafe -public interface TileArchive extends Closeable { +public interface WriteableTileArchive extends Closeable { + interface TileWriter extends Closeable { + void write(TileEncodingResult encodingResult); // TODO: exists for compatibility reasons @@ -54,12 +54,5 @@ default void printStats() {} */ void finish(PlanetilerConfig config); - - default byte[] getTile(TileCoord coord) { - return getTile(coord.x(), coord.y(), coord.z()); - } - - byte[] getTile(int x, int y, int z); - - CloseableIterator getAllTileCoords(); + // TODO update archive metadata } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java index be34388f85..70658d271b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java @@ -1,13 +1,14 @@ package com.onthegomap.planetiler.geo; +import com.onthegomap.planetiler.archive.WriteableTileArchive; import java.util.function.IntFunction; import java.util.function.ToDoubleBiFunction; import java.util.function.ToIntFunction; /** * Controls the sort order of {@link com.onthegomap.planetiler.collection.FeatureGroup}, which determines the ordering - * of {@link com.onthegomap.planetiler.writer.TileEncodingResult}s when written to - * {@link com.onthegomap.planetiler.writer.TileArchive.TileWriter}. + * of {@link com.onthegomap.planetiler.archive.TileEncodingResult}s when written to + * {@link WriteableTileArchive.TileWriter}. */ public enum TileOrder { TMS(TileCoord::encoded, TileCoord::decode, TileCoord::progressOnLevel), diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java index 85bcd91072..a2cc353cf5 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java @@ -7,6 +7,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.onthegomap.planetiler.archive.ReadableTileArchive; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileEncodingResult; +import com.onthegomap.planetiler.archive.WriteableTileArchive; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.TileCoord; @@ -15,9 +19,6 @@ import com.onthegomap.planetiler.util.CloseableIterator; import com.onthegomap.planetiler.util.Format; import com.onthegomap.planetiler.util.LayerStats; -import com.onthegomap.planetiler.writer.TileArchive; -import com.onthegomap.planetiler.writer.TileArchiveMetadata; -import com.onthegomap.planetiler.writer.TileEncodingResult; import java.io.IOException; import java.nio.file.Path; import java.sql.Connection; @@ -51,7 +52,7 @@ * * @see MBTiles Specification */ -public final class Mbtiles implements TileArchive { +public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive { // https://www.sqlite.org/src/artifact?ci=trunk&filename=magic.txt private static final int MBTILES_APPLICATION_ID = 0x4d504258; @@ -327,7 +328,7 @@ public Mbtiles vacuumAnalyze() { } /** Returns a writer that queues up inserts into the tile database(s) into large batches before executing them. */ - public TileArchive.TileWriter newTileWriter() { + public WriteableTileArchive.TileWriter newTileWriter() { if (compactDb) { return new BatchedCompactTileWriter(); } else { @@ -336,7 +337,7 @@ public TileArchive.TileWriter newTileWriter() { } // TODO: exists for compatibility purposes - public TileArchive.TileWriter newBatchedTileWriter() { + public WriteableTileArchive.TileWriter newBatchedTileWriter() { return newTileWriter(); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerStats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerStats.java index 8a9bbde601..4d3ec988d7 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerStats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerStats.java @@ -1,9 +1,9 @@ package com.onthegomap.planetiler.util; import com.fasterxml.jackson.annotation.JsonProperty; +import com.onthegomap.planetiler.archive.WriteableTileArchive; import com.onthegomap.planetiler.mbtiles.Mbtiles; import com.onthegomap.planetiler.render.RenderedFeature; -import com.onthegomap.planetiler.writer.TileArchive; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -18,8 +18,8 @@ /** * Tracks the feature attributes and zoom range of each layer to populate the archive output metadata. *

- * Matches the MBTiles spec for {@code vector_layers}, but can be reused by other {@link TileArchive} formats. To - * minimize overhead of stat collection, each updating thread should call {@link #handlerForThread()} first to get a + * Matches the MBTiles spec for {@code vector_layers}, but can be reused by other {@link WriteableTileArchive} formats. + * To minimize overhead of stat collection, each updating thread should call {@link #handlerForThread()} first to get a * thread-local handler that can update stats without contention. *

* diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 32201a93dc..d8709a0dcc 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -3,6 +3,8 @@ import static com.onthegomap.planetiler.TestUtils.*; import static org.junit.jupiter.api.Assertions.*; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileArchiveWriter; import com.onthegomap.planetiler.collection.FeatureGroup; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.LongLongMultimap; @@ -23,8 +25,6 @@ import com.onthegomap.planetiler.reader.osm.OsmRelationInfo; import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.util.BuildInfo; -import com.onthegomap.planetiler.writer.TileArchiveMetadata; -import com.onthegomap.planetiler.writer.TileArchiveWriter; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java index c4607b49ec..240e76a4d1 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java @@ -10,6 +10,7 @@ import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.VectorTile; +import com.onthegomap.planetiler.archive.TileArchiveWriter; import com.onthegomap.planetiler.geo.GeometryType; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.geo.TileOrder; @@ -17,7 +18,6 @@ import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.util.CloseableConsumer; import com.onthegomap.planetiler.util.Gzip; -import com.onthegomap.planetiler.writer.TileArchiveWriter; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java index 8d20194df2..696947b353 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java @@ -5,10 +5,10 @@ import com.google.common.math.IntMath; import com.onthegomap.planetiler.TestUtils; +import com.onthegomap.planetiler.archive.TileEncodingResult; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.util.LayerStats; -import com.onthegomap.planetiler.writer.TileEncodingResult; import java.io.IOException; import java.math.RoundingMode; import java.sql.SQLException; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java index cd0c607b56..9324add2c8 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java @@ -7,8 +7,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.onthegomap.planetiler.VectorTile; +import com.onthegomap.planetiler.archive.TileEncodingResult; import com.onthegomap.planetiler.geo.TileCoord; -import com.onthegomap.planetiler.writer.TileEncodingResult; import java.io.IOException; import java.util.List; import java.util.Map; diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java index 60fb6cd613..1c5a8e618b 100644 --- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java +++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java @@ -2,6 +2,8 @@ import com.onthegomap.planetiler.Planetiler; import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileArchiveWriter; import com.onthegomap.planetiler.collection.FeatureGroup; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.LongLongMultimap; @@ -13,8 +15,6 @@ import com.onthegomap.planetiler.reader.osm.OsmReader; import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.util.FileUtils; -import com.onthegomap.planetiler.writer.TileArchiveMetadata; -import com.onthegomap.planetiler.writer.TileArchiveWriter; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; From 25ff3512fe8bd98841785be63247312bab76b752 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sat, 4 Feb 2023 15:26:48 -0500 Subject: [PATCH 6/7] invoke log messages conditionally --- .../planetiler/archive/TileArchiveWriter.java | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java index c23fe2818a..ad95e1c101 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java @@ -335,28 +335,30 @@ private void tileWriter(Iterable tileBatches) throws ExecutionExcepti } private void printTileStats() { - Format format = Format.defaultInstance(); - LOGGER.debug("Tile stats:"); - long sumSize = 0; - long sumCount = 0; - long maxMax = 0; - for (int z = config.minzoom(); z <= config.maxzoom(); z++) { - long totalCount = tilesByZoom[z].get(); - long totalSize = totalTileSizesByZoom[z].get(); - sumSize += totalSize; - sumCount += totalCount; - long maxSize = maxTileSizesByZoom[z].get(); - maxMax = Math.max(maxMax, maxSize); - LOGGER.debug("z{} avg:{} max:{}", - z, - format.storage(totalCount == 0 ? 0 : (totalSize / totalCount), false), - format.storage(maxSize, false)); + if (LOGGER.isDebugEnabled()) { + Format format = Format.defaultInstance(); + LOGGER.debug("Tile stats:"); + long sumSize = 0; + long sumCount = 0; + long maxMax = 0; + for (int z = config.minzoom(); z <= config.maxzoom(); z++) { + long totalCount = tilesByZoom[z].get(); + long totalSize = totalTileSizesByZoom[z].get(); + sumSize += totalSize; + sumCount += totalCount; + long maxSize = maxTileSizesByZoom[z].get(); + maxMax = Math.max(maxMax, maxSize); + LOGGER.debug("z{} avg:{} max:{}", + z, + format.storage(totalCount == 0 ? 0 : (totalSize / totalCount), false), + format.storage(maxSize, false)); + } + LOGGER.debug("all avg:{} max:{}", + format.storage(sumCount == 0 ? 0 : (sumSize / sumCount), false), + format.storage(maxMax, false)); + LOGGER.debug(" # features: {}", format.integer(featuresProcessed.get())); + LOGGER.debug(" # tiles: {}", format.integer(this.tilesEmitted())); } - LOGGER.debug("all avg:{} max:{}", - format.storage(sumCount == 0 ? 0 : (sumSize / sumCount), false), - format.storage(maxMax, false)); - LOGGER.debug(" # features: {}", format.integer(featuresProcessed.get())); - LOGGER.debug(" # tiles: {}", format.integer(this.tilesEmitted())); } private long tilesEmitted() { From b8b9884cb8874637a3cdf7bc26f5fd08414fc2b0 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Sun, 5 Feb 2023 13:48:15 -0500 Subject: [PATCH 7/7] typo --- .../com/onthegomap/planetiler/archive/ReadableTileArchive.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java index 48a59ba224..2bc9960cf1 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java @@ -29,7 +29,7 @@ default byte[] getTile(TileCoord coord) { * * 
    * {@code
-   * try (var iter = archive.getAllTileCoord()) {
+   * try (var iter = archive.getAllTileCoords()) {
    *   while (iter.hasNext()) {
    *     var coord = iter.next();
    *     ...