diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 57b8ffa8cb..f10f4fec17 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -8,7 +8,7 @@ Planetiler builds a map in 3 phases: intermediate files on disk 2. [Sort Features](#2-sort-features) by tile ID 3. [Emit Vector Tiles](#3-emit-vector-tiles) by iterating through sorted features to group by tile ID, encoding, and - writing to the output MBTiles file + writing to the output tile archive User-defined [profiles](#profiles) customize the behavior of each part of this pipeline. @@ -96,7 +96,7 @@ of the intermediate features using a worker thread per core: ## 3) Emit Vector Tiles -[MbtilesWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java) is the main driver. +[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) is the main driver. First, a single-threaded reader reads features from disk: - [ExternalMergeSort](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java) emits @@ -104,7 +104,7 @@ First, a single-threaded reader reads features from disk: - [FeatureGroup](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java) collects consecutive features in the same tile into a `TileFeatures` instance, dropping features in the same group over the grouping limit to limit point label density -- Then [MbtilesWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java) groups tiles +- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) groups tiles into variable-sized batches for workers to process (complex tiles get their own batch to ensure workers stay busy while the writer thread waits for finished tiles in order) @@ -116,9 +116,9 @@ Then, process tile batches in worker threads (default 1 per core): - gzip each encoded tile - Pass the batch of encoded vector tiles to the writer thread -Finally, a single-threaded writer writes encoded vector tiles to the output MBTiles file: +Finally, a single-threaded writer writes encoded vector tiles to the output archive format: -- Create the largest prepared statement supported by SQLite (999 parameters) +- For MBTiles, create the largest prepared statement supported by SQLite (999 parameters) - Iterate through finished vector tile batches until the prepared statement is full, flush to disk, then repeat - Then flush any remaining tiles at the end diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java index 1d6bf9c87b..46b3b8ae3c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java @@ -35,7 +35,7 @@ /** * A collection of utilities for merging features with the same attributes in a rendered tile from * {@link Profile#postProcessLayerFeatures(String, int, List)} immediately before a tile is written to the output - * mbtiles file. + * archive. *

* Unlike postgis-based solutions that have a full view of all features after they are loaded into the database, the * planetiler engine only sees a single input feature at a time while processing source features, then only has diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index d26fa284de..8c14fa81ef 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -452,30 +452,29 @@ public Planetiler setProfile(Function profileProvider) { } /** - * Sets the location of the output {@code .mbtiles} file to write rendered tiles to. Fails if the file already exists. + * Sets the location of the output archive to write rendered tiles to. Fails if the archive already exists. *

- * To override the location of the file, set {@code argument=newpath.mbtiles} in the arguments. + * To override the location of the file, set {@code argument=newpath} in the arguments. * * @param argument the argument key to check for an override to {@code fallback} * @param fallback the fallback value if {@code argument} is not set in arguments * @return this runner instance for chaining - * @see MbtilesWriter + * @see TileArchiveWriter */ public Planetiler setOutput(String argument, Path fallback) { - this.output = arguments.file(argument, "mbtiles output file", fallback); + this.output = arguments.file(argument, "output tile archive", fallback); return this; } /** - * Sets the location of the output {@code .mbtiles} file to write rendered tiles to. Overwrites file if it already - * exists. + * Sets the location of the output archive to write rendered tiles to. Overwrites file if it already exists. *

- * To override the location of the file, set {@code argument=newpath.mbtiles} in the arguments. + * To override the location of the file, set {@code argument=newpath} in the arguments. * * @param argument the argument key to check for an override to {@code fallback} * @param fallback the fallback value if {@code argument} is not set in arguments * @return this runner instance for chaining - * @see MbtilesWriter + * @see TileArchiveWriter */ public Planetiler overwriteOutput(String argument, Path fallback) { this.overwrite = true; @@ -484,7 +483,7 @@ public Planetiler overwriteOutput(String argument, Path fallback) { /** * Reads all elements from all sourced that have been added, generates map features according to the profile, and - * writes the rendered tiles to the output mbtiles file. + * writes the rendered tiles to the output archive. * * @throws IllegalArgumentException if expected inputs have not been provided * @throws Exception if an error occurs while processing @@ -537,7 +536,7 @@ public void run() throws Exception { } } LOGGER.info(" sort: Sort rendered features by tile ID"); - LOGGER.info(" mbtiles: Encode each tile and write to {}", output); + LOGGER.info(" archive: Encode each tile and write to {}", output); } // in case any temp files are left from a previous run... @@ -574,7 +573,7 @@ public void run() throws Exception { stats.monitorFile("nodes", nodeDbPath); stats.monitorFile("features", featureDbPath); stats.monitorFile("multipolygons", multipolygonPath); - stats.monitorFile("mbtiles", output); + stats.monitorFile("archive", output); for (Stage stage : stages) { stage.task.run(); @@ -638,7 +637,7 @@ private void checkDiskSpace() { readPhase.addDisk(featureDbPath, featureSize, "temporary feature storage"); writePhase.addDisk(featureDbPath, featureSize, "temporary feature storage"); // output only needed during write phase - writePhase.addDisk(output, outputSize, "mbtiles output"); + writePhase.addDisk(output, outputSize, "archive output"); // if the user opts to remove an input source after reading to free up additional space for the output... for (var input : inputPaths) { if (input.freeAfterReading()) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java index 2a30a6839a..8bb2977bbe 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java @@ -31,7 +31,7 @@ * (i.e. one per layer) and forwarding each element/event to the handlers that care about it. */ public interface Profile { - // TODO might want to break this apart into sub-interfaces that ForwardingProfile (and MbtilesMetadata) can use too + // TODO might want to break this apart into sub-interfaces that ForwardingProfile (and TileArchiveMetadata) can use too /** * Allows profile to extract any information it needs from a {@link OsmElement.Node} during the first pass through OSM diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java index d0a07685ef..189ccb8c09 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java @@ -239,7 +239,7 @@ private void writeTileFeatures(int zoom, long id, FeatureCollector.Feature featu // Store lines with extra precision (2^scale) in intermediate feature storage so that // rounding does not introduce artificial endpoint intersections and confuse line merge // post-processing. Features need to be "unscaled" in FeatureGroup after line merging, - // and before emitting to output mbtiles. + // and before emitting to the output archive. scale = Math.max(config.maxzoom(), 14) - zoom; // need 14 bits to represent tile coordinates (4096 * 2 for buffer * 2 for zigzag encoding) // so cap the scale factor to avoid overflowing 32-bit integer space diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java index 3ac15efc22..75e554055a 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java @@ -99,7 +99,7 @@ default Timers.Finishable startStage(String name, boolean log) { /** Records that an input element was processed and emitted some output features in {@code layer}. */ void processedElement(String elemType, String layer); - /** Records that a tile has been written to the mbtiles output where compressed size is {@code bytes}. */ + /** Records that a tile has been written to the archive output where compressed size is {@code bytes}. */ void wroteTile(int zoom, int bytes); /** Returns the timers for all stages started with {@link #startStage(String)}. */ diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 93e9ec4071..788402eebe 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -264,11 +264,11 @@ void testMetadataButNoPoints() throws Exception { void testOverrideMetadata() throws Exception { var results = runWithReaderFeatures( Map.of( - "mbtiles_name", "mbtiles_name", - "mbtiles_description", "mbtiles_description", - "mbtiles_attribution", "mbtiles_attribution", - "mbtiles_version", "mbtiles_version", - "mbtiles_type", "mbtiles_type" + "tileset_name", "override_name", + "tileset_description", "override_description", + "tileset_attribution", "override_attribution", + "tileset_version", "override_version", + "tileset_type", "override_type" ), List.of(), (sourceFeature, features) -> { @@ -276,11 +276,11 @@ void testOverrideMetadata() throws Exception { ); assertEquals(Map.of(), results.tiles); assertSubmap(Map.of( - "name", "mbtiles_name", - "description", "mbtiles_description", - "attribution", "mbtiles_attribution", - "version", "mbtiles_version", - "type", "mbtiles_type" + "name", "override_name", + "description", "override_description", + "attribution", "override_attribution", + "version", "override_version", + "type", "override_type" ), results.metadata); }