diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 57b8ffa8cb..f10f4fec17 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -8,7 +8,7 @@ Planetiler builds a map in 3 phases: intermediate files on disk 2. [Sort Features](#2-sort-features) by tile ID 3. [Emit Vector Tiles](#3-emit-vector-tiles) by iterating through sorted features to group by tile ID, encoding, and - writing to the output MBTiles file + writing to the output tile archive User-defined [profiles](#profiles) customize the behavior of each part of this pipeline. @@ -96,7 +96,7 @@ of the intermediate features using a worker thread per core: ## 3) Emit Vector Tiles -[MbtilesWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java) is the main driver. +[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) is the main driver. First, a single-threaded reader reads features from disk: - [ExternalMergeSort](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java) emits @@ -104,7 +104,7 @@ First, a single-threaded reader reads features from disk: - [FeatureGroup](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java) collects consecutive features in the same tile into a `TileFeatures` instance, dropping features in the same group over the grouping limit to limit point label density -- Then [MbtilesWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java) groups tiles +- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) groups tiles into variable-sized batches for workers to process (complex tiles get their own batch to ensure workers stay busy while the writer thread waits for finished tiles in order) @@ -116,9 +116,9 @@ Then, process tile batches in worker threads (default 1 per core): - gzip each encoded tile - Pass the batch of encoded vector tiles to the writer thread -Finally, a single-threaded writer writes encoded vector tiles to the output MBTiles file: +Finally, a single-threaded writer writes encoded vector tiles to the output archive format: -- Create the largest prepared statement supported by SQLite (999 parameters) +- For MBTiles, create the largest prepared statement supported by SQLite (999 parameters) - Iterate through finished vector tile batches until the prepared statement is full, flush to disk, then repeat - Then flush any remaining tiles at the end diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java index 1d6bf9c87b..46b3b8ae3c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java @@ -35,7 +35,7 @@ /** * A collection of utilities for merging features with the same attributes in a rendered tile from * {@link Profile#postProcessLayerFeatures(String, int, List)} immediately before a tile is written to the output - * mbtiles file. + * archive. *
* Unlike postgis-based solutions that have a full view of all features after they are loaded into the database, the
* planetiler engine only sees a single input feature at a time while processing source features, then only has
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java
index d26fa284de..8c14fa81ef 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java
@@ -452,30 +452,29 @@ public Planetiler setProfile(Function
- * To override the location of the file, set {@code argument=newpath.mbtiles} in the arguments.
+ * To override the location of the file, set {@code argument=newpath} in the arguments.
*
* @param argument the argument key to check for an override to {@code fallback}
* @param fallback the fallback value if {@code argument} is not set in arguments
* @return this runner instance for chaining
- * @see MbtilesWriter
+ * @see TileArchiveWriter
*/
public Planetiler setOutput(String argument, Path fallback) {
- this.output = arguments.file(argument, "mbtiles output file", fallback);
+ this.output = arguments.file(argument, "output tile archive", fallback);
return this;
}
/**
- * Sets the location of the output {@code .mbtiles} file to write rendered tiles to. Overwrites file if it already
- * exists.
+ * Sets the location of the output archive to write rendered tiles to. Overwrites file if it already exists.
*
- * To override the location of the file, set {@code argument=newpath.mbtiles} in the arguments.
+ * To override the location of the file, set {@code argument=newpath} in the arguments.
*
* @param argument the argument key to check for an override to {@code fallback}
* @param fallback the fallback value if {@code argument} is not set in arguments
* @return this runner instance for chaining
- * @see MbtilesWriter
+ * @see TileArchiveWriter
*/
public Planetiler overwriteOutput(String argument, Path fallback) {
this.overwrite = true;
@@ -484,7 +483,7 @@ public Planetiler overwriteOutput(String argument, Path fallback) {
/**
* Reads all elements from all sourced that have been added, generates map features according to the profile, and
- * writes the rendered tiles to the output mbtiles file.
+ * writes the rendered tiles to the output archive.
*
* @throws IllegalArgumentException if expected inputs have not been provided
* @throws Exception if an error occurs while processing
@@ -537,7 +536,7 @@ public void run() throws Exception {
}
}
LOGGER.info(" sort: Sort rendered features by tile ID");
- LOGGER.info(" mbtiles: Encode each tile and write to {}", output);
+ LOGGER.info(" archive: Encode each tile and write to {}", output);
}
// in case any temp files are left from a previous run...
@@ -574,7 +573,7 @@ public void run() throws Exception {
stats.monitorFile("nodes", nodeDbPath);
stats.monitorFile("features", featureDbPath);
stats.monitorFile("multipolygons", multipolygonPath);
- stats.monitorFile("mbtiles", output);
+ stats.monitorFile("archive", output);
for (Stage stage : stages) {
stage.task.run();
@@ -638,7 +637,7 @@ private void checkDiskSpace() {
readPhase.addDisk(featureDbPath, featureSize, "temporary feature storage");
writePhase.addDisk(featureDbPath, featureSize, "temporary feature storage");
// output only needed during write phase
- writePhase.addDisk(output, outputSize, "mbtiles output");
+ writePhase.addDisk(output, outputSize, "archive output");
// if the user opts to remove an input source after reading to free up additional space for the output...
for (var input : inputPaths) {
if (input.freeAfterReading()) {
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java
index 2a30a6839a..8bb2977bbe 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java
@@ -31,7 +31,7 @@
* (i.e. one per layer) and forwarding each element/event to the handlers that care about it.
*/
public interface Profile {
- // TODO might want to break this apart into sub-interfaces that ForwardingProfile (and MbtilesMetadata) can use too
+ // TODO might want to break this apart into sub-interfaces that ForwardingProfile (and TileArchiveMetadata) can use too
/**
* Allows profile to extract any information it needs from a {@link OsmElement.Node} during the first pass through OSM
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java
index d0a07685ef..189ccb8c09 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java
@@ -239,7 +239,7 @@ private void writeTileFeatures(int zoom, long id, FeatureCollector.Feature featu
// Store lines with extra precision (2^scale) in intermediate feature storage so that
// rounding does not introduce artificial endpoint intersections and confuse line merge
// post-processing. Features need to be "unscaled" in FeatureGroup after line merging,
- // and before emitting to output mbtiles.
+ // and before emitting to the output archive.
scale = Math.max(config.maxzoom(), 14) - zoom;
// need 14 bits to represent tile coordinates (4096 * 2 for buffer * 2 for zigzag encoding)
// so cap the scale factor to avoid overflowing 32-bit integer space
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java
index 3ac15efc22..75e554055a 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java
@@ -99,7 +99,7 @@ default Timers.Finishable startStage(String name, boolean log) {
/** Records that an input element was processed and emitted some output features in {@code layer}. */
void processedElement(String elemType, String layer);
- /** Records that a tile has been written to the mbtiles output where compressed size is {@code bytes}. */
+ /** Records that a tile has been written to the archive output where compressed size is {@code bytes}. */
void wroteTile(int zoom, int bytes);
/** Returns the timers for all stages started with {@link #startStage(String)}. */
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
index 93e9ec4071..788402eebe 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
@@ -264,11 +264,11 @@ void testMetadataButNoPoints() throws Exception {
void testOverrideMetadata() throws Exception {
var results = runWithReaderFeatures(
Map.of(
- "mbtiles_name", "mbtiles_name",
- "mbtiles_description", "mbtiles_description",
- "mbtiles_attribution", "mbtiles_attribution",
- "mbtiles_version", "mbtiles_version",
- "mbtiles_type", "mbtiles_type"
+ "tileset_name", "override_name",
+ "tileset_description", "override_description",
+ "tileset_attribution", "override_attribution",
+ "tileset_version", "override_version",
+ "tileset_type", "override_type"
),
List.of(),
(sourceFeature, features) -> {
@@ -276,11 +276,11 @@ void testOverrideMetadata() throws Exception {
);
assertEquals(Map.of(), results.tiles);
assertSubmap(Map.of(
- "name", "mbtiles_name",
- "description", "mbtiles_description",
- "attribution", "mbtiles_attribution",
- "version", "mbtiles_version",
- "type", "mbtiles_type"
+ "name", "override_name",
+ "description", "override_description",
+ "attribution", "override_attribution",
+ "version", "override_version",
+ "type", "override_type"
), results.metadata);
}