Merge pull request #46 from Qwant/ver_3_13

Update to OpenMapTiles V3_13

.env

@@ -4,7 +4,7 @@
 TILESET_FILE=openmaptiles.yaml
 
 # Use 3-part patch version to ignore patch updates, e.g. 5.0.0
-TOOLS_VERSION=5.3
+TOOLS_VERSION=6.1
 
 # Make sure these values are in sync with the ones in .env-postgres file
 PGDATABASE=openmaptiles
@@ -18,16 +18,17 @@ PGPORT=5432
 # By default, the Makefile will use the content of data/$(area).bbox file if it exists.
 BBOX=-180.0,-85.0511,180.0,85.0511
 
-# Which zooms to generate in   make generate-tiles
+# Which zooms to generate with   make generate-tiles-pg
 MIN_ZOOM=0
 MAX_ZOOM=7
 
+# `MID_ZOOM` setting only works with `make generate-tiles-pg` command.  Make sure MID_ZOOM < MAX_ZOOM.
+# See https://github.com/openmaptiles/openmaptiles-tools/pull/383
+# MID_ZOOM=11
+
 # Use  true  (case sensitive) to allow data updates
 DIFF_MODE=false
 
-# Hide some output from Mapnik tile generation for clarity
-FILTER_MAPNIK_OUTPUT=1
-
 # Some area data like openstreetmap.fr can contain invalid references
 # that must be cleaned up before using it for borders -- set it to true.
 BORDERS_CLEANUP=false
@@ -49,9 +50,5 @@ MAX_PARALLEL_PSQL=5
 # Number of parallel threads to use when generating vector map tiles
 COPY_CONCURRENCY=10
 
-# Variables for generate tiles using PGquery
+# Variables for generate tiles using tilelive-pgquery
 PGHOSTS_LIST=
-# Uncomment when used PGquery
-#EXPORT_DIR=/import
-NO_GZIP=1
-USE_KEY_COLUMN=1

.github/workflows/integrity.yml

@@ -0,0 +1,71 @@
+# Workflow to run basic integrity checks on OMT`s new Pull Requests and commits pushed into OMT repo
+
+name: OpenMapTiles Integrity CI
+
+on:
+  push:
+    branches: [ master, master-tools ]
+  pull_request:
+
+jobs:
+
+  integrity_test:
+    name: Run integrity test
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout the changes
+        uses: actions/checkout@v2
+
+      - name: Run quickstart for a small area
+        env:
+          area: monaco
+          QUIET: 1
+        run: |
+          echo MIN_ZOOM=0 >> .env
+          echo MAX_ZOOM=14 >> .env
+          ./quickstart.sh $area
+
+      - name: Save quickstart.log
+        uses: actions/upload-artifact@v2
+        with:
+          name: quickstart.log
+          path: quickstart.log
+
+      - name: Test etldoc images
+        run: |
+          export TEST_MODE=yes
+          make generate-devdoc
+
+      - name: Run quickstart and update in DIFF mode
+        env:
+          area: europe/monaco
+          QUIET: 1
+        run: |
+          echo MIN_ZOOM=0 >> .env
+          echo MAX_ZOOM=14 >> .env
+          echo DIFF_MODE=true >> .env
+          # Cleanup
+          rm -fr data build cache
+          # Create data/$area.repl.json
+          make download-geofabrik area=$area
+          # Download 2+ month old data
+          export old_date=$(date --date="$(date +%Y-%m-15) -2 month" +'%y%m01')
+          echo Downloading $old_date extract of $area
+          docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "wget -O data/$area.osm.pbf http://download.geofabrik.de/$area-$old_date.osm.pbf"
+          # Initial import and tile generation
+          ./quickstart.sh $area
+          sleep 2
+          echo Downloading updates
+          # Loop to recover from potential "ERROR 429: Too Many Requests"
+          docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "
+            while ! osmupdate --keep-tempfiles --base-url=$(sed -n 's/ *\"replication_url\": //p' data/$area.repl.json) data/$area.osm.pbf data/changes.osc.gz ; do 
+              sleep 2;
+              echo Sleeping...;
+              sleep 630;
+            done"
+          echo Downloading updates completed
+          echo Importing updates
+          make import-diff
+          echo Generating new tiles
+          make generate-tiles-pg