Add option to build overview if necessary

docs/src/qsa-api/endpoints.md

@@ -55,15 +55,15 @@ Several themes can be associated with a layer like in QGIS Desktop, but only
 the current one is used when the `STYLE` parameter in OGC web services is
 empty.
 
-| Method  |                      URL                         |         Description                                                                                 |
-|---------|--------------------------------------------------|-----------------------------------------------------------------------------------------------------|
-| GET     | `/api/projects/{project}/layers`                 | List layers in project                                                                              |
-| GET     | `/api/projects/{project}/layers/{layer}`         | List layer's metadata                                                                               |
-| GET     | `/api/projects/{project}/layers/{layer}/map`     | WMS `GetMap` result with default parameters                                                         |
-| GET     | `/api/projects/{project}/layers/{layer}/map/url` | WMS `GetMap` URL with default parameters                                                            |
-| POST    | `/api/projects/{project}/layers`                 | Add layer to project with `type` (`vector` or `raster`), `name`, `datasource` and `crs` (optional)  |
-| POST    | `/api/projects/{project}/layers/{layer}/style`   | Add/Update layer's style with `name` (style name) and `current` (`true` or `false`)                 |
-| DELETE  | `/api/projects/{project}/layers/{layer}`         | Remove layer from project                                                                           |
+| Method  |                      URL                         |         Description                                                                                                                                |
+|---------|--------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|
+| GET     | `/api/projects/{project}/layers`                 | List layers in project                                                                                                                             |
+| GET     | `/api/projects/{project}/layers/{layer}`         | List layer's metadata                                                                                                                              |
+| GET     | `/api/projects/{project}/layers/{layer}/map`     | WMS `GetMap` result with default parameters                                                                                                        |
+| GET     | `/api/projects/{project}/layers/{layer}/map/url` | WMS `GetMap` URL with default parameters                                                                                                           |
+| POST    | `/api/projects/{project}/layers`                 | Add layer to project with `type` (`vector` or `raster`), `name`, `datasource`, `overview` (build overview for rasters on S3) and `crs` (optional)  |
+| POST    | `/api/projects/{project}/layers/{layer}/style`   | Add/Update layer's style with `name` (style name) and `current` (`true` or `false`)                                                                |
+| DELETE  | `/api/projects/{project}/layers/{layer}`         | Remove layer from project                                                                                                                          |
 
 Example:
 

docs/src/sandbox/raster/layers.md

@@ -12,7 +12,6 @@ $ curl "http://localhost:5000/api/projects/my_project/layers?schema=my_schema" \
   -X POST \
   -H 'Content-Type: application/json' \
   -d '{
-    "crs": 4326,
     "datasource":"/dem.tif",
     "name":"dem",
     "type":"raster"

qsa-api/pyproject.toml

@@ -11,6 +11,7 @@ click = "8.1.7"
 pyyaml = "^6.0.1"
 jsonschema = "^4.21.1"
 
+boto3 = "^1.34.123"
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

qsa-api/qsa_api/__init__.py

@@ -5,6 +5,7 @@
 
 
 # avoid "Application path not initialized" message
+os.environ["GDAL_PAM_PROXY_DIR"] = "/tmp"
 os.environ["QT_QPA_PLATFORM"] = "offscreen"
 
 QgsApplication.setPrefixPath("/usr", True)

qsa-api/qsa_api/api/projects.py

@@ -270,6 +270,7 @@ def project_add_layer(name):
             "datasource": {"type": "string"},
             "crs": {"type": "number"},
             "type": {"type": "string"},
+            "overview": {"type": "boolean"},
         },
     }
 
@@ -287,8 +288,12 @@ def project_add_layer(name):
         if "crs" in data:
             crs = int(data["crs"])
 
+        overview = False
+        if "overview" in data:
+            overview = data["overview"]
+
         rc, err = project.add_layer(
-            data["datasource"], data["type"], data["name"], crs
+            data["datasource"], data["type"], data["name"], crs, overview
         )
         if rc:
             return jsonify(rc), 201

qsa-api/qsa_api/api/symbology.py

@@ -23,18 +23,18 @@ def symbology_symbols_line():
 @symbology.get("/vector/polygon/single_symbol/fill/properties")
 def symbology_symbols_fill():
     props = QgsSimpleFillSymbolLayer().properties()
-    props[
-        "outline_style"
-    ] = "solid (no, solid, dash, dot, dash dot, dash dot dot)"
+    props["outline_style"] = (
+        "solid (no, solid, dash, dot, dash dot, dash dot dot)"
+    )
     return jsonify(props)
 
 
 @symbology.get("/vector/point/single_symbol/marker/properties")
 def symbology_symbols_marker():
     props = QgsSimpleMarkerSymbolLayer().properties()
-    props[
-        "outline_style"
-    ] = "solid (no, solid, dash, dot, dash dot, dash dot dot)"
+    props["outline_style"] = (
+        "solid (no, solid, dash, dot, dash dot, dash dot dot)"
+    )
     return jsonify(props)
 
 

qsa-api/qsa_api/config.py

@@ -1,6 +1,7 @@
 # coding: utf8
 
 import os
+from pathlib import Path
 
 
 class QSAConfig:
@@ -10,6 +11,10 @@ def is_valid(self) -> bool:
             return True
         return False
 
+    @property
+    def gdal_pam_proxy_dir(self) -> Path:
+        return Path(os.environ.get("GDAL_PAM_PROXY_DIR", ""))
+
     @property
     def monitoring_port(self) -> int:
         return int(os.environ.get("QSA_QGISSERVER_MONITORING_PORT", "0"))

qsa-api/qsa_api/mapproxy.py

@@ -31,10 +31,16 @@ def read(self) -> bool:
             with open(self._mapproxy_project, "r") as file:
                 self.cfg = yaml.safe_load(file)
         except yaml.scanner.ScannerError as e:
-            return False, f"Failed to load MapProxy configuration file {self._mapproxy_project}"
+            return (
+                False,
+                f"Failed to load MapProxy configuration file {self._mapproxy_project}",
+            )
 
         if self.cfg is None:
-            return False, f"Failed to load MapProxy configuration file {self._mapproxy_project}"
+            return (
+                False,
+                f"Failed to load MapProxy configuration file {self._mapproxy_project}",
+            )
 
         return True, ""
 
@@ -43,7 +49,9 @@ def clear_cache(self, layer_name: str) -> None:
         for d in cache_dir.glob(f"**/{layer_name}_cache_*"):
             shutil.rmtree(d)
 
-    def add_layer(self, name: str, bbox: list, srs: int, is_raster: bool) -> (bool, str):
+    def add_layer(
+        self, name: str, bbox: list, srs: int, is_raster: bool
+    ) -> (bool, str):
         if self.cfg is None:
             return False, "Invalid MapProxy configuration"
 
@@ -55,10 +63,7 @@ def add_layer(self, name: str, bbox: list, srs: int, is_raster: bool) -> (bool,
         lyr = {"name": name, "title": name, "sources": [f"{name}_cache"]}
         self.cfg["layers"].append(lyr)
 
-        c = {
-            "grids": ["webmercator"],
-            "sources": [f"{name}_wms"]
-        }
+        c = {"grids": ["webmercator"], "sources": [f"{name}_wms"]}
         if is_raster:
             c["use_direct_from_level"] = 14
             c["meta_size"] = [1, 1]

qsa-api/qsa_api/project.py

@@ -27,8 +27,8 @@
 
 from .mapproxy import QSAMapProxy
 from .utils import StorageBackend, config
-from .raster import RasterSymbologyRenderer
 from .vector import VectorSymbologyRenderer
+from .raster import RasterSymbologyRenderer, RasterOverview
 
 
 RENDERER_TAG_NAME = "renderer-v2"  # constant from core/symbology/renderer.h
@@ -305,7 +305,12 @@ def remove(self) -> None:
             QSAMapProxy(self.name).remove()
 
     def add_layer(
-        self, datasource: str, layer_type: str, name: str, epsg_code: int
+        self,
+        datasource: str,
+        layer_type: str,
+        name: str,
+        epsg_code: int,
+        overview: bool,
     ) -> (bool, str):
         t = self._layer_type(layer_type)
         if t is None:
@@ -319,6 +324,12 @@ def add_layer(
             lyr = QgsVectorLayer(datasource, name, provider)
         elif t == Qgis.LayerType.Raster:
             lyr = QgsRasterLayer(datasource, name, "gdal")
+
+            ovr = RasterOverview(lyr)
+            if overview and not ovr.is_valid():
+                rc, err = ovr.build()
+                if not rc:
+                    return False, err
         else:
             return False, "Invalid layer type"
 
@@ -353,14 +364,16 @@ def add_layer(
         )
 
         if self._mapproxy_enabled:
-            epsg_code = int(lyr.crs().authid().split(':')[1])
+            epsg_code = int(lyr.crs().authid().split(":")[1])
 
             mp = QSAMapProxy(self.name)
             rc, err = mp.read()
             if not rc:
                 return False, err
 
-            rc, err = mp.add_layer(name, bbox, epsg_code, t == Qgis.LayerType.Raster)
+            rc, err = mp.add_layer(
+                name, bbox, epsg_code, t == Qgis.LayerType.Raster
+            )
             if not rc:
                 return False, err
 

qsa-api/qsa_api/raster/__init__.py

@@ -1,3 +1,4 @@
 # coding: utf8
 
+from .overview import RasterOverview
 from .renderer import RasterSymbologyRenderer

qsa-api/qsa_api/raster/overview.py

@@ -0,0 +1,55 @@
+# coding: utf8
+
+import boto3
+from pathlib import Path
+
+from qgis.core import QgsRasterLayer, Qgis
+
+from ..config import QSAConfig
+
+
+class RasterOverview:
+    def __init__(self, layer: QgsRasterLayer) -> None:
+        self.layer = layer
+
+    def is_valid(self):
+        return self.layer.dataProvider().hasPyramids()
+
+    def build(self) -> (bool, str):
+        ds = self.layer.source()
+
+        # check if rasters stored on S3
+        if "/vsis3" not in ds:
+            return False, "Building overviews is only supported for S3 rasters"
+
+        # config overviews
+        levels = self.layer.dataProvider().buildPyramidList()
+        for idx, level in enumerate(levels):
+            levels[idx].setBuild(True)
+
+        # build overviews
+        fmt = Qgis.RasterPyramidFormat.GeoTiff
+        err = self.layer.dataProvider().buildPyramids(levels, "NEAREST", fmt)
+        if err:
+            return False, f"Cannot build overview ({err})"
+
+        # search ovr file in GDAL PAM directory
+        ovrfile = f"{Path(ds).name}.ovr"
+        ovrpath = next(
+            QSAConfig().gdal_pam_proxy_dir.glob(f"*{ovrfile}"), None
+        )
+        if not ovrpath:
+            return False, f"Cannot find OVR file in GDAL_PAM_PROXY_DIR"
+
+        # upload : not robust enough :/
+        bucket = ds.split("/")[2]
+        subdir = Path(ds.split(f"/vsis3/{bucket}/")[1]).parent
+        s3 = boto3.resource("s3")
+        s3.Bucket(bucket).upload_file(
+            ovrpath.as_posix(), (subdir / ovrfile).as_posix()
+        )
+
+        # clean
+        ovrpath.unlink()
+
+        return True, ""

qsa-api/qsa_api/raster/renderer.py

@@ -103,9 +103,13 @@ def style_to_json(path: Path) -> (dict, str):
 
         props = {}
         if renderer_type == RasterSymbologyRenderer.Type.SINGLE_BAND_GRAY:
-            props = RasterSymbologyRenderer._singlebandgray_properties(renderer)
+            props = RasterSymbologyRenderer._singlebandgray_properties(
+                renderer
+            )
         elif renderer_type == RasterSymbologyRenderer.Type.MULTI_BAND_COLOR:
-            props = RasterSymbologyRenderer._multibandcolor_properties(renderer)
+            props = RasterSymbologyRenderer._multibandcolor_properties(
+                renderer
+            )
 
         m["symbology"]["properties"] = props
 
@@ -141,9 +145,7 @@ def _multibandcolor_properties(renderer) -> dict:
 
         # red band
         if renderer.redContrastEnhancement():
-            red_ce = QgsContrastEnhancement(
-                renderer.redContrastEnhancement()
-            )
+            red_ce = QgsContrastEnhancement(renderer.redContrastEnhancement())
 
             props["red"]["min"] = red_ce.minimumValue()
             props["red"]["max"] = red_ce.maximumValue()
@@ -167,11 +169,18 @@ def _multibandcolor_properties(renderer) -> dict:
             # ce
             alg = red_ce.contrastEnhancementAlgorithm()
             props["contrast_enhancement"]["algorithm"] = "NoEnhancement"
-            if alg == QgsContrastEnhancement.ContrastEnhancementAlgorithm.StretchToMinimumMaximum:
-                props["contrast_enhancement"]["algorithm"] = "StretchToMinimumMaximum"
+            if (
+                alg
+                == QgsContrastEnhancement.ContrastEnhancementAlgorithm.StretchToMinimumMaximum
+            ):
+                props["contrast_enhancement"][
+                    "algorithm"
+                ] = "StretchToMinimumMaximum"
         else:
             # default behavior
-            props["contrast_enhancement"]["algorithm"] = "StretchToMinimumMaximum"
+            props["contrast_enhancement"][
+                "algorithm"
+            ] = "StretchToMinimumMaximum"
 
         return props
 
@@ -195,8 +204,13 @@ def _singlebandgray_properties(renderer) -> dict:
 
         alg = ce.contrastEnhancementAlgorithm()
         props["contrast_enhancement"]["algorithm"] = "NoEnhancement"
-        if alg == QgsContrastEnhancement.ContrastEnhancementAlgorithm.StretchToMinimumMaximum:
-            props["contrast_enhancement"]["algorithm"] = "StretchToMinimumMaximum"
+        if (
+            alg
+            == QgsContrastEnhancement.ContrastEnhancementAlgorithm.StretchToMinimumMaximum
+        ):
+            props["contrast_enhancement"][
+                "algorithm"
+            ] = "StretchToMinimumMaximum"
 
         limits = renderer.minMaxOrigin().limits()
         props["contrast_enhancement"]["limits_min_max"] = "UserDefined"