diff --git a/docs/src/qsa-api/endpoints.md b/docs/src/qsa-api/endpoints.md index 1c4bdb7..78595f6 100644 --- a/docs/src/qsa-api/endpoints.md +++ b/docs/src/qsa-api/endpoints.md @@ -55,15 +55,15 @@ Several themes can be associated with a layer like in QGIS Desktop, but only the current one is used when the `STYLE` parameter in OGC web services is empty. -| Method | URL | Description | -|---------|--------------------------------------------------|-----------------------------------------------------------------------------------------------------| -| GET | `/api/projects/{project}/layers` | List layers in project | -| GET | `/api/projects/{project}/layers/{layer}` | List layer's metadata | -| GET | `/api/projects/{project}/layers/{layer}/map` | WMS `GetMap` result with default parameters | -| GET | `/api/projects/{project}/layers/{layer}/map/url` | WMS `GetMap` URL with default parameters | -| POST | `/api/projects/{project}/layers` | Add layer to project with `type` (`vector` or `raster`), `name`, `datasource` and `crs` (optional) | -| POST | `/api/projects/{project}/layers/{layer}/style` | Add/Update layer's style with `name` (style name) and `current` (`true` or `false`) | -| DELETE | `/api/projects/{project}/layers/{layer}` | Remove layer from project | +| Method | URL | Description | +|---------|--------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| +| GET | `/api/projects/{project}/layers` | List layers in project | +| GET | `/api/projects/{project}/layers/{layer}` | List layer's metadata | +| GET | `/api/projects/{project}/layers/{layer}/map` | WMS `GetMap` result with default parameters | +| GET | `/api/projects/{project}/layers/{layer}/map/url` | WMS `GetMap` URL with default parameters | +| POST | `/api/projects/{project}/layers` | Add layer to project with `type` (`vector` or `raster`), `name`, `datasource`, `overview` (build overview for rasters on S3) and `crs` (optional) | +| POST | `/api/projects/{project}/layers/{layer}/style` | Add/Update layer's style with `name` (style name) and `current` (`true` or `false`) | +| DELETE | `/api/projects/{project}/layers/{layer}` | Remove layer from project | Example: diff --git a/docs/src/sandbox/raster/layers.md b/docs/src/sandbox/raster/layers.md index a86a0bc..d93f753 100644 --- a/docs/src/sandbox/raster/layers.md +++ b/docs/src/sandbox/raster/layers.md @@ -12,7 +12,6 @@ $ curl "http://localhost:5000/api/projects/my_project/layers?schema=my_schema" \ -X POST \ -H 'Content-Type: application/json' \ -d '{ - "crs": 4326, "datasource":"/dem.tif", "name":"dem", "type":"raster" diff --git a/qsa-api/pyproject.toml b/qsa-api/pyproject.toml index 6bf7846..1b5d315 100644 --- a/qsa-api/pyproject.toml +++ b/qsa-api/pyproject.toml @@ -11,6 +11,7 @@ click = "8.1.7" pyyaml = "^6.0.1" jsonschema = "^4.21.1" +boto3 = "^1.34.123" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/qsa-api/qsa_api/__init__.py b/qsa-api/qsa_api/__init__.py index 1d4d763..37284b7 100644 --- a/qsa-api/qsa_api/__init__.py +++ b/qsa-api/qsa_api/__init__.py @@ -5,6 +5,7 @@ # avoid "Application path not initialized" message +os.environ["GDAL_PAM_PROXY_DIR"] = "/tmp" os.environ["QT_QPA_PLATFORM"] = "offscreen" QgsApplication.setPrefixPath("/usr", True) diff --git a/qsa-api/qsa_api/api/projects.py b/qsa-api/qsa_api/api/projects.py index f112180..823ee6a 100644 --- a/qsa-api/qsa_api/api/projects.py +++ b/qsa-api/qsa_api/api/projects.py @@ -270,6 +270,7 @@ def project_add_layer(name): "datasource": {"type": "string"}, "crs": {"type": "number"}, "type": {"type": "string"}, + "overview": {"type": "boolean"}, }, } @@ -287,8 +288,12 @@ def project_add_layer(name): if "crs" in data: crs = int(data["crs"]) + overview = False + if "overview" in data: + overview = data["overview"] + rc, err = project.add_layer( - data["datasource"], data["type"], data["name"], crs + data["datasource"], data["type"], data["name"], crs, overview ) if rc: return jsonify(rc), 201 diff --git a/qsa-api/qsa_api/api/symbology.py b/qsa-api/qsa_api/api/symbology.py index 4b9a9a8..d9d2b84 100644 --- a/qsa-api/qsa_api/api/symbology.py +++ b/qsa-api/qsa_api/api/symbology.py @@ -23,18 +23,18 @@ def symbology_symbols_line(): @symbology.get("/vector/polygon/single_symbol/fill/properties") def symbology_symbols_fill(): props = QgsSimpleFillSymbolLayer().properties() - props[ - "outline_style" - ] = "solid (no, solid, dash, dot, dash dot, dash dot dot)" + props["outline_style"] = ( + "solid (no, solid, dash, dot, dash dot, dash dot dot)" + ) return jsonify(props) @symbology.get("/vector/point/single_symbol/marker/properties") def symbology_symbols_marker(): props = QgsSimpleMarkerSymbolLayer().properties() - props[ - "outline_style" - ] = "solid (no, solid, dash, dot, dash dot, dash dot dot)" + props["outline_style"] = ( + "solid (no, solid, dash, dot, dash dot, dash dot dot)" + ) return jsonify(props) diff --git a/qsa-api/qsa_api/config.py b/qsa-api/qsa_api/config.py index 00698f9..4922c0e 100644 --- a/qsa-api/qsa_api/config.py +++ b/qsa-api/qsa_api/config.py @@ -1,6 +1,7 @@ # coding: utf8 import os +from pathlib import Path class QSAConfig: @@ -10,6 +11,10 @@ def is_valid(self) -> bool: return True return False + @property + def gdal_pam_proxy_dir(self) -> Path: + return Path(os.environ.get("GDAL_PAM_PROXY_DIR", "")) + @property def monitoring_port(self) -> int: return int(os.environ.get("QSA_QGISSERVER_MONITORING_PORT", "0")) diff --git a/qsa-api/qsa_api/mapproxy.py b/qsa-api/qsa_api/mapproxy.py index 7401869..f0a974b 100644 --- a/qsa-api/qsa_api/mapproxy.py +++ b/qsa-api/qsa_api/mapproxy.py @@ -31,10 +31,16 @@ def read(self) -> bool: with open(self._mapproxy_project, "r") as file: self.cfg = yaml.safe_load(file) except yaml.scanner.ScannerError as e: - return False, f"Failed to load MapProxy configuration file {self._mapproxy_project}" + return ( + False, + f"Failed to load MapProxy configuration file {self._mapproxy_project}", + ) if self.cfg is None: - return False, f"Failed to load MapProxy configuration file {self._mapproxy_project}" + return ( + False, + f"Failed to load MapProxy configuration file {self._mapproxy_project}", + ) return True, "" @@ -43,7 +49,9 @@ def clear_cache(self, layer_name: str) -> None: for d in cache_dir.glob(f"**/{layer_name}_cache_*"): shutil.rmtree(d) - def add_layer(self, name: str, bbox: list, srs: int, is_raster: bool) -> (bool, str): + def add_layer( + self, name: str, bbox: list, srs: int, is_raster: bool + ) -> (bool, str): if self.cfg is None: return False, "Invalid MapProxy configuration" @@ -55,10 +63,7 @@ def add_layer(self, name: str, bbox: list, srs: int, is_raster: bool) -> (bool, lyr = {"name": name, "title": name, "sources": [f"{name}_cache"]} self.cfg["layers"].append(lyr) - c = { - "grids": ["webmercator"], - "sources": [f"{name}_wms"] - } + c = {"grids": ["webmercator"], "sources": [f"{name}_wms"]} if is_raster: c["use_direct_from_level"] = 14 c["meta_size"] = [1, 1] diff --git a/qsa-api/qsa_api/project.py b/qsa-api/qsa_api/project.py index c716866..d2187ba 100644 --- a/qsa-api/qsa_api/project.py +++ b/qsa-api/qsa_api/project.py @@ -27,8 +27,8 @@ from .mapproxy import QSAMapProxy from .utils import StorageBackend, config -from .raster import RasterSymbologyRenderer from .vector import VectorSymbologyRenderer +from .raster import RasterSymbologyRenderer, RasterOverview RENDERER_TAG_NAME = "renderer-v2" # constant from core/symbology/renderer.h @@ -305,7 +305,12 @@ def remove(self) -> None: QSAMapProxy(self.name).remove() def add_layer( - self, datasource: str, layer_type: str, name: str, epsg_code: int + self, + datasource: str, + layer_type: str, + name: str, + epsg_code: int, + overview: bool, ) -> (bool, str): t = self._layer_type(layer_type) if t is None: @@ -319,6 +324,12 @@ def add_layer( lyr = QgsVectorLayer(datasource, name, provider) elif t == Qgis.LayerType.Raster: lyr = QgsRasterLayer(datasource, name, "gdal") + + ovr = RasterOverview(lyr) + if overview and not ovr.is_valid(): + rc, err = ovr.build() + if not rc: + return False, err else: return False, "Invalid layer type" @@ -353,14 +364,16 @@ def add_layer( ) if self._mapproxy_enabled: - epsg_code = int(lyr.crs().authid().split(':')[1]) + epsg_code = int(lyr.crs().authid().split(":")[1]) mp = QSAMapProxy(self.name) rc, err = mp.read() if not rc: return False, err - rc, err = mp.add_layer(name, bbox, epsg_code, t == Qgis.LayerType.Raster) + rc, err = mp.add_layer( + name, bbox, epsg_code, t == Qgis.LayerType.Raster + ) if not rc: return False, err diff --git a/qsa-api/qsa_api/raster/__init__.py b/qsa-api/qsa_api/raster/__init__.py index 848b7d2..155401f 100644 --- a/qsa-api/qsa_api/raster/__init__.py +++ b/qsa-api/qsa_api/raster/__init__.py @@ -1,3 +1,4 @@ # coding: utf8 +from .overview import RasterOverview from .renderer import RasterSymbologyRenderer diff --git a/qsa-api/qsa_api/raster/overview.py b/qsa-api/qsa_api/raster/overview.py new file mode 100644 index 0000000..9e34c43 --- /dev/null +++ b/qsa-api/qsa_api/raster/overview.py @@ -0,0 +1,55 @@ +# coding: utf8 + +import boto3 +from pathlib import Path + +from qgis.core import QgsRasterLayer, Qgis + +from ..config import QSAConfig + + +class RasterOverview: + def __init__(self, layer: QgsRasterLayer) -> None: + self.layer = layer + + def is_valid(self): + return self.layer.dataProvider().hasPyramids() + + def build(self) -> (bool, str): + ds = self.layer.source() + + # check if rasters stored on S3 + if "/vsis3" not in ds: + return False, "Building overviews is only supported for S3 rasters" + + # config overviews + levels = self.layer.dataProvider().buildPyramidList() + for idx, level in enumerate(levels): + levels[idx].setBuild(True) + + # build overviews + fmt = Qgis.RasterPyramidFormat.GeoTiff + err = self.layer.dataProvider().buildPyramids(levels, "NEAREST", fmt) + if err: + return False, f"Cannot build overview ({err})" + + # search ovr file in GDAL PAM directory + ovrfile = f"{Path(ds).name}.ovr" + ovrpath = next( + QSAConfig().gdal_pam_proxy_dir.glob(f"*{ovrfile}"), None + ) + if not ovrpath: + return False, f"Cannot find OVR file in GDAL_PAM_PROXY_DIR" + + # upload : not robust enough :/ + bucket = ds.split("/")[2] + subdir = Path(ds.split(f"/vsis3/{bucket}/")[1]).parent + s3 = boto3.resource("s3") + s3.Bucket(bucket).upload_file( + ovrpath.as_posix(), (subdir / ovrfile).as_posix() + ) + + # clean + ovrpath.unlink() + + return True, "" diff --git a/qsa-api/qsa_api/raster/renderer.py b/qsa-api/qsa_api/raster/renderer.py index 4e7e98c..e1f88f1 100644 --- a/qsa-api/qsa_api/raster/renderer.py +++ b/qsa-api/qsa_api/raster/renderer.py @@ -103,9 +103,13 @@ def style_to_json(path: Path) -> (dict, str): props = {} if renderer_type == RasterSymbologyRenderer.Type.SINGLE_BAND_GRAY: - props = RasterSymbologyRenderer._singlebandgray_properties(renderer) + props = RasterSymbologyRenderer._singlebandgray_properties( + renderer + ) elif renderer_type == RasterSymbologyRenderer.Type.MULTI_BAND_COLOR: - props = RasterSymbologyRenderer._multibandcolor_properties(renderer) + props = RasterSymbologyRenderer._multibandcolor_properties( + renderer + ) m["symbology"]["properties"] = props @@ -141,9 +145,7 @@ def _multibandcolor_properties(renderer) -> dict: # red band if renderer.redContrastEnhancement(): - red_ce = QgsContrastEnhancement( - renderer.redContrastEnhancement() - ) + red_ce = QgsContrastEnhancement(renderer.redContrastEnhancement()) props["red"]["min"] = red_ce.minimumValue() props["red"]["max"] = red_ce.maximumValue() @@ -167,11 +169,18 @@ def _multibandcolor_properties(renderer) -> dict: # ce alg = red_ce.contrastEnhancementAlgorithm() props["contrast_enhancement"]["algorithm"] = "NoEnhancement" - if alg == QgsContrastEnhancement.ContrastEnhancementAlgorithm.StretchToMinimumMaximum: - props["contrast_enhancement"]["algorithm"] = "StretchToMinimumMaximum" + if ( + alg + == QgsContrastEnhancement.ContrastEnhancementAlgorithm.StretchToMinimumMaximum + ): + props["contrast_enhancement"][ + "algorithm" + ] = "StretchToMinimumMaximum" else: # default behavior - props["contrast_enhancement"]["algorithm"] = "StretchToMinimumMaximum" + props["contrast_enhancement"][ + "algorithm" + ] = "StretchToMinimumMaximum" return props @@ -195,8 +204,13 @@ def _singlebandgray_properties(renderer) -> dict: alg = ce.contrastEnhancementAlgorithm() props["contrast_enhancement"]["algorithm"] = "NoEnhancement" - if alg == QgsContrastEnhancement.ContrastEnhancementAlgorithm.StretchToMinimumMaximum: - props["contrast_enhancement"]["algorithm"] = "StretchToMinimumMaximum" + if ( + alg + == QgsContrastEnhancement.ContrastEnhancementAlgorithm.StretchToMinimumMaximum + ): + props["contrast_enhancement"][ + "algorithm" + ] = "StretchToMinimumMaximum" limits = renderer.minMaxOrigin().limits() props["contrast_enhancement"]["limits_min_max"] = "UserDefined" diff --git a/qsa-api/tests/test_api_storage_filesystem.py b/qsa-api/tests/test_api_storage_filesystem.py index b40e17f..9d64e3d 100644 --- a/qsa-api/tests/test_api_storage_filesystem.py +++ b/qsa-api/tests/test_api_storage_filesystem.py @@ -89,16 +89,12 @@ def test_vector_symbology_marker(self): self.assertTrue("outline_style" in j) def test_vector_symbology_rendering(self): - p = self.app.get( - "/api/symbology/vector/rendering/properties" - ) + p = self.app.get("/api/symbology/vector/rendering/properties") j = p.get_json() self.assertTrue("opacity" in j) def test_raster_symbology_rendering(self): - p = self.app.get( - "/api/symbology/raster/rendering/properties" - ) + p = self.app.get("/api/symbology/raster/rendering/properties") j = p.get_json() self.assertTrue("gamma" in j) self.assertTrue("brightness" in j) @@ -106,16 +102,12 @@ def test_raster_symbology_rendering(self): self.assertTrue("saturation" in j) def test_raster_symbology_singlebandgray(self): - p = self.app.get( - "/api/symbology/raster/singlebandgray/properties" - ) + p = self.app.get("/api/symbology/raster/singlebandgray/properties") j = p.get_json() self.assertTrue("contrast_enhancement" in j) def test_raster_symbology_multibandcolor(self): - p = self.app.get( - "/api/symbology/raster/multibandcolor/properties" - ) + p = self.app.get("/api/symbology/raster/multibandcolor/properties") j = p.get_json() self.assertTrue("contrast_enhancement" in j) @@ -207,12 +199,23 @@ def test_raster_style(self): data["type"] = "raster" data["name"] = "style_multibandcolor" data["symbology"] = {"type": "multibandcolor"} - data["symbology"]["properties"] = {"red": {"band": 1}, "blue": {"band": 1}, "green": {"band": 1}} - data["rendering"] = {"brightness": 10, "gamma": 1.0, "contrast": 3, "saturation": 2} + data["symbology"]["properties"] = { + "red": {"band": 1}, + "blue": {"band": 1}, + "green": {"band": 1}, + } + data["rendering"] = { + "brightness": 10, + "gamma": 1.0, + "contrast": 3, + "saturation": 2, + } p = self.app.post(f"/api/projects/{TEST_PROJECT_0}/styles", data) self.assertEqual(p.status_code, 201) - p = self.app.get(f"/api/projects/{TEST_PROJECT_0}/styles/style_multibandcolor") + p = self.app.get( + f"/api/projects/{TEST_PROJECT_0}/styles/style_multibandcolor" + ) print(p.get_json()) self.assertTrue("rendering" in p.get_json()) self.assertTrue("symbology" in p.get_json()) diff --git a/qsa-api/tests/utils.py b/qsa-api/tests/utils.py index adb66e5..04625a5 100644 --- a/qsa-api/tests/utils.py +++ b/qsa-api/tests/utils.py @@ -44,9 +44,9 @@ def __init__(self, projects_dir, projects_psql_service=""): os.environ["QSA_QGISSERVER_URL"] = "http://qgisserver/ogc/" os.environ["QSA_QGISSERVER_PROJECTS_DIR"] = projects_dir if projects_psql_service: - os.environ[ - "QSA_QGISSERVER_PROJECTS_PSQL_SERVICE" - ] = projects_psql_service + os.environ["QSA_QGISSERVER_PROJECTS_PSQL_SERVICE"] = ( + projects_psql_service + ) self.app.application.config["CONFIG"] = QSAConfig() self.app.application.config["DEBUG"] = True