From b82b3b0f2fc80a38badca84de1532b47b75abcb4 Mon Sep 17 00:00:00 2001 From: index-git Date: Fri, 20 Jan 2023 06:37:31 +0100 Subject: [PATCH 1/3] Remove redundant gets from directory --- src/layman/common/prime_db_schema/publications.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/layman/common/prime_db_schema/publications.py b/src/layman/common/prime_db_schema/publications.py index 03a6a1df2..0795f0e04 100644 --- a/src/layman/common/prime_db_schema/publications.py +++ b/src/layman/common/prime_db_schema/publications.py @@ -234,10 +234,10 @@ def get_publication_infos_with_metainfo(workspace_name=None, pub_type=None, styl 'image_mosaic': image_mosaic, 'updated_at': updated_at, '_table_uri': TableUri( - db_uri_str=external_table_uri.get('db_uri_str'), - schema=external_table_uri.get('schema'), - table=external_table_uri.get('table'), - geo_column=external_table_uri.get('geo_column'), + db_uri_str=external_table_uri['db_uri_str'], + schema=external_table_uri['schema'], + table=external_table_uri['table'], + geo_column=external_table_uri['geo_column'], ) if external_table_uri else None, 'native_bounding_box': [xmin, ymin, xmax, ymax], 'native_crs': db_util.get_crs(srid) if srid else None, From f65c929ef9764a5ade82af5ad11921b40382c214 Mon Sep 17 00:00:00 2001 From: index-git Date: Fri, 20 Jan 2023 06:33:37 +0100 Subject: [PATCH 2/3] Skip some tasks for vector layers from external table --- src/layman/layer/db/tasks.py | 3 +++ src/layman/layer/filesystem/tasks.py | 8 +++++++- src/layman/layer/rest_workspace_layers.py | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/layman/layer/db/tasks.py b/src/layman/layer/db/tasks.py index 26c18358e..3ad2b9b58 100644 --- a/src/layman/layer/db/tasks.py +++ b/src/layman/layer/db/tasks.py @@ -24,11 +24,14 @@ def refresh_table( workspace, layername, crs_id=None, + is_external_table=False ): db.ensure_workspace(workspace) if self.is_aborted(): raise AbortedException + if is_external_table: + return publ_info = layman_util.get_publication_info(workspace, LAYER_TYPE, layername, context={'keys': ['file']}) file_type = publ_info['file']['file_type'] if file_type == settings.FILE_TYPE_RASTER: diff --git a/src/layman/layer/filesystem/tasks.py b/src/layman/layer/filesystem/tasks.py index e7369d1a5..173262797 100644 --- a/src/layman/layer/filesystem/tasks.py +++ b/src/layman/layer/filesystem/tasks.py @@ -83,7 +83,11 @@ def refresh_input_chunk(self, workspace, layername, check_crs=True, overview_res bind=True, base=celery_app.AbortableTask ) -def refresh_gdal(self, workspace, layername, crs_id=None, overview_resampling=None, name_normalized_tif_by_layer=True): +def refresh_gdal(self, workspace, layername, + crs_id=None, + overview_resampling=None, + name_normalized_tif_by_layer=True, + is_external_table=False): def finish_gdal_process(process): if self.is_aborted(): logger.info(f'terminating GDAL process workspace.layer={workspace}.{layername}') @@ -99,6 +103,8 @@ def finish_gdal_process(process): if self.is_aborted(): raise AbortedException + if is_external_table: + return layer_info = layman_util.get_publication_info(workspace, LAYER_TYPE, layername, context={'keys': ['file']}) file_type = layer_info['file']['file_type'] if file_type != settings.FILE_TYPE_RASTER: diff --git a/src/layman/layer/rest_workspace_layers.py b/src/layman/layer/rest_workspace_layers.py index c2d22fdc7..d677c045c 100644 --- a/src/layman/layer/rest_workspace_layers.py +++ b/src/layman/layer/rest_workspace_layers.py @@ -164,6 +164,7 @@ def post(workspace): 'name_input_file_by_layer': name_input_file_by_layer, 'enable_more_main_files': enable_more_main_files, 'external_table_uri': external_table_uri, + 'is_external_table': bool(external_table_uri), } rest_common.setup_post_access_rights(request.form, task_options, actor_name) From 81a00680b8e4379e66566b4c74325303958e1a23 Mon Sep 17 00:00:00 2001 From: index-git Date: Thu, 19 Jan 2023 17:08:12 +0100 Subject: [PATCH 3/3] Store correct bbox and crs for layer with external table --- src/layman/layer/db/__init__.py | 22 ++++++++++-------- src/layman/layer/prime_db_schema/tasks.py | 23 ++++++++++++++----- .../layer_external_db/external_db_test.py | 13 ++++++++++- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/src/layman/layer/db/__init__.py b/src/layman/layer/db/__init__.py index 2ed8a03f8..2c77e2bf8 100644 --- a/src/layman/layer/db/__init__.py +++ b/src/layman/layer/db/__init__.py @@ -3,6 +3,7 @@ import os import logging import subprocess +from psycopg2 import sql from db import util as db_util, PG_CONN from layman.common.language import get_languages_iso639_2 @@ -494,26 +495,27 @@ def ensure_attributes(attribute_tuples): return missing_attributes -def get_bbox(schema, table_name, conn_cur=None): - query = f''' - with tmp as (select ST_Extent(l.wkb_geometry) as bbox - from {schema}.{table_name} l +def get_bbox(schema, table_name, conn_cur=None, column='wkb_geometry'): + query = sql.SQL(''' + with tmp as (select ST_Extent(l.{column}) as bbox + from {table} l ) select st_xmin(bbox), st_ymin(bbox), st_xmax(bbox), st_ymax(bbox) from tmp - ''' + ''').format( + table=sql.Identifier(schema, table_name), + column=sql.Identifier(column), + ) result = db_util.run_query(query, conn_cur=conn_cur)[0] return result -def get_crs(schema, table_name, conn_cur=None): - query = f''' - select Find_SRID('{schema}', '{table_name}', 'wkb_geometry'); - ''' - srid = db_util.run_query(query, conn_cur=conn_cur)[0][0] +def get_crs(schema, table_name, conn_cur=None, column='wkb_geometry'): + query = 'select Find_SRID(%s, %s, %s);' + srid = db_util.run_query(query, (schema, table_name, column), conn_cur=conn_cur)[0][0] crs = db_util.get_crs(srid) return crs diff --git a/src/layman/layer/prime_db_schema/tasks.py b/src/layman/layer/prime_db_schema/tasks.py index 5d004b955..9acbd2469 100644 --- a/src/layman/layer/prime_db_schema/tasks.py +++ b/src/layman/layer/prime_db_schema/tasks.py @@ -1,5 +1,6 @@ from celery.utils.log import get_task_logger +from db import util as db_util from layman.celery import AbortedException from layman.common import empty_method_returns_true from layman import celery_app, util as layman_util, settings @@ -26,12 +27,23 @@ def refresh_file_data( if self.is_aborted(): raise AbortedException - publ_info = layman_util.get_publication_info(username, LAYER_TYPE, layername, context={'keys': ['file', 'db_table']}) - file_type = publ_info['file']['file_type'] + publ_info = layman_util.get_publication_info(username, LAYER_TYPE, layername, context={'keys': ['file_type', 'table_uri']}) + if publ_info['_file_type'] == settings.FILE_TYPE_UNKNOWN: + publ_info_file = layman_util.get_publication_info(username, LAYER_TYPE, layername, context={'keys': ['file']}) + file_type = publ_info_file['file']['file_type'] + set_file_type(username, LAYER_TYPE, layername, file_type, ) + else: + file_type = publ_info['_file_type'] + if file_type == settings.FILE_TYPE_VECTOR: - table_name = publ_info['db_table']['name'] - bbox = db_get_bbox(username, table_name) - crs = db_get_crs(username, table_name) + if not publ_info.get('_table_uri'): + # We have to set file type into publications table before asking for table_uri, + # because for compressed files sent with chunks file_type would be UNKNOWN and table_uri not set + publ_info = layman_util.get_publication_info(username, LAYER_TYPE, layername, context={'keys': ['table_uri']}) + table_uri = publ_info['_table_uri'] + conn_cur = db_util.create_connection_cursor(db_uri_str=table_uri.db_uri_str) + bbox = db_get_bbox(table_uri.schema, table_uri.table, conn_cur=conn_cur, column=table_uri.geo_column) + crs = db_get_crs(table_uri.schema, table_uri.table, conn_cur=conn_cur, column=table_uri.geo_column) elif file_type == settings.FILE_TYPE_RASTER: bbox = gdal_get_bbox(username, layername) crs = gdal_get_crs(username, layername) @@ -41,7 +53,6 @@ def refresh_file_data( if self.is_aborted(): raise AbortedException - set_file_type(username, LAYER_TYPE, layername, file_type, ) set_bbox(username, LAYER_TYPE, layername, bbox, crs, ) if self.is_aborted(): diff --git a/tests/dynamic_data/publications/layer_external_db/external_db_test.py b/tests/dynamic_data/publications/layer_external_db/external_db_test.py index f00f0485b..dcf9f0706 100644 --- a/tests/dynamic_data/publications/layer_external_db/external_db_test.py +++ b/tests/dynamic_data/publications/layer_external_db/external_db_test.py @@ -22,6 +22,7 @@ 'table_name': 'all', 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'GEOMETRY', + 'exp_bounding_box': [15.0, 49.0, 15.3, 49.3], }, 'geometrycollection_mixed_case_table_name': { 'input_file_name': 'geometrycollection', @@ -29,6 +30,7 @@ 'table_name': 'MyGeometryCollection', 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'GEOMETRYCOLLECTION', + 'exp_bounding_box': [15.0, 45.0, 18, 46], }, 'linestring_dangerous_table_name': { 'input_file_name': 'linestring', @@ -36,6 +38,7 @@ 'table_name': DANGEROUS_NAME, 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'LINESTRING', + 'exp_bounding_box': [15.0, 49.0, 15.3, 49.3], }, 'multilinestring_dangerous_schema_name': { 'input_file_name': 'multilinestring', @@ -43,6 +46,7 @@ 'table_name': 'multilinestring', 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'MULTILINESTRING', + 'exp_bounding_box': [16.0, 47.0, 16.0, 48.5], }, 'multipoint_dangerous_geo_column_name': { 'input_file_name': 'multipoint', @@ -50,6 +54,7 @@ 'table_name': 'multipoint', 'geo_column_name': DANGEROUS_NAME, 'exp_geometry_type': 'MULTIPOINT', + 'exp_bounding_box': [15.0, 47.8, 15.0, 48.0], }, 'multipolygon': { 'input_file_name': 'multipolygon', @@ -57,6 +62,7 @@ 'table_name': 'multipolygon', 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'MULTIPOLYGON', + 'exp_bounding_box': [17.0, 47.0, 18.0, 48.5], }, 'point': { 'input_file_name': 'point', @@ -64,6 +70,7 @@ 'table_name': 'point', 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'POINT', + 'exp_bounding_box': [15.0, 49.0, 15.3, 49.3], }, 'polygon': { 'input_file_name': 'polygon', @@ -71,6 +78,7 @@ 'table_name': 'polygon', 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'POLYGON', + 'exp_bounding_box': [15.0, 49.0, 15.3, 49.3], }, } @@ -112,7 +120,7 @@ def test_layer(layer: Publication, rest_method, rest_args, params): rest_method(layer, args=rest_args) with app.app_context(): - publ_info = get_publication_info(layer.workspace, layer.type, layer.name, context={'keys': ['table_uri'], }) + publ_info = get_publication_info(layer.workspace, layer.type, layer.name, context={'keys': ['table_uri', 'native_crs', 'native_bounding_box'], }) table_uri = publ_info['_table_uri'] assert table_uri == TableUri( db_uri_str=external_db.URI_STR, @@ -121,4 +129,7 @@ def test_layer(layer: Publication, rest_method, rest_args, params): geo_column=geo_column, ) + assert publ_info['native_crs'] == 'EPSG:4326' + assert publ_info['native_bounding_box'] == params['exp_bounding_box'] + external_db.drop_table(schema, table)