diff --git a/doc/rest.md b/doc/rest.md index 36576f175..0ad443ad4 100644 --- a/doc/rest.md +++ b/doc/rest.md @@ -147,6 +147,7 @@ Body parameters: - *db_connection*, string - exactly one of `file` or `db_connection` must be set - format `postgresql://:@:/?schema=&table=&geo_column=` is expected with URI scheme `postgresql` and query parameters `schema`, `table`, and `geo_column` specified + - published table is required to have one-column primary key - *name*, string - computer-friendly identifier of the layer - must be unique among all layers of one workspace diff --git a/src/db/__init__.py b/src/db/__init__.py index 7426a579c..85c0cbe41 100644 --- a/src/db/__init__.py +++ b/src/db/__init__.py @@ -4,11 +4,12 @@ @dataclass class TableUri: - def __init__(self, *, db_uri_str, schema, table, geo_column): + def __init__(self, *, db_uri_str, schema, table, geo_column, primary_key_column): self.db_uri_str = db_uri_str self.schema = schema self.table = table self.geo_column = geo_column + self.primary_key_column = primary_key_column _db_uri_str: str _db_uri: parse.ParseResult diff --git a/src/layman/common/prime_db_schema/publications.py b/src/layman/common/prime_db_schema/publications.py index 0bb82320b..929b0afa9 100644 --- a/src/layman/common/prime_db_schema/publications.py +++ b/src/layman/common/prime_db_schema/publications.py @@ -238,6 +238,7 @@ def get_publication_infos_with_metainfo(workspace_name=None, pub_type=None, styl schema=external_table_uri['schema'], table=external_table_uri['table'], geo_column=external_table_uri['geo_column'], + primary_key_column=external_table_uri['primary_key_column'], ) if external_table_uri else None, '_is_external_table': bool(external_table_uri), 'native_bounding_box': [xmin, ymin, xmax, ymax], @@ -404,6 +405,7 @@ def insert_publication(workspace_name, info): 'schema': info["external_table_uri"].schema, 'table': info["external_table_uri"].table, 'geo_column': info["external_table_uri"].geo_column, + 'primary_key_column': info["external_table_uri"].primary_key_column, }) if info.get("external_table_uri") else None data = (id_workspace, diff --git a/src/layman/layer/db/__init__.py b/src/layman/layer/db/__init__.py index 453244092..59b6e925a 100644 --- a/src/layman/layer/db/__init__.py +++ b/src/layman/layer/db/__init__.py @@ -258,7 +258,7 @@ def get_number_of_features(schema, table_name, conn_cur=None): return rows[0][0] -def get_text_data(schema, table_name, conn_cur=None): +def get_text_data(schema, table_name, primary_key, conn_cur=None): _, cur = conn_cur or db_util.get_connection_cursor() col_names = get_text_column_names(schema, table_name, conn_cur=conn_cur) if len(col_names) == 0: @@ -270,11 +270,12 @@ def get_text_data(schema, table_name, conn_cur=None): statement = sql.SQL(""" select {fields} from {table} -order by ogc_fid +order by {primary_key} limit {limit} """).format( fields=sql.SQL(',').join([sql.Identifier(col) for col in col_names]), table=sql.Identifier(schema, table_name), + primary_key=sql.Identifier(primary_key), limit=sql.Literal(limit), ) try: @@ -297,8 +298,8 @@ def get_text_data(schema, table_name, conn_cur=None): return col_texts, limit -def get_text_languages(schema, table_name, *, conn_cur=None): - texts, num_rows = get_text_data(schema, table_name, conn_cur) +def get_text_languages(schema, table_name, primary_key, *, conn_cur=None): + texts, num_rows = get_text_data(schema, table_name, primary_key, conn_cur) all_langs = set() for text in texts: # skip short texts @@ -312,61 +313,61 @@ def get_text_languages(schema, table_name, *, conn_cur=None): return sorted(list(all_langs)) -def get_most_frequent_lower_distance_query(schema, table_name): +def get_most_frequent_lower_distance_query(schema, table_name, primary_key, geometry_column): query = sql.SQL(""" with t1 as ( select - row_number() over (partition by ogc_fid) AS dump_id, + row_number() over (partition by {primary_key}) AS dump_id, sub_view.* from ( SELECT - ogc_fid, (st_dump(wkb_geometry)).geom as geometry + {primary_key}, (st_dump({geometry_column})).geom as geometry FROM {table} ) sub_view -order by ST_NPoints(geometry), ogc_fid, dump_id +order by ST_NPoints(geometry), {primary_key}, dump_id limit 5000 ) , t2 as ( select - row_number() over (partition by ogc_fid, dump_id) AS ring_id, + row_number() over (partition by {primary_key}, dump_id) AS ring_id, sub_view.* from ( ( SELECT - dump_id, ogc_fid, ST_ExteriorRing((ST_DumpRings(geometry)).geom) as geometry + dump_id, {primary_key}, ST_ExteriorRing((ST_DumpRings(geometry)).geom) as geometry FROM t1 where st_geometrytype(geometry) = 'ST_Polygon' ) union all ( SELECT - dump_id, ogc_fid, geometry + dump_id, {primary_key}, geometry FROM t1 where st_geometrytype(geometry) = 'ST_LineString' ) ) sub_view -order by ST_NPoints(geometry), ogc_fid, dump_id, ring_id +order by ST_NPoints(geometry), {primary_key}, dump_id, ring_id limit 5000 ) , t2cumsum as ( select *, --ST_NPoints(geometry), - sum(ST_NPoints(geometry)) over (order by ST_NPoints(geometry), ogc_fid, dump_id, ring_id + sum(ST_NPoints(geometry)) over (order by ST_NPoints(geometry), {primary_key}, dump_id, ring_id rows between unbounded preceding and current row) as cum_sum_points from t2 ) , t3 as ( -SELECT ogc_fid, dump_id, ring_id, (ST_DumpPoints(st_transform(geometry, 4326))).* +SELECT {primary_key}, dump_id, ring_id, (ST_DumpPoints(st_transform(geometry, 4326))).* FROM t2cumsum where cum_sum_points < 50000 ) , t4 as MATERIALIZED ( - select t3.ogc_fid, t3.dump_id, t3.ring_id, t3.path[1] as point_idx, t3.geom as point1, t3p2.geom as point2 + select t3.{primary_key}, t3.dump_id, t3.ring_id, t3.path[1] as point_idx, t3.geom as point1, t3p2.geom as point2 from t3 - inner join t3 t3p2 on (t3.ogc_fid = t3p2.ogc_fid and + inner join t3 t3p2 on (t3.{primary_key} = t3p2.{primary_key} and t3.dump_id = t3p2.dump_id and t3.ring_id = t3p2.ring_id and t3.path[1] + 1 = t3p2.path[1]) ) , tdist as ( -SELECT ogc_fid, dump_id, ring_id, point_idx, +SELECT {primary_key}, dump_id, ring_id, point_idx, ST_DistanceSphere(point1, point2) as distance FROM t4 ) @@ -402,14 +403,16 @@ def get_most_frequent_lower_distance_query(schema, table_name): limit 1 """).format( table=sql.Identifier(schema, table_name), + primary_key=sql.Identifier(primary_key), + geometry_column=sql.Identifier(geometry_column), ) return query -def get_most_frequent_lower_distance(schema, table_name, conn_cur=None): +def get_most_frequent_lower_distance(schema, table_name, primary_key, geometry_column, conn_cur=None): _, cur = conn_cur or db_util.get_connection_cursor() - query = get_most_frequent_lower_distance_query(schema, table_name) + query = get_most_frequent_lower_distance_query(schema, table_name, primary_key, geometry_column) # print(f"\nget_most_frequent_lower_distance v1\nusername={username}, layername={layername}") # print(query) @@ -448,8 +451,8 @@ def get_most_frequent_lower_distance(schema, table_name, conn_cur=None): ] -def guess_scale_denominator(schema, table_name, *, conn_cur=None): - distance = get_most_frequent_lower_distance(schema, table_name, conn_cur=conn_cur) +def guess_scale_denominator(schema, table_name, primary_key, geometry_column, *, conn_cur=None): + distance = get_most_frequent_lower_distance(schema, table_name, primary_key, geometry_column, conn_cur=conn_cur) log_sd_list = [math.log10(sd) for sd in SCALE_DENOMINATORS] if distance is not None: coef = 2000 if distance > 100 else 1000 @@ -524,7 +527,7 @@ def ensure_attributes(attribute_tuples): return missing_attributes -def get_bbox(schema, table_name, conn_cur=None, column='wkb_geometry'): +def get_bbox(schema, table_name, conn_cur=None, column=settings.OGR_DEFAULT_GEOMETRY_COLUMN): query = sql.SQL(''' with tmp as (select ST_Extent(l.{column}) as bbox from {table} l @@ -542,14 +545,14 @@ def get_bbox(schema, table_name, conn_cur=None, column='wkb_geometry'): return result -def get_crs(schema, table_name, conn_cur=None, column='wkb_geometry'): +def get_crs(schema, table_name, conn_cur=None, column=settings.OGR_DEFAULT_GEOMETRY_COLUMN): query = 'select Find_SRID(%s, %s, %s);' srid = db_util.run_query(query, (schema, table_name, column), conn_cur=conn_cur)[0][0] crs = db_util.get_crs(srid) return crs -def get_geometry_types(schema, table_name, *, column_name='wkb_geometry', conn_cur=None): +def get_geometry_types(schema, table_name, *, column_name=settings.OGR_DEFAULT_GEOMETRY_COLUMN, conn_cur=None): conn, cur = conn_cur or db_util.get_connection_cursor() query = sql.SQL(""" select distinct ST_GeometryType({column}) as geometry_type_name diff --git a/src/layman/layer/db/db_test.py b/src/layman/layer/db/db_test.py index ae3721c17..d52bbe28a 100644 --- a/src/layman/layer/db/db_test.py +++ b/src/layman/layer/db/db_test.py @@ -175,12 +175,12 @@ def test_data_language(boundary_table): col_names = db.get_text_column_names(workspace, table_name) assert set(col_names) == set(['featurecla', 'name', 'name_alt']) with layman.app_context(): - text_data, _ = db.get_text_data(workspace, table_name) + text_data, _ = db.get_text_data(workspace, table_name, settings.OGR_DEFAULT_PRIMARY_KEY) # print(f"num_rows={num_rows}") assert len(text_data) == 1 assert text_data[0].startswith(' '.join(['International boundary (verify)'] * 100)) with layman.app_context(): - langs = db.get_text_languages(workspace, table_name) + langs = db.get_text_languages(workspace, table_name, settings.OGR_DEFAULT_PRIMARY_KEY) assert langs == ['eng'] @@ -214,7 +214,7 @@ def test_data_language_roads(road_table): 'vym_tahy_p' ]) with layman.app_context(): - langs = db.get_text_languages(workspace, table_name) + langs = db.get_text_languages(workspace, table_name, settings.OGR_DEFAULT_PRIMARY_KEY) assert langs == ['cze'] @@ -226,7 +226,7 @@ def test_populated_places_table(populated_places_table): col_names = db.get_text_column_names(workspace, table_name) assert len(col_names) == 31 with layman.app_context(): - langs = db.get_text_languages(workspace, table_name) + langs = db.get_text_languages(workspace, table_name, settings.OGR_DEFAULT_PRIMARY_KEY) assert set(langs) == set(['chi', 'eng', 'rus']) @@ -238,7 +238,7 @@ def test_data_language_countries(country_table): col_names = db.get_text_column_names(workspace, table_name) assert len(col_names) == 63 with layman.app_context(): - langs = db.get_text_languages(workspace, table_name) + langs = db.get_text_languages(workspace, table_name, settings.OGR_DEFAULT_PRIMARY_KEY) assert set(langs) == set([ 'ara', 'ben', @@ -266,13 +266,14 @@ def test_data_language_countries2(country110m_table): # assert len(col_names) == 63 with layman.app_context(): table_name = db.get_table_name(workspace, layername) - langs = db.get_text_languages(workspace, table_name) + langs = db.get_text_languages(workspace, table_name, settings.OGR_DEFAULT_PRIMARY_KEY) assert set(langs) == set(['eng']) def guess_scale_denominator(workspace, layer): table_name = db.get_table_name(workspace, layer) - return db.guess_scale_denominator(workspace, table_name) + return db.guess_scale_denominator(workspace, table_name, settings.OGR_DEFAULT_PRIMARY_KEY, + settings.OGR_DEFAULT_GEOMETRY_COLUMN) def test_guess_scale_denominator(country110m_table, country50m_table, country10m_table, diff --git a/src/layman/layer/db/table.py b/src/layman/layer/db/table.py index a35363188..db22d2601 100644 --- a/src/layman/layer/db/table.py +++ b/src/layman/layer/db/table.py @@ -57,6 +57,6 @@ def delete_layer(workspace, layername, conn_cur=None): def set_layer_srid(schema, table_name, srid, *, conn_cur=None): - query = '''SELECT UpdateGeometrySRID(%s, %s, 'wkb_geometry', %s);''' - params = (schema, table_name, srid) + query = '''SELECT UpdateGeometrySRID(%s, %s, %s, %s);''' + params = (schema, table_name, settings.OGR_DEFAULT_GEOMETRY_COLUMN, srid) db_util.run_query(query, params, conn_cur=conn_cur) diff --git a/src/layman/layer/micka/csw.py b/src/layman/layer/micka/csw.py index 0313a2e92..b909f6bd3 100644 --- a/src/layman/layer/micka/csw.py +++ b/src/layman/layer/micka/csw.py @@ -143,11 +143,13 @@ def get_template_path_and_values(workspace, layername, http_method): table_name = table_uri.table conn_cur = db_util.create_connection_cursor(db_uri_str=table_uri.db_uri_str) try: - languages = db.get_text_languages(table_uri.schema, table_name, conn_cur=conn_cur) + languages = db.get_text_languages(table_uri.schema, table_name, table_uri.primary_key_column, + conn_cur=conn_cur) except LaymanError: languages = [] try: - scale_denominator = db.guess_scale_denominator(table_uri.schema, table_name, conn_cur=conn_cur) + scale_denominator = db.guess_scale_denominator(table_uri.schema, table_name, table_uri.primary_key_column, + table_uri.geo_column, conn_cur=conn_cur) except LaymanError: scale_denominator = None spatial_resolution = { diff --git a/src/layman/layer/prime_db_schema/table.py b/src/layman/layer/prime_db_schema/table.py index 5ba4b549e..fc0fccff3 100644 --- a/src/layman/layer/prime_db_schema/table.py +++ b/src/layman/layer/prime_db_schema/table.py @@ -22,7 +22,8 @@ def get_layer_info(workspace, layername): db_uri_str=settings.PG_URI_STR, schema=workspace, table=f'layer_{uuid.replace("-", "_")}', - geo_column='wkb_geometry' + geo_column=settings.OGR_DEFAULT_GEOMETRY_COLUMN, + primary_key_column=settings.OGR_DEFAULT_PRIMARY_KEY, ) if info['_file_type'] == settings.FILE_TYPE_VECTOR and not info.get('_table_uri') else info.get('_table_uri') return info diff --git a/src/layman/layer/qgis/layer-template.qml b/src/layman/layer/qgis/layer-template.qml index ccd0ffe31..3fb99f7eb 100644 --- a/src/layman/layer/qgis/layer-template.qml +++ b/src/layman/layer/qgis/layer-template.qml @@ -6,7 +6,7 @@ {extent} {layer_name}_{layer_uuid} - dbname='{db_name}' host={db_host} port={db_port} user='{db_user}' password='{db_password}' sslmode=disable key='ogc_fid' + dbname='{db_name}' host={db_host} port={db_port} user='{db_user}' password='{db_password}' sslmode=disable key='{primary_key_column}' srid={srid} type={source_type} checkPrimaryKeyUnicity='1' table="{db_schema}"."{db_table}" ({geo_column}) diff --git a/src/layman/layer/qgis/project-template.qgs b/src/layman/layer/qgis/project-template.qgs index b4e355185..32e8be15e 100644 --- a/src/layman/layer/qgis/project-template.qgs +++ b/src/layman/layer/qgis/project-template.qgs @@ -12,7 +12,7 @@ - + diff --git a/src/layman/layer/qgis/util.py b/src/layman/layer/qgis/util.py index c03e90f4d..0b26837f7 100644 --- a/src/layman/layer/qgis/util.py +++ b/src/layman/layer/qgis/util.py @@ -78,6 +78,7 @@ def fill_layer_template(layer, uuid, native_bbox, crs, qml_xml, source_type, att source_type=source_type, db_schema=db_schema, db_table=table_name, + primary_key_column=table_uri.primary_key_column, geo_column=geo_column, layer_name=layer_name, layer_uuid=uuid, @@ -137,6 +138,7 @@ def fill_project_template(layer, layer_uuid, layer_qml, crs, epsg_codes, extent, source_type=source_type, db_schema=db_schema, db_table=table_name, + primary_key_column=table_uri.primary_key_column, geo_column=geo_column, layer_name=layer_name, layer_uuid=layer_uuid, diff --git a/src/layman/layer/qgis/wms.py b/src/layman/layer/qgis/wms.py index 0ea591f6e..122fd5d35 100644 --- a/src/layman/layer/qgis/wms.py +++ b/src/layman/layer/qgis/wms.py @@ -76,7 +76,7 @@ def save_qgs_file(workspace, layer): db_types = db.get_geometry_types(db_schema, table_name, conn_cur=conn_cur) db_cols = [ col for col in db.get_all_column_infos(db_schema, table_name, conn_cur=conn_cur, omit_geometry_columns=True) - if col.name not in ['ogc_fid'] + if col.name != table_uri.primary_key_column ] source_type = util.get_source_type(db_types, qml_geometry) layer_qml = util.fill_layer_template(layer, uuid, layer_bbox, crs, qml, source_type, db_cols, table_uri) diff --git a/src/layman/layer/util.py b/src/layman/layer/util.py index d6dead621..83defe014 100644 --- a/src/layman/layer/util.py +++ b/src/layman/layer/util.py @@ -343,10 +343,52 @@ def parse_and_validate_external_table_uri_str(external_table_uri_str): } }) + # https://stackoverflow.com/a/20537829 + query = f''' +SELECT + pg_attribute.attname, + format_type(pg_attribute.atttypid, pg_attribute.atttypmod) +FROM pg_index, pg_class, pg_attribute, pg_namespace +WHERE + pg_class.relname = %s AND + indrelid = pg_class.oid AND + nspname = %s AND + pg_class.relnamespace = pg_namespace.oid AND + pg_attribute.attrelid = pg_class.oid AND + pg_attribute.attnum = any(pg_index.indkey) + AND indisprimary''' + query_res = db_util.run_query(query, (table, schema), conn_cur=conn_cur, log_query=True) + primary_key_columns = [r[0] for r in query_res] + if len(query_res) == 0: + raise LaymanError(2, { + 'parameter': 'db_connection', + 'message': 'No primary key found in the table.', + 'expected': 'Table with one-column primary key.', + 'found': { + 'db_connection': external_table_uri_str, + 'schema': schema, + 'table': table, + 'primary_key_columns': primary_key_columns, + } + }) + if len(query_res) > 1: + raise LaymanError(2, { + 'parameter': 'db_connection', + 'message': 'Table with multi-column primary key.', + 'expected': 'Table with one-column primary key.', + 'found': { + 'db_connection': external_table_uri_str, + 'schema': schema, + 'table': table, + 'primary_key_columns': primary_key_columns, + } + }) + result = TableUri(db_uri_str=db_uri_str, schema=schema, table=table, geo_column=geo_column, + primary_key_column=primary_key_columns[0], ) return result diff --git a/src/layman/layer/util_test.py b/src/layman/layer/util_test.py index ede3aa441..ce9a6e33f 100644 --- a/src/layman/layer/util_test.py +++ b/src/layman/layer/util_test.py @@ -14,14 +14,16 @@ @pytest.fixture(scope="module") def ensure_tables(): tables = [ - ('schema_name', 'table_name', 'geo_wkb_column'), + ('schema_name', 'table_name', 'geo_wkb_column', ['my_id']), + ('schema_name', 'two_column_primary_key', 'geo_wkb_column', ['partial_id_1', 'partial_id_2']), + ('schema_name', 'no_primary_key', 'geo_wkb_column', []), ] - for schema, table, geo_column in tables: - external_db.ensure_table(schema, table, geo_column) + for schema, table, geo_column, primary_key in tables: + external_db.ensure_table(schema, table, geo_column, primary_key=primary_key) yield - for schema, table, _ in tables: + for schema, table, _, _ in tables: external_db.drop_table(schema, table) @@ -172,6 +174,7 @@ def successful(): schema='schema_name', table='table_name', geo_column='geo_wkb_column', + primary_key_column='my_id', )), ]) def test_parse_external_table_uri_str(external_table_uri_str, exp_result): @@ -299,7 +302,7 @@ def test_parse_external_table_uri_str(external_table_uri_str, exp_result): }, }, }, id='invalid_geo_column'), - pytest.param('postgresql://docker:docker@postgresql:5432/external_test_db?schema=no_schema&table=table_name&geo_column=no_wkb_geometry', { + pytest.param('postgresql://docker:docker@postgresql:5432/external_test_db?schema=no_schema&table=table_name&geo_column=geo_wkb_column', { 'http_code': 400, 'code': 2, 'detail': { @@ -307,12 +310,42 @@ def test_parse_external_table_uri_str(external_table_uri_str, exp_result): 'message': 'Table "no_schema"."table_name" not found in database.', 'expected': util.EXTERNAL_TABLE_URI_PATTERN, 'found': { - 'db_connection': 'postgresql://docker:docker@postgresql:5432/external_test_db?schema=no_schema&table=table_name&geo_column=no_wkb_geometry', + 'db_connection': 'postgresql://docker:docker@postgresql:5432/external_test_db?schema=no_schema&table=table_name&geo_column=geo_wkb_column', 'schema': 'no_schema', 'table': 'table_name', }, }, }, id='invalid_schema'), + pytest.param('postgresql://docker:docker@postgresql:5432/external_test_db?schema=schema_name&table=two_column_primary_key&geo_column=geo_wkb_column', { + 'http_code': 400, + 'code': 2, + 'detail': { + 'parameter': 'db_connection', + 'message': 'Table with multi-column primary key.', + 'expected': 'Table with one-column primary key.', + 'found': { + 'db_connection': 'postgresql://docker:docker@postgresql:5432/external_test_db?schema=schema_name&table=two_column_primary_key&geo_column=geo_wkb_column', + 'schema': 'schema_name', + 'table': 'two_column_primary_key', + 'primary_key_columns': ['partial_id_1', 'partial_id_2'], + }, + }, + }, id='two_column_primary_key'), + pytest.param('postgresql://docker:docker@postgresql:5432/external_test_db?schema=schema_name&table=no_primary_key&geo_column=geo_wkb_column', { + 'http_code': 400, + 'code': 2, + 'detail': { + 'parameter': 'db_connection', + 'message': 'No primary key found in the table.', + 'expected': 'Table with one-column primary key.', + 'found': { + 'db_connection': 'postgresql://docker:docker@postgresql:5432/external_test_db?schema=schema_name&table=no_primary_key&geo_column=geo_wkb_column', + 'schema': 'schema_name', + 'table': 'no_primary_key', + 'primary_key_columns': [], + }, + }, + }, id='no_primary_key'), ]) def test_validate_external_table_uri_str(external_table_uri_str, exp_error): with pytest.raises(LaymanError) as exc_info: diff --git a/src/layman/upgrade/upgrade_v1_17_test.py b/src/layman/upgrade/upgrade_v1_17_test.py index 15c080e0d..fa1531d74 100644 --- a/src/layman/upgrade/upgrade_v1_17_test.py +++ b/src/layman/upgrade/upgrade_v1_17_test.py @@ -176,10 +176,12 @@ def publish_layer(workspace, layer, *, file_path, style_type, style_file, ): db_types = db.get_geometry_types(workspace, table_name) db_cols = [ col for col in db.get_all_column_infos(workspace, table_name) - if col.name not in ['wkb_geometry', 'ogc_fid'] + if col.name not in [settings.OGR_DEFAULT_GEOMETRY_COLUMN, settings.OGR_DEFAULT_PRIMARY_KEY] ] source_type = qgis_util.get_source_type(db_types, qml_geometry) - table_uri = TableUri(db_uri_str=settings.PG_URI_STR, table=table_name, schema=workspace, geo_column='wkb_geometry') + table_uri = TableUri(db_uri_str=settings.PG_URI_STR, table=table_name, schema=workspace, + geo_column=settings.OGR_DEFAULT_GEOMETRY_COLUMN, + primary_key_column=settings.OGR_DEFAULT_PRIMARY_KEY) layer_qml = qgis_util.fill_layer_template(layer, uuid_str, bbox, crs, qml, source_type, db_cols, table_uri) qgs_str = qgis_util.fill_project_template(layer, uuid_str, layer_qml, crs, settings.LAYMAN_OUTPUT_SRS_LIST, bbox, source_type, table_uri) diff --git a/src/layman_settings.py b/src/layman_settings.py index d4c971fbd..43cc6a2cb 100644 --- a/src/layman_settings.py +++ b/src/layman_settings.py @@ -251,3 +251,6 @@ RESERVED_WORKSPACE_NAMES = {REST_USERS_PREFIX, REST_WORKSPACES_PREFIX} # PREFERRED_LANGUAGES = ['cs', 'en'] + +OGR_DEFAULT_PRIMARY_KEY = 'ogc_fid' +OGR_DEFAULT_GEOMETRY_COLUMN = 'wkb_geometry' diff --git a/test_tools/external_db.py b/test_tools/external_db.py index 9ad8512ad..98a2c01dd 100644 --- a/test_tools/external_db.py +++ b/test_tools/external_db.py @@ -33,17 +33,33 @@ def ensure_schema(schema): db_util.run_statement(statement, conn_cur=conn_cur) -def ensure_table(schema, name, geo_column): +def ensure_table(schema, name, geo_column, *, primary_key=None): + primary_key = ['id'] if primary_key is None else primary_key + ensure_schema(schema) - statement = sql.SQL('create table {table} ({geo_column} geometry(Geometry, 4326))').format( + columns = [] + for col in primary_key: + columns.append(sql.SQL('{column} serial').format( + column=sql.Identifier(col) + )) + columns.append(sql.SQL('{geo_column} geometry(Geometry, 4326)').format( + geo_column=sql.Identifier(geo_column) + )) + if primary_key: + columns.append(sql.SQL('PRIMARY KEY ({columns})').format( + columns=sql.SQL(',').join(sql.Identifier(c) for c in primary_key) + )) + + statement = sql.SQL('create table {table} ({columns})').format( table=sql.Identifier(schema, name), - geo_column=sql.Identifier(geo_column), + columns=sql.SQL(',').join(columns), ) conn_cur = db_util.create_connection_cursor(URI_STR) db_util.run_statement(statement, conn_cur=conn_cur) -def import_table(input_file_path, *, table=None, schema='public', geo_column='wkb_geometry'): +def import_table(input_file_path, *, table=None, schema='public', geo_column=settings.OGR_DEFAULT_GEOMETRY_COLUMN, + primary_key_column=settings.OGR_DEFAULT_PRIMARY_KEY): table = table or os.path.splitext(os.path.basename(input_file_path))[0] ensure_schema(schema) @@ -57,6 +73,7 @@ def import_table(input_file_path, *, table=None, schema='public', geo_column='wk '-lco', f'LAUNDER=NO', '-lco', f'EXTRACT_SCHEMA_FROM_LAYER_NAME=NO', '-lco', f'GEOMETRY_NAME={geo_column}', + '-lco', f"FID={primary_key_column if primary_key_column is not None else f'{settings.OGR_DEFAULT_PRIMARY_KEY}'}", '-f', 'PostgreSQL', target_db, input_file_path, @@ -68,6 +85,14 @@ def import_table(input_file_path, *, table=None, schema='public', geo_column='wk return_code = process.poll() assert return_code == 0 and not stdout and not stderr, f"return_code={return_code}, stdout={stdout}, stderr={stderr}" + if primary_key_column is None: + conn_cur = db_util.create_connection_cursor(URI_STR) + statement = sql.SQL("alter table {table} drop column {primary_key}").format( + table=sql.Identifier(schema, table), + primary_key=sql.Identifier(settings.OGR_DEFAULT_PRIMARY_KEY), + ) + db_util.run_statement(statement, conn_cur=conn_cur) + def drop_table(schema, name): statement = sql.SQL('drop table {table}').format( diff --git a/tests/asserts/final/publication/internal.py b/tests/asserts/final/publication/internal.py index f7790d11f..e6c042c87 100644 --- a/tests/asserts/final/publication/internal.py +++ b/tests/asserts/final/publication/internal.py @@ -227,7 +227,8 @@ def correct_values_in_detail(workspace, publ_type, name, *, exp_publication_deta db_uri_str='postgresql://docker:docker@postgresql:5432/layman_test', schema=workspace, table=db_table, - geo_column='wkb_geometry' + geo_column='wkb_geometry', + primary_key_column='ogc_fid', ) util.recursive_dict_update(expected_detail, { diff --git a/tests/dynamic_data/publications/layer_external_db/external_db_test.py b/tests/dynamic_data/publications/layer_external_db/external_db_test.py index f39276366..6e3546f1e 100644 --- a/tests/dynamic_data/publications/layer_external_db/external_db_test.py +++ b/tests/dynamic_data/publications/layer_external_db/external_db_test.py @@ -24,6 +24,7 @@ 'style_file': None, 'schema_name': 'public', 'table_name': 'all', + 'primary_key_column': 'ogc_fid', 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'GEOMETRY', 'exp_native_bounding_box': [15.0, 49.0, 15.3, 49.3], @@ -34,6 +35,7 @@ 'style_file': None, 'schema_name': 'public', 'table_name': 'MyGeometryCollection', + 'primary_key_column': 'ogc_fid', 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'GEOMETRYCOLLECTION', 'exp_native_bounding_box': [15.0, 45.0, 18.0, 46.0], @@ -44,6 +46,7 @@ 'style_file': None, 'schema_name': 'public', 'table_name': DANGEROUS_NAME, + 'primary_key_column': 'ogc_fid', 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'LINESTRING', 'exp_native_bounding_box': [15.0, 49.0, 15.3, 49.3], @@ -54,6 +57,7 @@ 'style_file': None, 'schema_name': DANGEROUS_NAME, 'table_name': 'multilinestring', + 'primary_key_column': 'ogc_fid', 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'MULTILINESTRING', 'exp_native_bounding_box': [16.0, 47.0, 16.0, 48.5], @@ -64,26 +68,29 @@ 'style_file': None, 'schema_name': 'public', 'table_name': 'multipoint', + 'primary_key_column': 'ogc_fid', 'geo_column_name': DANGEROUS_NAME, 'exp_geometry_type': 'MULTIPOINT', 'exp_native_bounding_box': [15.0, 47.8, 15.0, 48.0], 'exp_imported_into_GS': False, }, - 'multipolygon_qml': { + 'multipolygon_qml_custom_id_column': { 'input_file_name': 'multipolygon', 'style_file': 'tests/dynamic_data/publications/layer_external_db/multipolygon.qml', 'schema_name': 'public', 'table_name': 'multipolygon', + 'primary_key_column': 'my_id', 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'MULTIPOLYGON', 'exp_native_bounding_box': [17.0, 47.0, 18.0, 48.5], 'exp_bounding_box': [1892431.3434856508, 5942074.072431108, 2003750.8342789242, 6190443.809135445], }, - 'point': { + 'point_custom_id_column': { 'input_file_name': 'point', 'style_file': None, 'schema_name': 'public', 'table_name': 'point', + 'primary_key_column': 'my_id2', 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'POINT', 'exp_native_bounding_box': [15.0, 49.0, 15.3, 49.3], @@ -94,6 +101,7 @@ 'style_file': None, 'schema_name': 'public', 'table_name': 'polygon', + 'primary_key_column': 'ogc_fid', 'geo_column_name': 'wkb_geometry', 'exp_geometry_type': 'POLYGON', 'exp_native_bounding_box': [15.0, 49.0, 15.3, 49.3], @@ -138,8 +146,10 @@ def test_layer(layer: Publication, key, rest_method, rest_args, params): schema = params['schema_name'] table = params['table_name'] geo_column = params['geo_column_name'] + primary_key_column = params['primary_key_column'] - external_db.import_table(file_path, table=table, schema=schema, geo_column=geo_column) + external_db.import_table(file_path, table=table, schema=schema, geo_column=geo_column, + primary_key_column=primary_key_column) conn_cur = db_util.create_connection_cursor(external_db.URI_STR) query = f'''select type from geometry_columns where f_table_schema = %s and f_table_name = %s and f_geometry_column = %s''' result = db_util.run_query(query, (schema, table, geo_column), conn_cur=conn_cur) @@ -157,6 +167,7 @@ def test_layer(layer: Publication, key, rest_method, rest_args, params): schema=schema, table=table, geo_column=geo_column, + primary_key_column=primary_key_column ) assert publ_info['native_crs'] == 'EPSG:4326' diff --git a/tests/dynamic_data/publications/layer_external_db/thumbnail_multipolygon_qml.png b/tests/dynamic_data/publications/layer_external_db/thumbnail_multipolygon_qml_custom_id_column.png similarity index 100% rename from tests/dynamic_data/publications/layer_external_db/thumbnail_multipolygon_qml.png rename to tests/dynamic_data/publications/layer_external_db/thumbnail_multipolygon_qml_custom_id_column.png diff --git a/tests/dynamic_data/publications/layer_external_db/thumbnail_point.png b/tests/dynamic_data/publications/layer_external_db/thumbnail_point_custom_id_column.png similarity index 100% rename from tests/dynamic_data/publications/layer_external_db/thumbnail_point.png rename to tests/dynamic_data/publications/layer_external_db/thumbnail_point_custom_id_column.png