Add data versioning

src/layman/upgrade/__init__.py

@@ -0,0 +1,66 @@
+import logging
+import psycopg2
+
+from layman.upgrade import upgrade_v1_8, upgrade_v1_9
+from layman import settings
+from layman.common.prime_db_schema import util as db_util
+DB_SCHEMA = settings.LAYMAN_PRIME_SCHEMA
+
+logger = logging.getLogger(__name__)
+
+
+DATA_MIGRATIONS = [
+    ((1, 9, 0), [upgrade_v1_9.initialize_data_versioning, ]),
+]
+
+
+def get_max_data_version():
+    max_migration = max(DATA_MIGRATIONS, key=lambda x: x[0])
+    return max_migration[0] + (len(max_migration[1]) - 1, )
+
+
+def get_current_data_version():
+    # This table should have only one row (or none for the first time)
+    current_version = None
+    try:
+        sql_select = f'''select major_version, minor_version, patch_version, migration from {DB_SCHEMA}.data_version;'''
+        sql_result = db_util.run_query(sql_select, encapsulate_exception=False)
+        row_count = len(sql_result)
+        if row_count == 1:
+            current_version_sql = sql_result[0]
+            current_version = (current_version_sql[0], current_version_sql[1], current_version_sql[2], current_version_sql[3])
+        elif row_count == 0:
+            current_version = (-1, -1, -1, -1)
+        else:
+            assert row_count == 1
+    except psycopg2.errors.UndefinedTable:
+        current_version = (-1, -1, -1, -1)
+    return current_version
+
+
+def upgrade():
+    logger.info(f'Checking all upgrades')
+    if upgrade_v1_8.older_than_1_8():
+        upgrade_v1_8.upgrade_1_8()
+
+    current_data_version = get_current_data_version()
+    max_data_version = get_max_data_version()
+    logger.info(f'  Current data version = {current_data_version}, Maximum data version = {max_data_version}')
+    migration_list_full = [(version + (index,), migration, )
+                           for version, migration_list in DATA_MIGRATIONS
+                           for index, migration in enumerate(migration_list)
+                           if version + (index,) > current_data_version]
+    migration_list_full.sort(key=lambda x: x[0])
+    for version, migration in migration_list_full:
+        logger.info(f'  Starting migration #{version[3]} for release v{version[0]}.{version[1]}.{version[2]}: {migration}')
+        migration()
+        sql_insert = f'''update {DB_SCHEMA}.data_version set
+            major_version = %s,
+            minor_version = %s,
+            patch_version = %s,
+            migration = %s;'''
+        db_util.run_statement(sql_insert, version)
+
+    final_data_version = get_current_data_version()
+    assert final_data_version == max_data_version
+    logger.info(f'  Checking all upgrades DONE, final data version = {final_data_version}')

src/layman/upgrade/upgrade_test.py

@@ -0,0 +1,37 @@
+import pytest
+
+from layman import upgrade, app, settings
+from layman.common.prime_db_schema import util as db_util
+DB_SCHEMA = settings.LAYMAN_PRIME_SCHEMA
+
+
+def test_upgrade_run():
+    with app.app_context():
+        upgrade.upgrade()
+
+
+@pytest.mark.parametrize('sql_command, expected_value', [
+    (f'''DROP TABLE IF EXISTS {DB_SCHEMA}.data_version''', (-1, -1, -1, -1)),
+    (f'''DROP TABLE IF EXISTS {DB_SCHEMA}.data_version; CREATE TABLE IF NOT EXISTS {DB_SCHEMA}.data_version
+        (
+            major_version integer not null,
+            minor_version integer not null,
+            patch_version integer not null,
+            migration integer not null
+        )
+        TABLESPACE pg_default;''', (-1, -1, -1, -1)),
+    (f'''DROP TABLE IF EXISTS {DB_SCHEMA}.data_version; CREATE TABLE IF NOT EXISTS {DB_SCHEMA}.data_version
+        (
+            major_version integer not null,
+            minor_version integer not null,
+            patch_version integer not null,
+            migration integer not null
+        )
+        TABLESPACE pg_default;
+        insert into {DB_SCHEMA}.data_version (major_version, minor_version, patch_version, migration) values (1, 9, 0, 0); commit;''', (1, 9, 0, 0)),
+])
+def test_get_current_data_version(sql_command, expected_value):
+    with app.app_context():
+        db_util.run_statement(sql_command)
+        currrent_data_version = upgrade.get_current_data_version()
+        assert currrent_data_version == expected_value

src/layman/upgrade.py → src/layman/upgrade/upgrade_v1_8.py

@@ -1,11 +1,11 @@
+from urllib.parse import urljoin
 import logging
 import requests
-from urllib.parse import urljoin
 
 from layman import app, settings
+from layman.common import geoserver
 from layman.common.prime_db_schema import schema_initialization
 from layman.layer.util import get_layer_infos
-from layman.common import geoserver
 
 logger = logging.getLogger(__name__)
 
@@ -24,7 +24,7 @@ def upgrade_1_8():
                                             settings.RIGHTS_EVERYONE_ROLE)
 
         logger.info(f'  Ensuring users')
-        from .util import get_usernames, ensure_whole_user, check_username
+        from ..util import get_usernames, ensure_whole_user, check_username
         all_usernames = get_usernames()
         for username in all_usernames:
             logger.info(f'    Ensuring user {username}')
@@ -56,10 +56,3 @@ def upgrade_1_8():
                 for type in ['read', 'write']:
                     security_read_roles = geoserver.layman_users_to_geoserver_roles(info['access_rights'][type])
                     geoserver.ensure_layer_security_roles(username, layer, security_read_roles, type[0], settings.LAYMAN_GS_AUTH)
-
-
-def upgrade():
-    logger.info(f'Checking all upgrades')
-    if older_than_1_8():
-        upgrade_1_8()
-    logger.info(f'Checking all upgrades DONE')

src/layman/upgrade/upgrade_v1_9.py

@@ -0,0 +1,32 @@
+import logging
+
+from layman import settings
+from layman.common.prime_db_schema import util as db_util
+DB_SCHEMA = settings.LAYMAN_PRIME_SCHEMA
+
+logger = logging.getLogger(__name__)
+
+
+def initialize_data_versioning():
+    logger.info(f'    Starting data versioning initialization')
+
+    sql_create_table = f'''CREATE TABLE IF NOT EXISTS {DB_SCHEMA}.data_version
+    (
+        major_version integer not null,
+        minor_version integer not null,
+        patch_version integer not null,
+        migration integer not null
+    )
+    TABLESPACE pg_default;'''
+    db_util.run_statement(sql_create_table)
+
+    # This table should have only one row and now should have none, otherwise something is wrong.
+    sql_select_count = f'''select count(*) from {DB_SCHEMA}.data_version'''
+    row_count = db_util.run_query(sql_select_count)[0][0]
+    assert row_count == 0
+
+    # Set initialization value to 0
+    sql_insert = f'''insert into {DB_SCHEMA}.data_version (major_version, minor_version, patch_version, migration) values (1, 9, 0, 0);'''
+    db_util.run_statement(sql_insert)
+
+    logger.info(f'    Data versioning initialization DONE')