From d85797163d3cc4af2bfc66be6c6de863363aefb7 Mon Sep 17 00:00:00 2001 From: index-git Date: Wed, 6 Jan 2021 17:40:00 +0100 Subject: [PATCH] Add data versioning --- src/layman/upgrade/__init__.py | 66 +++++++++++++++++++ src/layman/upgrade/upgrade_test.py | 37 +++++++++++ .../{upgrade.py => upgrade/upgrade_v1_8.py} | 13 +--- src/layman/upgrade/upgrade_v1_9.py | 32 +++++++++ 4 files changed, 138 insertions(+), 10 deletions(-) create mode 100644 src/layman/upgrade/__init__.py create mode 100644 src/layman/upgrade/upgrade_test.py rename src/layman/{upgrade.py => upgrade/upgrade_v1_8.py} (90%) create mode 100644 src/layman/upgrade/upgrade_v1_9.py diff --git a/src/layman/upgrade/__init__.py b/src/layman/upgrade/__init__.py new file mode 100644 index 000000000..4b2d4b916 --- /dev/null +++ b/src/layman/upgrade/__init__.py @@ -0,0 +1,66 @@ +import logging +import psycopg2 + +from layman.upgrade import upgrade_v1_8, upgrade_v1_9 +from layman import settings +from layman.common.prime_db_schema import util as db_util +DB_SCHEMA = settings.LAYMAN_PRIME_SCHEMA + +logger = logging.getLogger(__name__) + + +DATA_MIGRATIONS = [ + ((1, 9, 0), [upgrade_v1_9.initialize_data_versioning, ]), +] + + +def get_max_data_version(): + max_migration = max(DATA_MIGRATIONS, key=lambda x: x[0]) + return max_migration[0] + (len(max_migration[1]) - 1, ) + + +def get_current_data_version(): + # This table should have only one row (or none for the first time) + current_version = None + try: + sql_select = f'''select major_version, minor_version, patch_version, migration from {DB_SCHEMA}.data_version;''' + sql_result = db_util.run_query(sql_select, encapsulate_exception=False) + row_count = len(sql_result) + if row_count == 1: + current_version_sql = sql_result[0] + current_version = (current_version_sql[0], current_version_sql[1], current_version_sql[2], current_version_sql[3]) + elif row_count == 0: + current_version = (-1, -1, -1, -1) + else: + assert row_count == 1 + except psycopg2.errors.UndefinedTable: + current_version = (-1, -1, -1, -1) + return current_version + + +def upgrade(): + logger.info(f'Checking all upgrades') + if upgrade_v1_8.older_than_1_8(): + upgrade_v1_8.upgrade_1_8() + + current_data_version = get_current_data_version() + max_data_version = get_max_data_version() + logger.info(f' Current data version = {current_data_version}, Maximum data version = {max_data_version}') + migration_list_full = [(version + (index,), migration, ) + for version, migration_list in DATA_MIGRATIONS + for index, migration in enumerate(migration_list) + if version + (index,) > current_data_version] + migration_list_full.sort(key=lambda x: x[0]) + for version, migration in migration_list_full: + logger.info(f' Starting migration #{version[3]} for release v{version[0]}.{version[1]}.{version[2]}: {migration}') + migration() + sql_insert = f'''update {DB_SCHEMA}.data_version set + major_version = %s, + minor_version = %s, + patch_version = %s, + migration = %s;''' + db_util.run_statement(sql_insert, version) + + final_data_version = get_current_data_version() + assert final_data_version == max_data_version + logger.info(f' Checking all upgrades DONE, final data version = {final_data_version}') diff --git a/src/layman/upgrade/upgrade_test.py b/src/layman/upgrade/upgrade_test.py new file mode 100644 index 000000000..96f4a93e8 --- /dev/null +++ b/src/layman/upgrade/upgrade_test.py @@ -0,0 +1,37 @@ +import pytest + +from layman import upgrade, app, settings +from layman.common.prime_db_schema import util as db_util +DB_SCHEMA = settings.LAYMAN_PRIME_SCHEMA + + +def test_upgrade_run(): + with app.app_context(): + upgrade.upgrade() + + +@pytest.mark.parametrize('sql_command, expected_value', [ + (f'''DROP TABLE IF EXISTS {DB_SCHEMA}.data_version''', (-1, -1, -1, -1)), + (f'''DROP TABLE IF EXISTS {DB_SCHEMA}.data_version; CREATE TABLE IF NOT EXISTS {DB_SCHEMA}.data_version + ( + major_version integer not null, + minor_version integer not null, + patch_version integer not null, + migration integer not null + ) + TABLESPACE pg_default;''', (-1, -1, -1, -1)), + (f'''DROP TABLE IF EXISTS {DB_SCHEMA}.data_version; CREATE TABLE IF NOT EXISTS {DB_SCHEMA}.data_version + ( + major_version integer not null, + minor_version integer not null, + patch_version integer not null, + migration integer not null + ) + TABLESPACE pg_default; + insert into {DB_SCHEMA}.data_version (major_version, minor_version, patch_version, migration) values (1, 9, 0, 0); commit;''', (1, 9, 0, 0)), +]) +def test_get_current_data_version(sql_command, expected_value): + with app.app_context(): + db_util.run_statement(sql_command) + currrent_data_version = upgrade.get_current_data_version() + assert currrent_data_version == expected_value diff --git a/src/layman/upgrade.py b/src/layman/upgrade/upgrade_v1_8.py similarity index 90% rename from src/layman/upgrade.py rename to src/layman/upgrade/upgrade_v1_8.py index b6c433daf..53cc95018 100644 --- a/src/layman/upgrade.py +++ b/src/layman/upgrade/upgrade_v1_8.py @@ -1,11 +1,11 @@ +from urllib.parse import urljoin import logging import requests -from urllib.parse import urljoin from layman import app, settings +from layman.common import geoserver from layman.common.prime_db_schema import schema_initialization from layman.layer.util import get_layer_infos -from layman.common import geoserver logger = logging.getLogger(__name__) @@ -24,7 +24,7 @@ def upgrade_1_8(): settings.RIGHTS_EVERYONE_ROLE) logger.info(f' Ensuring users') - from .util import get_usernames, ensure_whole_user, check_username + from ..util import get_usernames, ensure_whole_user, check_username all_usernames = get_usernames() for username in all_usernames: logger.info(f' Ensuring user {username}') @@ -56,10 +56,3 @@ def upgrade_1_8(): for type in ['read', 'write']: security_read_roles = geoserver.layman_users_to_geoserver_roles(info['access_rights'][type]) geoserver.ensure_layer_security_roles(username, layer, security_read_roles, type[0], settings.LAYMAN_GS_AUTH) - - -def upgrade(): - logger.info(f'Checking all upgrades') - if older_than_1_8(): - upgrade_1_8() - logger.info(f'Checking all upgrades DONE') diff --git a/src/layman/upgrade/upgrade_v1_9.py b/src/layman/upgrade/upgrade_v1_9.py new file mode 100644 index 000000000..59ebcf7fe --- /dev/null +++ b/src/layman/upgrade/upgrade_v1_9.py @@ -0,0 +1,32 @@ +import logging + +from layman import settings +from layman.common.prime_db_schema import util as db_util +DB_SCHEMA = settings.LAYMAN_PRIME_SCHEMA + +logger = logging.getLogger(__name__) + + +def initialize_data_versioning(): + logger.info(f' Starting data versioning initialization') + + sql_create_table = f'''CREATE TABLE IF NOT EXISTS {DB_SCHEMA}.data_version + ( + major_version integer not null, + minor_version integer not null, + patch_version integer not null, + migration integer not null + ) + TABLESPACE pg_default;''' + db_util.run_statement(sql_create_table) + + # This table should have only one row and now should have none, otherwise something is wrong. + sql_select_count = f'''select count(*) from {DB_SCHEMA}.data_version''' + row_count = db_util.run_query(sql_select_count)[0][0] + assert row_count == 0 + + # Set initialization value to 0 + sql_insert = f'''insert into {DB_SCHEMA}.data_version (major_version, minor_version, patch_version, migration) values (1, 9, 0, 0);''' + db_util.run_statement(sql_insert) + + logger.info(f' Data versioning initialization DONE')