From 9a457ad2eb989765a3f3f360f3ce600e5fb0c3a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Safin?= Date: Thu, 14 Mar 2024 18:52:35 +0100 Subject: [PATCH 1/9] slightly change of get_new_backup_path and update code acordingly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rafał Safin --- backuper/backup_targets/mariadb.py | 5 +++-- backuper/backup_targets/mysql.py | 6 ++++-- backuper/backup_targets/postgresql.py | 6 ++++-- backuper/core.py | 6 ++---- tests/test_backup_target_mariadb.py | 12 ++++-------- tests/test_backup_target_mysql.py | 13 +++++-------- tests/test_backup_target_postgresql.py | 13 +++++-------- tests/test_core.py | 8 -------- 8 files changed, 27 insertions(+), 42 deletions(-) diff --git a/backuper/backup_targets/mariadb.py b/backuper/backup_targets/mariadb.py index 919ba9a..68f7f0a 100644 --- a/backuper/backup_targets/mariadb.py +++ b/backuper/backup_targets/mariadb.py @@ -93,8 +93,9 @@ def _mariadb_connection(self) -> str: def _backup(self) -> Path: escaped_dbname = core.safe_text_version(self.target_model.db) - name = f"{escaped_dbname}_{self.db_version}" - out_file = core.get_new_backup_path(self.env_name, name, sql=True) + escaped_version = core.safe_text_version(self.db_version) + name = f"{escaped_dbname}_{escaped_version}" + out_file = core.get_new_backup_path(self.env_name, name).with_suffix(".sql") db = shlex.quote(self.target_model.db) shell_mariadb_dump_db = ( f"mariadb-dump --defaults-file={self.option_file} " diff --git a/backuper/backup_targets/mysql.py b/backuper/backup_targets/mysql.py index 325cd17..7adc49f 100644 --- a/backuper/backup_targets/mysql.py +++ b/backuper/backup_targets/mysql.py @@ -93,8 +93,10 @@ def _mysql_connection(self) -> str: def _backup(self) -> Path: escaped_dbname = core.safe_text_version(self.target_model.db) - name = f"{escaped_dbname}_{self.db_version}" - out_file = core.get_new_backup_path(self.env_name, name, sql=True) + escaped_version = core.safe_text_version(self.db_version) + name = f"{escaped_dbname}_{escaped_version}" + + out_file = core.get_new_backup_path(self.env_name, name).with_suffix(".sql") db = shlex.quote(self.target_model.db) shell_mysqldump_db = ( diff --git a/backuper/backup_targets/postgresql.py b/backuper/backup_targets/postgresql.py index c11e60f..1b1ff3e 100644 --- a/backuper/backup_targets/postgresql.py +++ b/backuper/backup_targets/postgresql.py @@ -109,8 +109,10 @@ def _postgres_connection(self) -> str: def _backup(self) -> Path: escaped_dbname = core.safe_text_version(self.target_model.db) - name = f"{escaped_dbname}_{self.db_version}" - out_file = core.get_new_backup_path(self.env_name, name, sql=True) + escaped_version = core.safe_text_version(self.db_version) + name = f"{escaped_dbname}_{escaped_version}" + + out_file = core.get_new_backup_path(self.env_name, name).with_suffix(".sql") shell_pg_dump_db = ( f"pg_dump --clean --if-exists -v -O -d " f"{self.escaped_conn_uri} -f {out_file}" diff --git a/backuper/core.py b/backuper/core.py index d699ebe..2798a27 100644 --- a/backuper/core.py +++ b/backuper/core.py @@ -61,17 +61,15 @@ def remove_path(path: Path) -> None: shutil.rmtree(path=path) -def get_new_backup_path(env_name: str, name: str, sql: bool = False) -> Path: +def get_new_backup_path(env_name: str, name: str) -> Path: base_dir_path = config.CONST_BACKUP_FOLDER_PATH / env_name base_dir_path.mkdir(mode=0o700, exist_ok=True, parents=True) new_file = ( f"{env_name}_" f"{datetime.now(UTC).strftime('%Y%m%d_%H%M')}_" f"{name}_" - f"{secrets.token_urlsafe(3)}" + f"{secrets.token_urlsafe(6)}" ) - if sql: - new_file += ".sql" return base_dir_path / new_file diff --git a/tests/test_backup_target_mariadb.py b/tests/test_backup_target_mariadb.py index 59a3e02..07fcdac 100644 --- a/tests/test_backup_target_mariadb.py +++ b/tests/test_backup_target_mariadb.py @@ -36,19 +36,15 @@ def test_mariadb_connection_fail( @freeze_time("2022-12-11") @pytest.mark.parametrize("mariadb_target", ALL_MARIADB_DBS_TARGETS) -def test_run_mariadb_dump( - mariadb_target: MariaDBTargetModel, - monkeypatch: pytest.MonkeyPatch, -) -> None: - mock = Mock(return_value="fixed_dbname") - monkeypatch.setattr(core, "safe_text_version", mock) - +def test_run_mariadb_dump(mariadb_target: MariaDBTargetModel) -> None: db = MariaDB(target_model=mariadb_target) out_backup = db._backup() + escaped_name = "database_12" + escaped_version = db.db_version.replace(".", "") out_file = ( f"{db.env_name}/" - f"{db.env_name}_20221211_0000_fixed_dbname_{db.db_version}_{CONST_TOKEN_URLSAFE}.sql" + f"{db.env_name}_20221211_0000_{escaped_name}_{escaped_version}_{CONST_TOKEN_URLSAFE}.sql" ) out_path = config.CONST_BACKUP_FOLDER_PATH / out_file assert out_backup == out_path diff --git a/tests/test_backup_target_mysql.py b/tests/test_backup_target_mysql.py index 4405cdd..33c6274 100644 --- a/tests/test_backup_target_mysql.py +++ b/tests/test_backup_target_mysql.py @@ -32,19 +32,16 @@ def test_mysql_connection_fail( @freeze_time("2022-12-11") @pytest.mark.parametrize("mysql_target", ALL_MYSQL_DBS_TARGETS) -def test_run_mysqldump( - mysql_target: MySQLTargetModel, - monkeypatch: pytest.MonkeyPatch, -) -> None: - mock = Mock(return_value="fixed_dbname") - monkeypatch.setattr(core, "safe_text_version", mock) - +def test_run_mysqldump(mysql_target: MySQLTargetModel) -> None: db = MySQL(target_model=mysql_target) out_backup = db._backup() + escaped_name = "database_12" + escaped_version = db.db_version.replace(".", "") + out_file = ( f"{db.env_name}/" - f"{db.env_name}_20221211_0000_fixed_dbname_{db.db_version}_{CONST_TOKEN_URLSAFE}.sql" + f"{db.env_name}_20221211_0000_{escaped_name}_{escaped_version}_{CONST_TOKEN_URLSAFE}.sql" ) out_path = config.CONST_BACKUP_FOLDER_PATH / out_file assert out_backup == out_path diff --git a/tests/test_backup_target_postgresql.py b/tests/test_backup_target_postgresql.py index 6c3d01b..7e846da 100644 --- a/tests/test_backup_target_postgresql.py +++ b/tests/test_backup_target_postgresql.py @@ -38,19 +38,16 @@ def test_postgres_connection_fail( @freeze_time("2022-12-11") @pytest.mark.parametrize("postgres_target", ALL_POSTGRES_DBS_TARGETS) -def test_run_pg_dump( - postgres_target: PostgreSQLTargetModel, - monkeypatch: pytest.MonkeyPatch, -) -> None: - mock = Mock(return_value="fixed_dbname") - monkeypatch.setattr(core, "safe_text_version", mock) - +def test_run_pg_dump(postgres_target: PostgreSQLTargetModel) -> None: db = PostgreSQL(target_model=postgres_target) out_backup = db._backup() + escaped_name = "database_12" + escaped_version = db.db_version.replace(".", "") + out_file = ( f"{db.env_name}/" - f"{db.env_name}_20221211_0000_fixed_dbname_{db.db_version}_{CONST_TOKEN_URLSAFE}.sql" + f"{db.env_name}_20221211_0000_{escaped_name}_{escaped_version}_{CONST_TOKEN_URLSAFE}.sql" ) out_path = config.CONST_BACKUP_FOLDER_PATH / out_file assert out_backup == out_path diff --git a/tests/test_core.py b/tests/test_core.py index 4584958..712da9f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -54,14 +54,6 @@ def test_get_new_backup_path() -> None: assert str(new_path) == str(expected_path) -@freeze_time("2022-12-11") -def test_get_new_backup_path_sql() -> None: - new_path = core.get_new_backup_path("env_name", "db_string", sql=True) - expected_file = "env_name/env_name_20221211_0000_db_string_mock.sql" - expected_path = config.CONST_BACKUP_FOLDER_PATH / expected_file - assert str(new_path) == str(expected_path) - - @pytest.mark.parametrize("integrity", [True, False]) def test_run_create_zip_archive( tmp_path: Path, integrity: str, monkeypatch: pytest.MonkeyPatch From 88b7393c658d8cc1cf4202a06488879b0bb7eec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Safin?= Date: Thu, 14 Mar 2024 18:54:17 +0100 Subject: [PATCH 2/9] remove unused imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rafał Safin --- tests/test_backup_target_mariadb.py | 1 - tests/test_backup_target_mysql.py | 1 - tests/test_backup_target_postgresql.py | 1 - 3 files changed, 3 deletions(-) diff --git a/tests/test_backup_target_mariadb.py b/tests/test_backup_target_mariadb.py index 07fcdac..20e74b3 100644 --- a/tests/test_backup_target_mariadb.py +++ b/tests/test_backup_target_mariadb.py @@ -1,7 +1,6 @@ # Copyright: (c) 2024, Rafał Safin # GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) -from unittest.mock import Mock import pytest from freezegun import freeze_time diff --git a/tests/test_backup_target_mysql.py b/tests/test_backup_target_mysql.py index 33c6274..8e93e08 100644 --- a/tests/test_backup_target_mysql.py +++ b/tests/test_backup_target_mysql.py @@ -1,7 +1,6 @@ # Copyright: (c) 2024, Rafał Safin # GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) -from unittest.mock import Mock import pytest from freezegun import freeze_time diff --git a/tests/test_backup_target_postgresql.py b/tests/test_backup_target_postgresql.py index 7e846da..7533b64 100644 --- a/tests/test_backup_target_postgresql.py +++ b/tests/test_backup_target_postgresql.py @@ -1,7 +1,6 @@ # Copyright: (c) 2024, Rafał Safin # GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) -from unittest.mock import Mock import pytest from freezegun import freeze_time From 349a6e9527f3b063653e979282abded4f6430625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Safin?= Date: Thu, 14 Mar 2024 20:19:04 +0100 Subject: [PATCH 3/9] add unzip test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rafał Safin --- tests/test_core.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/test_core.py b/tests/test_core.py index 712da9f..53f8dc0 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -2,6 +2,7 @@ # GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) import os +import shlex from pathlib import Path from unittest.mock import Mock @@ -55,7 +56,7 @@ def test_get_new_backup_path() -> None: @pytest.mark.parametrize("integrity", [True, False]) -def test_run_create_zip_archive( +def test_run_create_zip_archive_out_path_exists( tmp_path: Path, integrity: str, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(config.options, "ZIP_SKIP_INTEGRITY_CHECK", integrity) @@ -68,6 +69,24 @@ def test_run_create_zip_archive( assert fake_backup_file_out.exists() +def test_run_create_zip_archive_can_be_unzipped_using_unzip(tmp_path: Path) -> None: + fake_backup_file = tmp_path / "test_archive" + + with open(fake_backup_file, "w") as f: + f.write("xxxąć”©#$%") + + archive_file = core.run_create_zip_archive(fake_backup_file) + fake_backup_file.unlink() + + passwd = shlex.quote(config.options.ZIP_ARCHIVE_PASSWORD.get_secret_value()) + shell_unzip = f"unzip -P {passwd} -d {tmp_path} {archive_file}" + + core.run_subprocess(shell_unzip) + + assert fake_backup_file.exists() + assert fake_backup_file.read_text() == "xxxąć”©#$%" + + test_data = [ ( [ From 85a59e13e5072483d11499c197468dc75ec64c92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Safin?= Date: Thu, 14 Mar 2024 20:37:54 +0100 Subject: [PATCH 4/9] add backup target file and folder tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rafał Safin --- tests/test_backup_target_file.py | 31 +++++++++++++++++++++++ tests/test_backup_target_folder.py | 34 ++++++++++++++++++++++++++ tests/test_backup_target_mariadb.py | 2 +- tests/test_backup_target_mysql.py | 2 +- tests/test_backup_target_postgresql.py | 2 +- 5 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 tests/test_backup_target_file.py create mode 100644 tests/test_backup_target_folder.py diff --git a/tests/test_backup_target_file.py b/tests/test_backup_target_file.py new file mode 100644 index 0000000..045afa2 --- /dev/null +++ b/tests/test_backup_target_file.py @@ -0,0 +1,31 @@ +# Copyright: (c) 2024, Rafał Safin +# GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) + + +from freezegun import freeze_time + +from backuper import config +from backuper.backup_targets.file import File + +from .conftest import CONST_TOKEN_URLSAFE, FILE_1 + + +@freeze_time("2024-03-14") +def test_run_file_backup_output_file_has_proper_name() -> None: + file = File(target_model=FILE_1) + out_backup = file.make_backup() + + file_name = FILE_1.abs_path.name + out_file = ( + f"{file.env_name}/" + f"{file.env_name}_20240314_0000_{file_name}_{CONST_TOKEN_URLSAFE}" + ) + out_path = config.CONST_BACKUP_FOLDER_PATH / out_file + assert out_backup == out_path + + +def test_run_file_backup_output_file_has_exact_same_content() -> None: + file = File(target_model=FILE_1) + out_backup = file.make_backup() + + assert out_backup.read_text() == FILE_1.abs_path.read_text() diff --git a/tests/test_backup_target_folder.py b/tests/test_backup_target_folder.py new file mode 100644 index 0000000..d91f07b --- /dev/null +++ b/tests/test_backup_target_folder.py @@ -0,0 +1,34 @@ +# Copyright: (c) 2024, Rafał Safin +# GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) + + +from freezegun import freeze_time + +from backuper import config +from backuper.backup_targets.folder import Folder + +from .conftest import CONST_TOKEN_URLSAFE, FOLDER_1 + + +@freeze_time("2024-03-14") +def test_run_folder_backup_output_folder_has_proper_name() -> None: + folder = Folder(target_model=FOLDER_1) + out_backup = folder.make_backup() + + folder_name = FOLDER_1.abs_path.name + out_file = ( + f"{folder.env_name}/" + f"{folder.env_name}_20240314_0000_{folder_name}_{CONST_TOKEN_URLSAFE}" + ) + out_path = config.CONST_BACKUP_FOLDER_PATH / out_file + assert out_backup == out_path + + +def test_run_folder_backup_output_file_in_folder_has_exact_same_content() -> None: + folder = Folder(target_model=FOLDER_1) + out_backup = folder.make_backup() + + file_in_folder = FOLDER_1.abs_path / "file.txt" + file_in_out_folder = out_backup / "file.txt" + + assert file_in_folder.read_text() == file_in_out_folder.read_text() diff --git a/tests/test_backup_target_mariadb.py b/tests/test_backup_target_mariadb.py index 20e74b3..305e451 100644 --- a/tests/test_backup_target_mariadb.py +++ b/tests/test_backup_target_mariadb.py @@ -37,7 +37,7 @@ def test_mariadb_connection_fail( @pytest.mark.parametrize("mariadb_target", ALL_MARIADB_DBS_TARGETS) def test_run_mariadb_dump(mariadb_target: MariaDBTargetModel) -> None: db = MariaDB(target_model=mariadb_target) - out_backup = db._backup() + out_backup = db.make_backup() escaped_name = "database_12" escaped_version = db.db_version.replace(".", "") diff --git a/tests/test_backup_target_mysql.py b/tests/test_backup_target_mysql.py index 8e93e08..0d60e17 100644 --- a/tests/test_backup_target_mysql.py +++ b/tests/test_backup_target_mysql.py @@ -33,7 +33,7 @@ def test_mysql_connection_fail( @pytest.mark.parametrize("mysql_target", ALL_MYSQL_DBS_TARGETS) def test_run_mysqldump(mysql_target: MySQLTargetModel) -> None: db = MySQL(target_model=mysql_target) - out_backup = db._backup() + out_backup = db.make_backup() escaped_name = "database_12" escaped_version = db.db_version.replace(".", "") diff --git a/tests/test_backup_target_postgresql.py b/tests/test_backup_target_postgresql.py index 7533b64..1a2a919 100644 --- a/tests/test_backup_target_postgresql.py +++ b/tests/test_backup_target_postgresql.py @@ -39,7 +39,7 @@ def test_postgres_connection_fail( @pytest.mark.parametrize("postgres_target", ALL_POSTGRES_DBS_TARGETS) def test_run_pg_dump(postgres_target: PostgreSQLTargetModel) -> None: db = PostgreSQL(target_model=postgres_target) - out_backup = db._backup() + out_backup = db.make_backup() escaped_name = "database_12" escaped_version = db.db_version.replace(".", "") From a33f9bbe10a08a892600b5b41176d088ba1d4f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Safin?= Date: Thu, 14 Mar 2024 20:47:39 +0100 Subject: [PATCH 5/9] escape folder and file names on backup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rafał Safin --- backuper/backup_targets/file.py | 6 +++--- backuper/backup_targets/folder.py | 6 +++--- backuper/backup_targets/mariadb.py | 2 ++ backuper/backup_targets/postgresql.py | 1 + tests/test_backup_target_file.py | 4 ++-- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/backuper/backup_targets/file.py b/backuper/backup_targets/file.py index 01fe462..b786426 100644 --- a/backuper/backup_targets/file.py +++ b/backuper/backup_targets/file.py @@ -17,9 +17,9 @@ def __init__(self, target_model: SingleFileTargetModel) -> None: self.target_model: SingleFileTargetModel = target_model def _backup(self) -> Path: - out_file = core.get_new_backup_path( - self.env_name, self.target_model.abs_path.name - ) + escaped_filename = core.safe_text_version(self.target_model.abs_path.name) + + out_file = core.get_new_backup_path(self.env_name, escaped_filename) shell_create_file_symlink = f"ln -s {self.target_model.abs_path} {out_file}" log.debug("start ln in subprocess: %s", shell_create_file_symlink) diff --git a/backuper/backup_targets/folder.py b/backuper/backup_targets/folder.py index e815278..52b0e0d 100644 --- a/backuper/backup_targets/folder.py +++ b/backuper/backup_targets/folder.py @@ -17,9 +17,9 @@ def __init__(self, target_model: DirectoryTargetModel) -> None: self.target_model: DirectoryTargetModel = target_model def _backup(self) -> Path: - out_file = core.get_new_backup_path( - self.env_name, self.target_model.abs_path.name - ) + escaped_foldername = core.safe_text_version(self.target_model.abs_path.name) + + out_file = core.get_new_backup_path(self.env_name, escaped_foldername) shell_create_dir_symlink = f"ln -s {self.target_model.abs_path} {out_file}" log.debug("start ln in subprocess: %s", shell_create_dir_symlink) diff --git a/backuper/backup_targets/mariadb.py b/backuper/backup_targets/mariadb.py index 68f7f0a..f17b6a1 100644 --- a/backuper/backup_targets/mariadb.py +++ b/backuper/backup_targets/mariadb.py @@ -95,7 +95,9 @@ def _backup(self) -> Path: escaped_dbname = core.safe_text_version(self.target_model.db) escaped_version = core.safe_text_version(self.db_version) name = f"{escaped_dbname}_{escaped_version}" + out_file = core.get_new_backup_path(self.env_name, name).with_suffix(".sql") + db = shlex.quote(self.target_model.db) shell_mariadb_dump_db = ( f"mariadb-dump --defaults-file={self.option_file} " diff --git a/backuper/backup_targets/postgresql.py b/backuper/backup_targets/postgresql.py index 1b1ff3e..b238556 100644 --- a/backuper/backup_targets/postgresql.py +++ b/backuper/backup_targets/postgresql.py @@ -113,6 +113,7 @@ def _backup(self) -> Path: name = f"{escaped_dbname}_{escaped_version}" out_file = core.get_new_backup_path(self.env_name, name).with_suffix(".sql") + shell_pg_dump_db = ( f"pg_dump --clean --if-exists -v -O -d " f"{self.escaped_conn_uri} -f {out_file}" diff --git a/tests/test_backup_target_file.py b/tests/test_backup_target_file.py index 045afa2..2f803b3 100644 --- a/tests/test_backup_target_file.py +++ b/tests/test_backup_target_file.py @@ -15,10 +15,10 @@ def test_run_file_backup_output_file_has_proper_name() -> None: file = File(target_model=FILE_1) out_backup = file.make_backup() - file_name = FILE_1.abs_path.name + escaped_file_name = FILE_1.abs_path.name.replace(".", "") out_file = ( f"{file.env_name}/" - f"{file.env_name}_20240314_0000_{file_name}_{CONST_TOKEN_URLSAFE}" + f"{file.env_name}_20240314_0000_{escaped_file_name}_{CONST_TOKEN_URLSAFE}" ) out_path = config.CONST_BACKUP_FOLDER_PATH / out_file assert out_backup == out_path From 8a21ee6cdf2c61ba2d5e6e00728f78f50b3f4e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Safin?= Date: Thu, 14 Mar 2024 23:10:21 +0100 Subject: [PATCH 6/9] make pydantic models frozen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rafał Safin --- backuper/models/backup_target_models.py | 3 +++ backuper/models/upload_provider_models.py | 4 +++- tests/test_backup_target_mariadb.py | 9 +++------ tests/test_backup_target_mysql.py | 9 +++------ tests/test_backup_target_postgresql.py | 9 +++------ 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/backuper/models/backup_target_models.py b/backuper/models/backup_target_models.py index 0de524e..bc22f9d 100644 --- a/backuper/models/backup_target_models.py +++ b/backuper/models/backup_target_models.py @@ -7,6 +7,7 @@ from croniter import croniter from pydantic import ( BaseModel, + ConfigDict, Field, SecretStr, field_validator, @@ -25,6 +26,8 @@ class TargetModel(BaseModel): ge=0, le=36600, default=config.options.BACKUP_MIN_RETENTION_DAYS ) + model_config = ConfigDict(frozen=True) + @field_validator("cron_rule") def cron_rule_is_valid(cls, cron_rule: str) -> str: if not croniter.is_valid(cron_rule): diff --git a/backuper/models/upload_provider_models.py b/backuper/models/upload_provider_models.py index 7b24d55..b543180 100644 --- a/backuper/models/upload_provider_models.py +++ b/backuper/models/upload_provider_models.py @@ -3,7 +3,7 @@ import base64 -from pydantic import BaseModel, SecretStr, field_validator +from pydantic import BaseModel, ConfigDict, SecretStr, field_validator from backuper import config @@ -11,6 +11,8 @@ class ProviderModel(BaseModel): name: str + model_config = ConfigDict(frozen=True) + class DebugProviderModel(ProviderModel): name: str = config.UploadProviderEnum.LOCAL_FILES_DEBUG diff --git a/tests/test_backup_target_mariadb.py b/tests/test_backup_target_mariadb.py index 305e451..ba3dcc7 100644 --- a/tests/test_backup_target_mariadb.py +++ b/tests/test_backup_target_mariadb.py @@ -23,14 +23,11 @@ def test_mariadb_connection_success(mariadb_target: MariaDBTargetModel) -> None: @pytest.mark.parametrize("mariadb_target", ALL_MARIADB_DBS_TARGETS) -def test_mariadb_connection_fail( - mariadb_target: MariaDBTargetModel, - monkeypatch: pytest.MonkeyPatch, -) -> None: +def test_mariadb_connection_fail(mariadb_target: MariaDBTargetModel) -> None: with pytest.raises(core.CoreSubprocessError): # simulate not existing db port 9999 and connection err - monkeypatch.setattr(mariadb_target, "port", 9999) - MariaDB(target_model=mariadb_target) + target_model = mariadb_target.model_copy(update={"port": 9999}) + MariaDB(target_model=target_model) @freeze_time("2022-12-11") diff --git a/tests/test_backup_target_mysql.py b/tests/test_backup_target_mysql.py index 0d60e17..9e1c015 100644 --- a/tests/test_backup_target_mysql.py +++ b/tests/test_backup_target_mysql.py @@ -19,14 +19,11 @@ def test_mysql_connection_success(mysql_target: MySQLTargetModel) -> None: @pytest.mark.parametrize("mysql_target", ALL_MYSQL_DBS_TARGETS) -def test_mysql_connection_fail( - mysql_target: MySQLTargetModel, - monkeypatch: pytest.MonkeyPatch, -) -> None: +def test_mysql_connection_fail(mysql_target: MySQLTargetModel) -> None: with pytest.raises(core.CoreSubprocessError): # simulate not existing db port 9999 and connection err - monkeypatch.setattr(mysql_target, "port", 9999) - MySQL(target_model=mysql_target) + target_model = mysql_target.model_copy(update={"port": 9999}) + MySQL(target_model=target_model) @freeze_time("2022-12-11") diff --git a/tests/test_backup_target_postgresql.py b/tests/test_backup_target_postgresql.py index 1a2a919..53e3c74 100644 --- a/tests/test_backup_target_postgresql.py +++ b/tests/test_backup_target_postgresql.py @@ -25,14 +25,11 @@ def test_postgres_connection_success( @pytest.mark.parametrize("postgres_target", ALL_POSTGRES_DBS_TARGETS) -def test_postgres_connection_fail( - postgres_target: PostgreSQLTargetModel, - monkeypatch: pytest.MonkeyPatch, -) -> None: +def test_postgres_connection_fail(postgres_target: PostgreSQLTargetModel) -> None: with pytest.raises(core.CoreSubprocessError): # simulate not existing db port 9999 and connection err - monkeypatch.setattr(postgres_target, "port", 9999) - PostgreSQL(target_model=postgres_target) + target_model = postgres_target.model_copy(update={"port": 9999}) + PostgreSQL(target_model=target_model) @freeze_time("2022-12-11") From 171f7fbd23304cf53951f38896f2b05b9a127507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Safin?= Date: Fri, 15 Mar 2024 00:08:50 +0100 Subject: [PATCH 7/9] add postgres end to end test with dump and restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rafał Safin --- tests/test_backup_target_postgresql.py | 63 ++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/test_backup_target_postgresql.py b/tests/test_backup_target_postgresql.py index 53e3c74..9f29575 100644 --- a/tests/test_backup_target_postgresql.py +++ b/tests/test_backup_target_postgresql.py @@ -2,6 +2,8 @@ # GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) +import shlex + import pytest from freezegun import freeze_time @@ -47,3 +49,64 @@ def test_run_pg_dump(postgres_target: PostgreSQLTargetModel) -> None: ) out_path = config.CONST_BACKUP_FOLDER_PATH / out_file assert out_backup == out_path + + +@pytest.mark.parametrize("postgres_target", ALL_POSTGRES_DBS_TARGETS) +def test_end_to_end_successful_restore_after_backup( + postgres_target: PostgreSQLTargetModel, +) -> None: + db = PostgreSQL(target_model=postgres_target) + core.run_subprocess( + f"psql -d {db.escaped_conn_uri} -w --command " + "'DROP DATABASE IF EXISTS test_db;'", + ) + core.run_subprocess( + f"psql -d {db.escaped_conn_uri} -w --command 'CREATE DATABASE test_db;'", + ) + + test_db_target = postgres_target.model_copy(update={"db": "test_db"}) + test_db = PostgreSQL(target_model=test_db_target) + + table_query = ( + "CREATE TABLE my_table " + "(id SERIAL PRIMARY KEY, " + "name VARCHAR (50) UNIQUE NOT NULL, " + "age INTEGER);" + ) + core.run_subprocess( + f"psql -d {test_db.escaped_conn_uri} -w --command '{table_query}'", + ) + + insert_query = shlex.quote( + "INSERT INTO my_table (name, age) " + "VALUES ('Geralt z Rivii', 60),('rafsaf', 24);" + ) + + core.run_subprocess( + f"psql -d {test_db.escaped_conn_uri} -w --command {insert_query}", + ) + + test_db_backup = test_db.make_backup() + + core.run_subprocess( + f"psql -d {db.escaped_conn_uri} -w --command 'DROP DATABASE test_db;'", + ) + core.run_subprocess( + f"psql -d {db.escaped_conn_uri} -w --command 'CREATE DATABASE test_db;'", + ) + + core.run_subprocess( + f"psql -d {test_db.escaped_conn_uri} -w < {test_db_backup}", + ) + + result = core.run_subprocess( + f"psql -d {test_db.escaped_conn_uri} -w --command 'select * from my_table;'", + ) + + assert result == ( + " id | name | age \n" + "----+----------------+-----\n" + " 1 | Geralt z Rivii | 60\n" + " 2 | rafsaf | 24\n" + "(2 rows)\n\n" + ) From ccc84c73f85ee2b9e02717b23719194c004f7ae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Safin?= Date: Fri, 15 Mar 2024 00:42:21 +0100 Subject: [PATCH 8/9] add mariadb end to end dump restore test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rafał Safin --- backuper/backup_targets/mariadb.py | 7 ++- tests/test_backup_target_mariadb.py | 72 +++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/backuper/backup_targets/mariadb.py b/backuper/backup_targets/mariadb.py index f17b6a1..195945d 100644 --- a/backuper/backup_targets/mariadb.py +++ b/backuper/backup_targets/mariadb.py @@ -24,6 +24,7 @@ class MariaDB(BaseBackupTarget): def __init__(self, target_model: MariaDBTargetModel) -> None: super().__init__(target_model) self.target_model: MariaDBTargetModel = target_model + self.db_name = shlex.quote(self.target_model.db) self.option_file: Path = self._init_option_file() self.db_version: str = self._mariadb_connection() @@ -65,9 +66,8 @@ def _mariadb_connection(self) -> str: raise log.debug("start mariadb connection") try: - db = shlex.quote(self.target_model.db) result = core.run_subprocess( - f"mariadb --defaults-file={self.option_file} {db} " + f"mariadb --defaults-file={self.option_file} {self.db_name} " f"--execute='SELECT version();'", ) except core.CoreSubprocessError as conn_err: @@ -98,10 +98,9 @@ def _backup(self) -> Path: out_file = core.get_new_backup_path(self.env_name, name).with_suffix(".sql") - db = shlex.quote(self.target_model.db) shell_mariadb_dump_db = ( f"mariadb-dump --defaults-file={self.option_file} " - f"--result-file={out_file} --verbose {db}" + f"--result-file={out_file} --verbose {self.db_name}" ) log.debug("start mariadbdump in subprocess: %s", shell_mariadb_dump_db) core.run_subprocess(shell_mariadb_dump_db) diff --git a/tests/test_backup_target_mariadb.py b/tests/test_backup_target_mariadb.py index ba3dcc7..ce89dff 100644 --- a/tests/test_backup_target_mariadb.py +++ b/tests/test_backup_target_mariadb.py @@ -2,8 +2,11 @@ # GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) +import shlex + import pytest from freezegun import freeze_time +from pydantic import SecretStr from backuper import config, core from backuper.backup_targets.mariadb import MariaDB @@ -44,3 +47,72 @@ def test_run_mariadb_dump(mariadb_target: MariaDBTargetModel) -> None: ) out_path = config.CONST_BACKUP_FOLDER_PATH / out_file assert out_backup == out_path + + +@pytest.mark.parametrize("mariadb_target", ALL_MARIADB_DBS_TARGETS) +def test_end_to_end_successful_restore_after_backup( + mariadb_target: MariaDBTargetModel, +) -> None: + root_target_model = mariadb_target.model_copy( + update={ + "user": "root", + "password": SecretStr(f"root-{mariadb_target.password.get_secret_value()}"), + } + ) + + db = MariaDB(target_model=root_target_model) + core.run_subprocess( + f"mariadb --defaults-file={db.option_file} {db.db_name} --execute=" + "'DROP DATABASE IF EXISTS test_db;'", + ) + core.run_subprocess( + f"mariadb --defaults-file={db.option_file} {db.db_name} --execute=" + "'CREATE DATABASE test_db;'", + ) + + test_db_target = root_target_model.model_copy(update={"db": "test_db"}) + test_db = MariaDB(target_model=test_db_target) + + table_query = ( + "CREATE TABLE my_table " + "(id SERIAL PRIMARY KEY, " + "name VARCHAR (50) UNIQUE NOT NULL, " + "age INTEGER);" + ) + core.run_subprocess( + f"mariadb --defaults-file={test_db.option_file} {test_db.db_name} " + f"--execute='{table_query}'", + ) + + insert_query = shlex.quote( + "INSERT INTO my_table (name, age) " + "VALUES ('Geralt z Rivii', 60),('rafsaf', 24);" + ) + + core.run_subprocess( + f"mariadb --defaults-file={test_db.option_file} {test_db.db_name} " + f"--execute={insert_query}", + ) + + test_db_backup = test_db.make_backup() + + core.run_subprocess( + f"mariadb --defaults-file={db.option_file} {db.db_name} --execute=" + "'DROP DATABASE test_db;'", + ) + core.run_subprocess( + f"mariadb --defaults-file={db.option_file} {db.db_name} --execute=" + "'CREATE DATABASE test_db;'", + ) + + core.run_subprocess( + f"mariadb --defaults-file={test_db.option_file} {test_db.db_name}" + f" < {test_db_backup}", + ) + + result = core.run_subprocess( + f"mariadb --defaults-file={test_db.option_file} {test_db.db_name}" + " --execute='select * from my_table;'", + ) + + assert result == ("id\tname\tage\n" "1\tGeralt z Rivii\t60\n" "2\trafsaf\t24\n") From 69c0b570133ad6df2a6be345b3783f08e7c72ebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Safin?= Date: Fri, 15 Mar 2024 00:48:13 +0100 Subject: [PATCH 9/9] add mysql dump and restore end to end test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rafał Safin --- backuper/backup_targets/mysql.py | 7 ++- tests/test_backup_target_mariadb.py | 2 +- tests/test_backup_target_mysql.py | 72 ++++++++++++++++++++++++++ tests/test_backup_target_postgresql.py | 3 +- 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/backuper/backup_targets/mysql.py b/backuper/backup_targets/mysql.py index 7adc49f..825c300 100644 --- a/backuper/backup_targets/mysql.py +++ b/backuper/backup_targets/mysql.py @@ -24,6 +24,7 @@ class MySQL(BaseBackupTarget): def __init__(self, target_model: MySQLTargetModel) -> None: super().__init__(target_model) self.target_model: MySQLTargetModel = target_model + self.db_name = shlex.quote(self.target_model.db) self.option_file: Path = self._init_option_file() self.db_version: str = self._mysql_connection() @@ -65,9 +66,8 @@ def _mysql_connection(self) -> str: raise log.debug("start mysql connection") try: - db = shlex.quote(self.target_model.db) result = core.run_subprocess( - f"mariadb --defaults-file={self.option_file} {db} " + f"mariadb --defaults-file={self.option_file} {self.db_name} " "--execute='SELECT version();'", ) except core.CoreSubprocessError as err: @@ -98,10 +98,9 @@ def _backup(self) -> Path: out_file = core.get_new_backup_path(self.env_name, name).with_suffix(".sql") - db = shlex.quote(self.target_model.db) shell_mysqldump_db = ( f"mariadb-dump --defaults-file={self.option_file} " - f"--result-file={out_file} --verbose {db}" + f"--result-file={out_file} --verbose {self.db_name}" ) log.debug("start mysqldump in subprocess: %s", shell_mysqldump_db) core.run_subprocess(shell_mysqldump_db) diff --git a/tests/test_backup_target_mariadb.py b/tests/test_backup_target_mariadb.py index ce89dff..7409513 100644 --- a/tests/test_backup_target_mariadb.py +++ b/tests/test_backup_target_mariadb.py @@ -112,7 +112,7 @@ def test_end_to_end_successful_restore_after_backup( result = core.run_subprocess( f"mariadb --defaults-file={test_db.option_file} {test_db.db_name}" - " --execute='select * from my_table;'", + " --execute='select * from my_table order by id asc;'", ) assert result == ("id\tname\tage\n" "1\tGeralt z Rivii\t60\n" "2\trafsaf\t24\n") diff --git a/tests/test_backup_target_mysql.py b/tests/test_backup_target_mysql.py index 9e1c015..3135092 100644 --- a/tests/test_backup_target_mysql.py +++ b/tests/test_backup_target_mysql.py @@ -2,8 +2,11 @@ # GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) +import shlex + import pytest from freezegun import freeze_time +from pydantic import SecretStr from backuper import config, core from backuper.backup_targets.mysql import MySQL @@ -41,3 +44,72 @@ def test_run_mysqldump(mysql_target: MySQLTargetModel) -> None: ) out_path = config.CONST_BACKUP_FOLDER_PATH / out_file assert out_backup == out_path + + +@pytest.mark.parametrize("mysql_target", ALL_MYSQL_DBS_TARGETS) +def test_end_to_end_successful_restore_after_backup( + mysql_target: MySQLTargetModel, +) -> None: + root_target_model = mysql_target.model_copy( + update={ + "user": "root", + "password": SecretStr(f"root-{mysql_target.password.get_secret_value()}"), + } + ) + + db = MySQL(target_model=root_target_model) + core.run_subprocess( + f"mariadb --defaults-file={db.option_file} {db.db_name} --execute=" + "'DROP DATABASE IF EXISTS test_db;'", + ) + core.run_subprocess( + f"mariadb --defaults-file={db.option_file} {db.db_name} --execute=" + "'CREATE DATABASE test_db;'", + ) + + test_db_target = root_target_model.model_copy(update={"db": "test_db"}) + test_db = MySQL(target_model=test_db_target) + + table_query = ( + "CREATE TABLE my_table " + "(id SERIAL PRIMARY KEY, " + "name VARCHAR (50) UNIQUE NOT NULL, " + "age INTEGER);" + ) + core.run_subprocess( + f"mariadb --defaults-file={test_db.option_file} {test_db.db_name} " + f"--execute='{table_query}'", + ) + + insert_query = shlex.quote( + "INSERT INTO my_table (name, age) " + "VALUES ('Geralt z Rivii', 60),('rafsaf', 24);" + ) + + core.run_subprocess( + f"mariadb --defaults-file={test_db.option_file} {test_db.db_name} " + f"--execute={insert_query}", + ) + + test_db_backup = test_db.make_backup() + + core.run_subprocess( + f"mariadb --defaults-file={db.option_file} {db.db_name} --execute=" + "'DROP DATABASE test_db;'", + ) + core.run_subprocess( + f"mariadb --defaults-file={db.option_file} {db.db_name} --execute=" + "'CREATE DATABASE test_db;'", + ) + + core.run_subprocess( + f"mariadb --defaults-file={test_db.option_file} {test_db.db_name}" + f" < {test_db_backup}", + ) + + result = core.run_subprocess( + f"mariadb --defaults-file={test_db.option_file} {test_db.db_name}" + " --execute='select * from my_table order by id asc;'", + ) + + assert result == ("id\tname\tage\n" "1\tGeralt z Rivii\t60\n" "2\trafsaf\t24\n") diff --git a/tests/test_backup_target_postgresql.py b/tests/test_backup_target_postgresql.py index 9f29575..b76d201 100644 --- a/tests/test_backup_target_postgresql.py +++ b/tests/test_backup_target_postgresql.py @@ -100,7 +100,8 @@ def test_end_to_end_successful_restore_after_backup( ) result = core.run_subprocess( - f"psql -d {test_db.escaped_conn_uri} -w --command 'select * from my_table;'", + f"psql -d {test_db.escaped_conn_uri} -w --command " + "'select * from my_table order by id asc;'", ) assert result == (