Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat: Improve backup retention (for database backups) #4818

Merged
merged 13 commits into from
Jan 14, 2025
Merged

Feat: Improve backup retention (for database backups) #4818

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
04bcf01
feat: DB migration for Backup retention
peaklabs-dev Jan 13, 2025
e037ed7
feat: UI for backup retention settings
peaklabs-dev Jan 13, 2025
fb01aed
feat: new global s3 and local backup deletion function
peaklabs-dev Jan 13, 2025
a8b77b3
feat: use new backup deletion functions
peaklabs-dev Jan 13, 2025
f0d15af
chore: improve code
peaklabs-dev Jan 13, 2025
a5fadb0
fix: function calls
peaklabs-dev Jan 13, 2025
71edab4
fix: UI
peaklabs-dev Jan 13, 2025
c03b629
fix: deletion of single backup
peaklabs-dev Jan 13, 2025
3dfca4e
fix: backup job deletion - delete all backups from s3 and local
peaklabs-dev Jan 13, 2025
9eebeb9
fix: use new removeOldBackups function
peaklabs-dev Jan 13, 2025
3347eb3
fix: retention functions and folder deletion for local backups
peaklabs-dev Jan 13, 2025
e9f691b
fix: storage retention setting
peaklabs-dev Jan 13, 2025
8709a77
fix: db without s3 should still backup
peaklabs-dev Jan 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 3 additions & 14 deletions app/Jobs/DatabaseBackupJob.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,6 @@ public function handle(): void
throw new \Exception('Unsupported database type');
}
$size = $this->calculate_size();
$this->remove_old_backups();
if ($this->backup->save_s3) {
$this->upload_to_s3();
}
Expand All @@ -323,6 +322,9 @@ public function handle(): void
$this->team?->notify(new BackupFailed($this->backup, $this->database, $this->backup_output, $database));
}
}
if ($this->backup_log && $this->backup_log->status === 'success') {
removeOldBackups($this->backup);
}
} catch (\Throwable $e) {
throw $e;
} finally {
Expand Down Expand Up @@ -457,19 +459,6 @@ private function calculate_size()
return instant_remote_process(["du -b $this->backup_location | cut -f1"], $this->server, false);
}

private function remove_old_backups(): void
{
if ($this->backup->number_of_backups_locally === 0) {
$deletable = $this->backup->executions()->where('status', 'success');
} else {
$deletable = $this->backup->executions()->where('status', 'success')->skip($this->backup->number_of_backups_locally - 1);
}
foreach ($deletable->get() as $execution) {
delete_backup_locally($execution->filename, $this->server);
$execution->delete();
}
}

private function upload_to_s3(): void
{
try {
Expand Down
116 changes: 55 additions & 61 deletions app/Livewire/Project/Database/BackupEdit.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,23 @@ class BackupEdit extends Component
#[Validate(['string'])]
public string $timezone = '';

#[Validate(['required', 'integer', 'min:1'])]
public int $numberOfBackupsLocally = 1;
#[Validate(['required', 'integer'])]
public int $databaseBackupRetentionAmountLocally = 0;

#[Validate(['required', 'integer'])]
public ?int $databaseBackupRetentionDaysLocally = 0;

#[Validate(['required', 'numeric', 'min:0'])]
public ?float $databaseBackupRetentionMaxStorageLocally = 0;

#[Validate(['required', 'integer'])]
public ?int $databaseBackupRetentionAmountS3 = 0;

#[Validate(['required', 'integer'])]
public ?int $databaseBackupRetentionDaysS3 = 0;

#[Validate(['required', 'numeric', 'min:0'])]
public ?float $databaseBackupRetentionMaxStorageS3 = 0;

#[Validate(['required', 'boolean'])]
public bool $saveS3 = false;
Expand Down Expand Up @@ -73,7 +88,12 @@ public function syncData(bool $toModel = false)
if ($toModel) {
$this->backup->enabled = $this->backupEnabled;
$this->backup->frequency = $this->frequency;
$this->backup->number_of_backups_locally = $this->numberOfBackupsLocally;
$this->backup->database_backup_retention_amount_locally = $this->databaseBackupRetentionAmountLocally;
$this->backup->database_backup_retention_days_locally = $this->databaseBackupRetentionDaysLocally;
$this->backup->database_backup_retention_max_storage_locally = $this->databaseBackupRetentionMaxStorageLocally;
$this->backup->database_backup_retention_amount_s3 = $this->databaseBackupRetentionAmountS3;
$this->backup->database_backup_retention_days_s3 = $this->databaseBackupRetentionDaysS3;
$this->backup->database_backup_retention_max_storage_s3 = $this->databaseBackupRetentionMaxStorageS3;
$this->backup->save_s3 = $this->saveS3;
$this->backup->s3_storage_id = $this->s3StorageId;
$this->backup->databases_to_backup = $this->databasesToBackup;
Expand All @@ -84,7 +104,12 @@ public function syncData(bool $toModel = false)
$this->backupEnabled = $this->backup->enabled;
$this->frequency = $this->backup->frequency;
$this->timezone = data_get($this->backup->server(), 'settings.server_timezone', 'Instance timezone');
$this->numberOfBackupsLocally = $this->backup->number_of_backups_locally;
$this->databaseBackupRetentionAmountLocally = $this->backup->database_backup_retention_amount_locally;
$this->databaseBackupRetentionDaysLocally = $this->backup->database_backup_retention_days_locally;
$this->databaseBackupRetentionMaxStorageLocally = $this->backup->database_backup_retention_max_storage_locally;
$this->databaseBackupRetentionAmountS3 = $this->backup->database_backup_retention_amount_s3;
$this->databaseBackupRetentionDaysS3 = $this->backup->database_backup_retention_days_s3;
$this->databaseBackupRetentionMaxStorageS3 = $this->backup->database_backup_retention_max_storage_s3;
$this->saveS3 = $this->backup->save_s3;
$this->s3StorageId = $this->backup->s3_storage_id;
$this->databasesToBackup = $this->backup->databases_to_backup;
Expand All @@ -103,11 +128,29 @@ public function delete($password)
}

try {
if ($this->delete_associated_backups_locally) {
$this->deleteAssociatedBackupsLocally();
$server = null;
if ($this->backup->database instanceof \App\Models\ServiceDatabase) {
$server = $this->backup->database->service->destination->server;
} elseif ($this->backup->database->destination && $this->backup->database->destination->server) {
$server = $this->backup->database->destination->server;
}
if ($this->delete_associated_backups_s3) {
$this->deleteAssociatedBackupsS3();

$filenames = $this->backup->executions()
->whereNotNull('filename')
->where('filename', '!=', '')
->where('scheduled_database_backup_id', $this->backup->id)
->pluck('filename')
->filter()
->all();

if (! empty($filenames)) {
if ($this->delete_associated_backups_locally && $server) {
deleteBackupsLocally($filenames, $server);
}

if ($this->delete_associated_backups_s3 && $this->backup->s3) {
deleteBackupsS3($filenames, $this->backup->s3);
}
}

$this->backup->delete();
Expand All @@ -123,7 +166,9 @@ public function delete($password)
} else {
return redirect()->route('project.database.backup.index', $this->parameters);
}
} catch (\Throwable $e) {
} catch (\Exception $e) {
$this->dispatch('error', 'Failed to delete backup: '.$e->getMessage());

return handleError($e, $this);
}
}
Expand Down Expand Up @@ -160,63 +205,12 @@ public function submit()
}
}

private function deleteAssociatedBackupsLocally()
{
$executions = $this->backup->executions;
$backupFolder = null;

foreach ($executions as $execution) {
if ($this->backup->database->getMorphClass() === \App\Models\ServiceDatabase::class) {
$server = $this->backup->database->service->destination->server;
} else {
$server = $this->backup->database->destination->server;
}

if (! $backupFolder) {
$backupFolder = dirname($execution->filename);
}

delete_backup_locally($execution->filename, $server);
$execution->delete();
}

if (str($backupFolder)->isNotEmpty()) {
$this->deleteEmptyBackupFolder($backupFolder, $server);
}
}

private function deleteAssociatedBackupsS3()
{
// Add function to delete backups from S3
}

private function deleteAssociatedBackupsSftp()
{
// Add function to delete backups from SFTP
}

private function deleteEmptyBackupFolder($folderPath, $server)
{
$checkEmpty = instant_remote_process(["[ -z \"$(ls -A '$folderPath')\" ] && echo 'empty' || echo 'not empty'"], $server);

if (trim($checkEmpty) === 'empty') {
instant_remote_process(["rmdir '$folderPath'"], $server);

$parentFolder = dirname($folderPath);
$checkParentEmpty = instant_remote_process(["[ -z \"$(ls -A '$parentFolder')\" ] && echo 'empty' || echo 'not empty'"], $server);

if (trim($checkParentEmpty) === 'empty') {
instant_remote_process(["rmdir '$parentFolder'"], $server);
}
}
}

public function render()
{
return view('livewire.project.database.backup-edit', [
'checkboxes' => [
['id' => 'delete_associated_backups_locally', 'label' => __('database.delete_backups_locally')],
// ['id' => 'delete_associated_backups_s3', 'label' => 'All backups associated with this backup job from this database will be permanently deleted from the selected S3 Storage.']
['id' => 'delete_associated_backups_s3', 'label' => 'All backups associated with this backup job for this database will be permanently deleted from the selected S3 Storage.'],
// ['id' => 'delete_associated_backups_sftp', 'label' => 'All backups associated with this backup job from this database will be permanently deleted from the selected SFTP Storage.']
],
]);
Expand Down
36 changes: 19 additions & 17 deletions app/Livewire/Project/Database/BackupExecutions.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ class BackupExecutions extends Component

public $setDeletableBackup;

public $delete_backup_s3 = true;
public $delete_backup_s3 = false;

public $delete_backup_sftp = true;
public $delete_backup_sftp = false;

public function getListeners()
{
Expand Down Expand Up @@ -57,23 +57,25 @@ public function deleteBackup($executionId, $password)
return;
}

if ($execution->scheduledDatabaseBackup->database->getMorphClass() === \App\Models\ServiceDatabase::class) {
delete_backup_locally($execution->filename, $execution->scheduledDatabaseBackup->database->service->destination->server);
} else {
delete_backup_locally($execution->filename, $execution->scheduledDatabaseBackup->database->destination->server);
}
$server = $execution->scheduledDatabaseBackup->database->getMorphClass() === \App\Models\ServiceDatabase::class
? $execution->scheduledDatabaseBackup->database->service->destination->server
: $execution->scheduledDatabaseBackup->database->destination->server;

if ($this->delete_backup_s3) {
// Add logic to delete from S3
}
try {
if ($execution->filename) {
deleteBackupsLocally($execution->filename, $server);

if ($this->delete_backup_sftp) {
// Add logic to delete from SFTP
}
if ($this->delete_backup_s3 && $execution->scheduledDatabaseBackup->s3) {
deleteBackupsS3($execution->filename, $execution->scheduledDatabaseBackup->s3);
}
}

$execution->delete();
$this->dispatch('success', 'Backup deleted.');
$this->refreshBackupExecutions();
$execution->delete();
$this->dispatch('success', 'Backup deleted.');
$this->refreshBackupExecutions();
} catch (\Exception $e) {
$this->dispatch('error', 'Failed to delete backup: '.$e->getMessage());
}
}

public function download_file($exeuctionId)
Expand Down Expand Up @@ -143,7 +145,7 @@ public function render()
return view('livewire.project.database.backup-executions', [
'checkboxes' => [
['id' => 'delete_backup_s3', 'label' => 'Delete the selected backup permanently form S3 Storage'],
['id' => 'delete_backup_sftp', 'label' => 'Delete the selected backup permanently form SFTP Storage'],
// ['id' => 'delete_backup_sftp', 'label' => 'Delete the selected backup permanently form SFTP Storage'],
],
]);
}
Expand Down
Loading
Loading