From 53918a64732288428695ecaaf3d7d638100de564 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 15 Jan 2024 08:45:12 +0000 Subject: [PATCH 1/4] lib/ZnapZend.pm: createSnapshot(): be pedantic about "implicit value of org.znapzend:enabled" for sub-datasets Signed-off-by: Jim Klimov --- lib/ZnapZend.pm | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/ZnapZend.pm b/lib/ZnapZend.pm index 61478920..2695b17a 100644 --- a/lib/ZnapZend.pm +++ b/lib/ZnapZend.pm @@ -1054,7 +1054,15 @@ my $createSnapshot = sub { # restrict the list to the datasets that are descendant from the current ###my @dataSetList = grep /^$backupSet->{src}($|\/)/, @{$self->zZfs->listDataSets()}; my @dataSetList = @{$self->zZfs->listDataSets(undef, $backupSet->{src}, 1)}; + if ( @dataSetList ) { + # the default sub-dataset enablement value is implicitly "on" + # (technically, the value inherited from $backupSet which we + # are currently processing, because it is enabled) + my $enabled_default = 'on'; + if (defined($backupSet->{enabled})) { + $enabled_default = $backupSet->{enabled}; + } # for each dataset: if the property "enabled" is set to "off", set the # newly created snapshot for removal @@ -1066,9 +1074,9 @@ my $createSnapshot = sub { print STDERR '# ' . join(' ', @cmd) . "\n" if $self->debug; open my $prop, '-|', @cmd; - # if the property does not exist, the command will just return. In this case, - # the value is implicit "on" - $prop = <$prop> || "on"; + # if the property does not exist, the command will just return. + # In this case, use the default determined above. + $prop = <$prop> || $enabled_default; chomp($prop); if ( $prop eq 'off' ) { push(@dataSetsExplicitlyDisabled, $dataSet . '@' . $snapshotSuffix); From 288edc28edd81517b72cd8ff275a913e44074474 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 15 Jan 2024 08:53:56 +0000 Subject: [PATCH 2/4] lib/ZnapZend.pm: createSnapshot(): update comments about creation and cleanup of snapshots on enabled=off sub-datasets Signed-off-by: Jim Klimov --- lib/ZnapZend.pm | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/ZnapZend.pm b/lib/ZnapZend.pm index 2695b17a..fee5b251 100644 --- a/lib/ZnapZend.pm +++ b/lib/ZnapZend.pm @@ -1042,10 +1042,13 @@ my $createSnapshot = sub { } } - # Remove snapshots from descendant subsystems that have the property - # "enabled" set to "off", if the "recursive" flag is set to "on", - # so their newly created snapshots are discarded quickly and disk - # space is not abused by something we do not back up subsequently. + # First we snapshot the backupSet recursively and atomically, but + # later we would remove snapshots from descendant subsystems that + # have the property "enabled" set to "off", if the "recursive" flag + # is set to "on", so their newly created snapshots are discarded + # quickly and disk space is not abused by something we do not back + # up subsequently (it might still become blocked if znapzend crashes + # or the host reboots between such snapshot creation and clean-up). # This only applies if we made a single-command recursive snapshot. if ($backupSet->{recursive} eq 'on') { @@ -1064,7 +1067,7 @@ my $createSnapshot = sub { $enabled_default = $backupSet->{enabled}; } - # for each dataset: if the property "enabled" is set to "off", set the + # for each sub-dataset: if the property "enabled" is set to "off", set the # newly created snapshot for removal my @dataSetsExplicitlyDisabled = (); for my $dataSet (@dataSetList){ @@ -1083,12 +1086,12 @@ my $createSnapshot = sub { } } - # remove the snapshots previously marked - # removal here is non-recursive to allow for fine-grained control - if ( @dataSetsExplicitlyDisabled ){ - $self->zLog->info("Requesting removal of marked datasets: ". join( ", ", @dataSetsExplicitlyDisabled)); - $self->zZfs->destroySnapshots(\@dataSetsExplicitlyDisabled, 0); - } + # Remove the snapshots previously marked (if any). Note that the + # removal here is non-recursive to allow for fine-grained control: + if (@dataSetsExplicitlyDisabled) { + $self->zLog->info("Requesting removal of marked datasets: ". join( ", ", @dataSetsExplicitlyDisabled)); + $self->zZfs->destroySnapshots(\@dataSetsExplicitlyDisabled, 0); + } } } From 60c4cb7fc96a850acce0b8409ddf932a9457cad1 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 15 Jan 2024 09:26:14 +0000 Subject: [PATCH 3/4] lib/ZnapZend.pm: refactor listDisabledSourceDescendants() out of createSnapshot() so it can also be used in sendRecvCleanup() Signed-off-by: Jim Klimov --- lib/ZnapZend.pm | 109 +++++++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 43 deletions(-) diff --git a/lib/ZnapZend.pm b/lib/ZnapZend.pm index fee5b251..51fe5c0e 100644 --- a/lib/ZnapZend.pm +++ b/lib/ZnapZend.pm @@ -145,6 +145,58 @@ my $killThemAll = sub { exit 0; }; +# Return an array of dataset names which are local descendants of the +# provided $backupSet->{src} and have an explicit org.znapzend:enabled=off +# (note that currently each disabled descendant must be explicit - maybe +# an also-explicit recursion handling is a good idea for the future). +# The array may be empty if not in recursive mode, or no descendants +# exist, or none are disabled. +my $listDisabledSourceDescendants = sub { + my $self = shift; + my $backupSet = shift; + + #no HUP handler in child + $SIG{HUP} = 'IGNORE'; + + my @dataSetsExplicitlyDisabled = (); + if ($backupSet->{recursive} eq 'on') { + $self->zLog->info("checking for explicitly excluded ZFS dependent datasets under '$backupSet->{src}'"); + + # restrict the list to the datasets that are descendant from the current + ###my @dataSetList = grep /^$backupSet->{src}($|\/)/, @{$self->zZfs->listDataSets()}; + my @dataSetList = @{$self->zZfs->listDataSets(undef, $backupSet->{src}, 1)}; + + if ( @dataSetList ) { + # the default sub-dataset enablement value is implicitly "on" + # (technically, the value inherited from $backupSet which we + # are currently processing, because it is enabled) + my $enabled_default = 'on'; + if (defined($backupSet->{enabled})) { + $enabled_default = $backupSet->{enabled}; + } + + # for each sub-dataset: if the property "enabled" is set to "off", set the + # newly created snapshot for removal + for my $dataSet (@dataSetList){ + # get the value for org.znapzend property + my @cmd = (@{$self->zZfs->priv}, qw(zfs get -H -s local -o value org.znapzend:enabled), $dataSet); + print STDERR '# ' . join(' ', @cmd) . "\n" if $self->debug; + open my $prop, '-|', @cmd; + + # if the property does not exist, the command will just return. + # In this case, use the default determined above. + $prop = <$prop> || $enabled_default; + chomp($prop); + if ( $prop eq 'off' ) { + push(@dataSetsExplicitlyDisabled, $dataSet); + } + } + } + } + + return @dataSetsExplicitlyDisabled; +}; + my $refreshBackupPlans = sub { my $self = shift; my $recurse = shift; @@ -301,6 +353,9 @@ my $sendRecvCleanup = sub { my $srcSubDataSets = $backupSet->{recursive} eq 'on' ? $self->zZfs->listSubDataSets($backupSet->{src}) : [ $backupSet->{src} ]; + my @dataSetsExplicitlyDisabled = $self->$listDisabledSourceDescendants($backupSet); + $self->zLog->info('dataSetsExplicitlyDisabled: ' . @dataSetsExplicitlyDisabled); + #loop through all destinations for my $dst (sort grep { /^dst_[^_]+$/ } keys %$backupSet){ my ($key) = $dst =~ /dst_([^_]+)$/; @@ -379,7 +434,7 @@ my $sendRecvCleanup = sub { my $dstDataSet = $srcDataSet; $dstDataSet =~ s/^\Q$backupSet->{src}\E/$backupSet->{$dst}/; - $self->zLog->debug('sending snapshots from ' . $srcDataSet . ' to ' . $dstDataSet); + $self->zLog->debug('sending snapshots from ' . $srcDataSet . ' to ' . $dstDataSet . ((grep (/^\Q$srcDataSet\E$/, @dataSetsExplicitlyDisabled)) ? ": not enabled, should be skipped" : "")); { local $@; eval { @@ -1050,49 +1105,17 @@ my $createSnapshot = sub { # up subsequently (it might still become blocked if znapzend crashes # or the host reboots between such snapshot creation and clean-up). # This only applies if we made a single-command recursive snapshot. - if ($backupSet->{recursive} eq 'on') { - - $self->zLog->info("checking for explicitly excluded ZFS dependent datasets under '$backupSet->{src}'"); - - # restrict the list to the datasets that are descendant from the current - ###my @dataSetList = grep /^$backupSet->{src}($|\/)/, @{$self->zZfs->listDataSets()}; - my @dataSetList = @{$self->zZfs->listDataSets(undef, $backupSet->{src}, 1)}; - - if ( @dataSetList ) { - # the default sub-dataset enablement value is implicitly "on" - # (technically, the value inherited from $backupSet which we - # are currently processing, because it is enabled) - my $enabled_default = 'on'; - if (defined($backupSet->{enabled})) { - $enabled_default = $backupSet->{enabled}; - } - - # for each sub-dataset: if the property "enabled" is set to "off", set the - # newly created snapshot for removal - my @dataSetsExplicitlyDisabled = (); - for my $dataSet (@dataSetList){ - - # get the value for org.znapzend property - my @cmd = (@{$self->zZfs->priv}, qw(zfs get -H -s local -o value org.znapzend:enabled), $dataSet); - print STDERR '# ' . join(' ', @cmd) . "\n" if $self->debug; - open my $prop, '-|', @cmd; - - # if the property does not exist, the command will just return. - # In this case, use the default determined above. - $prop = <$prop> || $enabled_default; - chomp($prop); - if ( $prop eq 'off' ) { - push(@dataSetsExplicitlyDisabled, $dataSet . '@' . $snapshotSuffix); - } - } - - # Remove the snapshots previously marked (if any). Note that the - # removal here is non-recursive to allow for fine-grained control: - if (@dataSetsExplicitlyDisabled) { - $self->zLog->info("Requesting removal of marked datasets: ". join( ", ", @dataSetsExplicitlyDisabled)); - $self->zZfs->destroySnapshots(\@dataSetsExplicitlyDisabled, 0); - } + my @dataSetsExplicitlyDisabled = $self->$listDisabledSourceDescendants($backupSet); + + # Remove the snapshots previously marked (if any). Note that the + # removal here is non-recursive to allow for fine-grained control: + if (@dataSetsExplicitlyDisabled) { + my @snapshotsToRemove = (); + for my $dataSet (@dataSetsExplicitlyDisabled) { + push (@snapshotsToRemove, $dataSet . '@' . $snapshotSuffix); } + $self->zLog->info("Requesting removal of marked snapshots from \"disabled\" datasets: ". join( ", ", @snapshotsToRemove)); + $self->zZfs->destroySnapshots(\@snapshotsToRemove, 0); } #clean up env variables From 942b096cc54f4131b1cae181ce271fb56b3f1dab Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 15 Jan 2024 09:56:51 +0000 Subject: [PATCH 4/4] lib/ZnapZend.pm: refactor listDisabledSourceDescendants() calls to be done in refreshBackupPlans() once Track the list of names as @{$backupSet->{srcDisabledDescendants}} Signed-off-by: Jim Klimov --- lib/ZnapZend.pm | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/ZnapZend.pm b/lib/ZnapZend.pm index 51fe5c0e..260585ab 100644 --- a/lib/ZnapZend.pm +++ b/lib/ZnapZend.pm @@ -194,7 +194,7 @@ my $listDisabledSourceDescendants = sub { } } - return @dataSetsExplicitlyDisabled; + return \@dataSetsExplicitlyDisabled; }; my $refreshBackupPlans = sub { @@ -212,6 +212,13 @@ my $refreshBackupPlans = sub { or die "No backup set defined or enabled, yet. run 'znapzendzetup' to setup znapzend\n"; for my $backupSet (@{$self->backupSets}){ + $backupSet->{srcDisabledDescendants} = $self->$listDisabledSourceDescendants($backupSet); + if ($self->debug) { + for my $ds (@{$backupSet->{srcDisabledDescendants}}) { + $self->zLog->info('Found disabled sub-dataset: ' . $ds); + } + } + $backupSet->{srcPlanHash} = $self->zTime->backupPlanToHash($backupSet->{src_plan}); #check destination for remote pre-command for (keys %$backupSet){ @@ -353,8 +360,10 @@ my $sendRecvCleanup = sub { my $srcSubDataSets = $backupSet->{recursive} eq 'on' ? $self->zZfs->listSubDataSets($backupSet->{src}) : [ $backupSet->{src} ]; - my @dataSetsExplicitlyDisabled = $self->$listDisabledSourceDescendants($backupSet); - $self->zLog->info('dataSetsExplicitlyDisabled: ' . @dataSetsExplicitlyDisabled); + my @dataSetsExplicitlyDisabled = (); + if (defined($backupSet->{srcDisabledDescendants})) { + @dataSetsExplicitlyDisabled = @{$backupSet->{srcDisabledDescendants}}; + } #loop through all destinations for my $dst (sort grep { /^dst_[^_]+$/ } keys %$backupSet){ @@ -1105,7 +1114,10 @@ my $createSnapshot = sub { # up subsequently (it might still become blocked if znapzend crashes # or the host reboots between such snapshot creation and clean-up). # This only applies if we made a single-command recursive snapshot. - my @dataSetsExplicitlyDisabled = $self->$listDisabledSourceDescendants($backupSet); + my @dataSetsExplicitlyDisabled = (); + if (defined($backupSet->{srcDisabledDescendants})) { + @dataSetsExplicitlyDisabled = @{$backupSet->{srcDisabledDescendants}}; + } # Remove the snapshots previously marked (if any). Note that the # removal here is non-recursive to allow for fine-grained control: