wip

tests/test_zfsautobackup34.py

@@ -1,54 +1,54 @@
 from basetest import *
 
+
 class TestZfsAutobackup34(unittest2.TestCase):
     """various new 3.4 features"""
 
     def setUp(self):
         prepare_zpools()
-        self.longMessage=True
+        self.longMessage = True
 
     def test_no_bookmark_source_support(self):
         """test if everything is fine when source has no bookmark support (has no features at all even)"""
 
         subprocess.check_call("zpool destroy test_source1", shell=True)
         subprocess.check_call("zpool create -d test_source1 /dev/ram0", shell=True)
         shelltest("zpool get all test_source1")
-        subprocess.check_call("zfs create -p test_source1/fs1/sub", shell=True) # recreate with no features at all
+        subprocess.check_call("zfs create -p test_source1/fs1/sub", shell=True)  # recreate with no features at all
         subprocess.check_call("zfs set autobackup:test=true test_source1/fs1", shell=True)
 
         with mocktime("20101111000001"):
             self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run())
 
-        #should fallback on holds on the source snapshot
-        r=shelltest("zfs holds test_source1/fs1@test-20101111000001")
+        # should fallback on holds on the source snapshot
+        r = shelltest("zfs holds test_source1/fs1@test-20101111000001")
         self.assertIn("zfs_autobackup:test", r)
 
     def test_no_bookmark_target_support(self):
         """test if everything is fine when target has no bookmark support (has no features at all even)"""
-        #NOTE: not sure if its ok if only the source supports bookmarks, so currently zfs-autobackup requires both sides to support bookmarks to enable it.
+        # NOTE: not sure if its ok if only the source supports bookmarks, so currently zfs-autobackup requires both sides to support bookmarks to enable it.
 
         subprocess.check_call("zpool destroy test_target1", shell=True)
         subprocess.check_call("zpool create -d test_target1 /dev/ram2", shell=True)
 
         with mocktime("20101111000001"):
             self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run())
 
-        #should fallback on holds on the source snapshot
-        r=shelltest("zfs holds test_source1/fs1@test-20101111000001")
+        # should fallback on holds on the source snapshot
+        r = shelltest("zfs holds test_source1/fs1@test-20101111000001")
         self.assertIn("zfs_autobackup:test", r)
 
-
     def test_select_bookmark_or_snapshot(self):
         """test if zfs autobackup chooses the most recent common matching dataset when there are both bookmarks and snapshots, some with the wrong GUID"""
 
         with mocktime("20101111000001"):
             self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run())
 
-        #destroy stuff and see if it still selects the correct ones
+        # destroy stuff and see if it still selects the correct ones
         shelltest("zfs destroy test_source2/fs2/sub@test-20101111000001")
         shelltest("zfs destroy test_source1/fs1/sub#test-20101111000001")
 
-        #bookmark with incorrect GUID, should fallback to snapshot
+        # bookmark with incorrect GUID, should fallback to snapshot
         shelltest("zfs destroy test_source1/fs1#test-20101111000001")
         shelltest("zfs snapshot test_source1/fs1@wrong")
         shelltest("zfs bookmark test_source1/fs1@wrong \#test-20101111000001")
@@ -57,9 +57,8 @@ def test_select_bookmark_or_snapshot(self):
         with mocktime("20101111000002"):
             self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run())
 
-
-        r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS)
-        self.assertMultiLineEqual(r,"""
+        r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS)
+        self.assertMultiLineEqual(r, """
 test_source1
 test_source1/fs1
 test_source1/fs1@test-20101111000001
@@ -92,20 +91,19 @@ def test_select_bookmark_or_snapshot(self):
 test_target1/test_source2/fs2/sub@test-20101111000002
 """)
 
-        #while we're here, check that there are no holds on source common snapshot (since bookmarks replace holds on source side)
-        r=shelltest("zfs holds test_source2/fs2/sub@test-20101111000002")
+        # while we're here, check that there are no holds on source common snapshot (since bookmarks replace holds on source side)
+        r = shelltest("zfs holds test_source2/fs2/sub@test-20101111000002")
         self.assertNotIn("zfs_autobackup:test", r)
 
-
     def test_disable_bookmarks(self):
         """test if we can disable it on an existing backup with bookmarks, with --no-bookmarks and get the old behaviour (holds on source)"""
 
-        #first with bookmarks enabled
+        # first with bookmarks enabled
         with mocktime("20101111000001"):
             self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run())
 
-        r=shelltest("zfs list -H -o name -r -t all test_source1")
-        self.assertMultiLineEqual(r,"""
+        r = shelltest("zfs list -H -o name -r -t all test_source1")
+        self.assertMultiLineEqual(r, """
 test_source1
 test_source1/fs1
 test_source1/fs1@test-20101111000001
@@ -115,12 +113,13 @@ def test_disable_bookmarks(self):
 test_source1/fs1/sub#test-20101111000001
 """)
 
-        #disable it
+        # disable it
         with mocktime("20101111000002"):
-            self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-bookmarks".split(" ")).run())
+            self.assertFalse(ZfsAutobackup(
+                "test test_target1 --no-progress --verbose --allow-empty --no-bookmarks".split(" ")).run())
 
-        r=shelltest("zfs list -H -o name -r -t all test_source1")
-        self.assertMultiLineEqual(r,"""
+        r = shelltest("zfs list -H -o name -r -t all test_source1")
+        self.assertMultiLineEqual(r, """
 test_source1
 test_source1/fs1
 test_source1/fs1@test-20101111000001
@@ -130,12 +129,12 @@ def test_disable_bookmarks(self):
 test_source1/fs1/sub@test-20101111000002
 """)
 
-        #re-enable
+        # re-enable
         with mocktime("20101111000003"):
             self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run())
 
-        r=shelltest("zfs list -H -o name -r -t all test_source1")
-        self.assertMultiLineEqual(r,"""
+        r = shelltest("zfs list -H -o name -r -t all test_source1")
+        self.assertMultiLineEqual(r, """
 test_source1
 test_source1/fs1
 test_source1/fs1@test-20101111000001
@@ -150,21 +149,23 @@ def test_disable_bookmarks(self):
 """)
 
     def test_tags(self):
-
-
         with mocktime("20101111000001"):
-            self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test1".split(" ")).run())
+            self.assertFalse(
+                ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test1".split(" ")).run())
 
         with mocktime("20101111000002"):
-            self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test2".split(" ")).run())
+            self.assertFalse(
+                ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test2".split(" ")).run())
 
         with mocktime("20101111000003"):
             # make sure the thinner sees and cleans up the old snaphots that have a tag
-            self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --keep-source=2 --keep-target=2".split(" ")).run())
+            self.assertFalse(ZfsAutobackup(
+                "test test_target1 --no-progress --verbose --allow-empty --keep-source=2 --keep-target=2".split(
+                    " ")).run())
 
-        r=shelltest("zfs list -H -r -t snapshot -o name "+TEST_POOLS)
+        r = shelltest("zfs list -H -r -t snapshot -o name " + TEST_POOLS)
 
-        self.assertMultiLineEqual(r,"""
+        self.assertMultiLineEqual(r, """
 test_source1/fs1@test-20101111000002_test2
 test_source1/fs1@test-20101111000003
 test_source1/fs1/sub@test-20101111000002_test2
@@ -179,3 +180,35 @@ def test_tags(self):
 test_target1/test_source2/fs2/sub@test-20101111000003
 """)
 
+    def test_double_send_bookmark(self):
+        """test sending the same snaphots to 2 targets, and check if they each use their own bookmark and delete them correctly."""
+
+        shelltest("zfs create test_target1/a")
+        shelltest("zfs create test_target1/b")
+
+        # full
+        with mocktime("20101111000001"):
+            self.assertFalse(
+                ZfsAutobackup(
+                    "test test_target1/a --no-progress --verbose --allow-empty --tag tag1".split(" ")).run())
+
+        # increment, should be from bookmark
+        with mocktime("20101111000002"):
+            self.assertFalse(
+                ZfsAutobackup(
+                    "test test_target1/a --no-progress --verbose --allow-empty --tag tag1".split(" ")).run())
+
+        # to target b, now each has one full + two incrementals, which should be from their own bookmarks.
+        with mocktime("20101111000003"):
+            self.assertFalse(
+                ZfsAutobackup(
+                    "test test_target1/b --no-progress --verbose --no-snapshot --allow-empty".split(
+                        " ")).run())
+
+        # result:
+        # for target a the bookmarks should be at 20101111000002, for target b the bookmarks should be at 20101111000003
+        r = shelltest("zfs list -H -r -t all -o name " + TEST_POOLS)
+
+        self.assertMultiLineEqual(r, """
+...
+""")

zfs_autobackup/ZfsAutobackup.py

@@ -357,8 +357,9 @@ def check_target_names(self, source_node, source_datasets, target_node):
             target_datasets[target_name] = source_dataset
 
     # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters:
-    def sync_datasets(self, source_node, source_datasets, target_node):
+    def sync_datasets(self, source_node, source_datasets, target_node, bookmark_tag):
         """Sync datasets, or thin-only on both sides
+        :type bookmark_tag: str
         :type target_node: ZfsNode
         :type source_datasets: list of ZfsDataset
         :type source_node: ZfsNode
@@ -419,7 +420,8 @@ def sync_datasets(self, source_node, source_datasets, target_node):
                                               send_pipes=send_pipes, recv_pipes=recv_pipes,
                                               decrypt=self.args.decrypt, encrypt=self.args.encrypt,
                                               zfs_compressed=self.args.zfs_compressed, force=self.args.force,
-                                              guid_check=not self.args.no_guid_check, use_bookmarks=use_bookmarks)
+                                              guid_check=not self.args.no_guid_check, use_bookmarks=use_bookmarks,
+                                              bookmark_tag=bookmark_tag)
             except Exception as e:
 
                 fail_count = fail_count + 1
@@ -556,7 +558,7 @@ def run(self):
                 fail_count = self.sync_datasets(
                     source_node=source_node,
                     source_datasets=source_datasets,
-                    target_node=target_node)
+                    target_node=target_node, bookmark_tag=target_dataset.properties['guid'])
 
             # no target specified, run in snapshot-only mode
             else:

zfs_autobackup/ZfsDataset.py

@@ -646,16 +646,17 @@ def is_changed_ours(self, min_changed_bytes=1):
 
         return True
 
-    def bookmark(self):
-        """Bookmark this snapshot, and return the bookmark"""
+    def bookmark(self, tag):
+        """Bookmark this snapshot, and return the bookmark."""
+        # NOTE: we use the tag to add the target_path GUID, so that we can have multiple bookmarks for the same snapshot, but for different target. This is to make sure you can send a backup to two locations, without them interfering with eachothers bookmarks.
 
         if not self.is_snapshot:
             raise (Exception("Can only bookmark a snapshot!"))
 
         self.debug("Bookmarking")
 
         cmd = [
-            "zfs", "bookmark", self.name, "#" + self.suffix
+            "zfs", "bookmark", self.name, "#" + self.tagless_suffix + self.zfs_node.tag_seperator + tag
         ]
 
         self.zfs_node.run(cmd=cmd)
@@ -772,6 +773,7 @@ def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, s
 
         cmd.extend(send_pipes)
 
+        self.error(cmd)
         output_pipe = self.zfs_node.run(cmd, pipe=True, readonly=True)
 
         return output_pipe
@@ -1237,7 +1239,7 @@ def handle_incompatible_snapshots(self, incompatible_target_snapshots, destroy_i
     def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties,
                        ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots,
                        no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check,
-                       use_bookmarks):
+                       use_bookmarks, bookmark_tag):
         """sync this dataset's snapshots to target_dataset, while also thinning
         out old snapshots along the way.
 
@@ -1256,6 +1258,8 @@ def sync_snapshots(self, target_dataset, features, show_progress, filter_propert
             :type also_other_snapshots: bool
             :type no_send: bool
             :type guid_check: bool
+            :type use_bookmarks: bool
+            :type bookmark_tag: str
         """
 
         # self.verbose("-> {}".format(target_dataset))
@@ -1344,7 +1348,7 @@ def sync_snapshots(self, target_dataset, features, show_progress, filter_propert
 
             # bookmark common snapshot on source, or use holds if bookmarks are not enabled.
             if use_bookmarks:
-                source_bookmark = source_snapshot.bookmark()
+                source_bookmark = source_snapshot.bookmark(bookmark_tag)
                 # note: destroy source_snapshot when obsolete at this point?
             else:
                 source_bookmark = None