From 1a8f92fae16359e8dc8f96d6c540a5db3fe3c0d7 Mon Sep 17 00:00:00 2001 From: xarx00 Date: Thu, 28 May 2020 12:30:02 +0200 Subject: [PATCH 1/9] Make bsync working in Windows --- bsync | 44 ++++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/bsync b/bsync index 1a66053..42b6f95 100755 --- a/bsync +++ b/bsync @@ -138,8 +138,15 @@ def rsync_init(sshSrc,dirnameSrc, sshDst,dirnameDst): # #rsync ssh/dir2 --> local/dir1 #rsync local/dir2 --> ssh/dir1 - rsyncsrc = getdirstr(sshSrc, dirnameSrc)+"/" - rsyncdst = getdirstr(sshDst, dirnameDst)+"/" + rsyncsrc = getdirstr(sshSrc, dirnameSrc) + rsyncdst = getdirstr(sshDst, dirnameDst) + if os.name == "nt": + # workaround for bug in cwRsync; expecting it to run under CygWin + rsyncsrc = re.sub(r'^(.):', r'/cygdrive/\1', rsyncsrc) + rsyncdst = re.sub(r'^(.):', r'/cygdrive/\1', rsyncdst) + else: + rsyncsrc += "/" + rsyncdst += "/" args = [ "-a", "--files-from=-", "--from0", "--no-implied-dirs", "--out-format=rsync: %n%L" ] if ssh != None: @@ -207,8 +214,15 @@ def fs_check_perms(ssh, dirname): # check with rsync that directories are identical (-c flag) def rsync_check(sshSrc,dirnameSrc, sshDst,dirnameDst): - rsyncsrc = getdirstr(sshSrc, dirnameSrc)+"/" - rsyncdst = getdirstr(sshDst, dirnameDst)+"/" + rsyncsrc = getdirstr(sshSrc, dirnameSrc) + rsyncdst = getdirstr(sshDst, dirnameDst) + if os.name == "nt": + # workaround for bug in cwRsync; expecting it to run under CygWin + rsyncsrc = re.sub(r'^(.):', r'/cygdrive/\1', rsyncsrc) + rsyncdst = re.sub(r'^(.):', r'/cygdrive/\1', rsyncdst) + else: + rsyncsrc += "/" + rsyncdst += "/" args = [ "-anO", "--delete", "--out-format=%n%L", "--exclude=/.bsync-snap-*" ] if ssh != None: @@ -217,7 +231,8 @@ def rsync_check(sshSrc,dirnameSrc, sshDst,dirnameDst): diff = subprocess.check_output(["rsync"]+args+[rsyncsrc, rsyncdst], universal_newlines=True).split("\n") diff.remove("") - diff.remove("./") + if os.name != "nt": + diff.remove("./") if len(diff) != 0: sys.exit("Error: rsync_check differences:\n"+str(diff)) @@ -227,16 +242,16 @@ def rsync_check(sshSrc,dirnameSrc, sshDst,dirnameDst): def make_snapshot(ssh,dirname, oldsnapname, newsnapname): global findformat, findcmdlocal, findcmdremote - cmd = " %s -fprintf %s '%s'" % (quote(dirname), quote(dirname+"/"+newsnapname), findformat) + cmd = [dirname, "-fprintf", dirname+newsnapname, findformat] if oldsnapname!=None: - cmd+= " && rm -f "+quote(dirname+"/"+oldsnapname) + cmd+= ["&&", "rm", "-f", dirname+oldsnapname] # remove inconsistent newsnap if error in find - cmd+= " || ( rm -f "+quote(dirname+"/"+newsnapname)+" && false )" + cmd+= ["||", "(", "rm", "-f", dirname+newsnapname, "&&", "false", ")"] if ssh==None: - ret = subprocess.call(findcmdlocal+cmd, shell=True) + ret = subprocess.call([findcmdlocal]+cmd, shell=True) else: - ret = subprocess.call(ssh.getcmdlist()+[findcmdremote+cmd]) + ret = subprocess.call(ssh.getcmdlist()+[findcmdremote]+cmd) if ret != 0: sys.exit("Error making a snapshot.") @@ -741,6 +756,7 @@ def usage(): usage+= " -v Verbose\n" usage+= " -i Ignore permissions\n" usage+= " -b Batch mode (exit on conflict)\n" + usage+= " -c Check that directories are identical\n" usage+= " -p PORT Port for SSH\n" usage+= " -o SSHARGS Custom options for SSH\n" printerr(usage) @@ -783,10 +799,10 @@ dir2name = args[1] # get ssh connection ssh = ssh1 = ssh2 = None -if ':' in dir1name: +if ':' in dir1name[2:]: sshuserhost, dir1name = dir1name.split(':', 1) ssh = ssh1 = SshCon(sshuserhost, sshport, sshargs) -if ':' in dir2name: +if ':' in dir2name[2:]: sshuserhost, dir2name = dir2name.split(':', 1) ssh = ssh2 = SshCon(sshuserhost, sshport, sshargs) if ssh1!=None and ssh2!=None: @@ -800,8 +816,8 @@ rsync_check_install(ssh) findcmdlocal, findcmdremote = find_check_command(ssh) # add trailing slashes (to avoid problems with symlinked dirs) -dir1name = os.path.join(dir1name, '') -dir2name = os.path.join(dir2name, '') +dir1name = os.path.join(dir1name, '').replace('\\', '/') +dir2name = os.path.join(dir2name, '').replace('\\', '/') # try to get console width, for displaying actions, if running interactive try: From e3e9dc3d298593d4b3d5720c20a0454cd21178f5 Mon Sep 17 00:00:00 2001 From: xarx00 Date: Thu, 28 May 2020 12:32:24 +0200 Subject: [PATCH 2/9] Allow skipping conflicts and leave them unresolved --- bsync | 128 ++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 98 insertions(+), 30 deletions(-) diff --git a/bsync b/bsync index 42b6f95..d0b2661 100755 --- a/bsync +++ b/bsync @@ -43,7 +43,7 @@ def tostr(o): # a file record from snapshots (original file) class OrigFile(): - def __init__(self, inode1,inode2, path,type,date,size,perms): + def __init__(self, inode1,inode2, path,type,date,size,perms, r1,r2): self.path = path self.i1 = inode1 self.i2 = inode2 @@ -51,6 +51,8 @@ class OrigFile(): self.date = date self.size = size self.perms = perms + self.r1 = r1 + self.r2 = r2 # a file record from an actual directory class DirFile(): @@ -237,9 +239,52 @@ def rsync_check(sshSrc,dirnameSrc, sshDst,dirnameDst): if len(diff) != 0: sys.exit("Error: rsync_check differences:\n"+str(diff)) +# correct the snapshot with records from the original loaded snaphot, +# effectively reseting the unresolved records to their original value +def fix_unresolved(ssh,dirname, snapname, unresolved): + sbproc = None + f = None + try: + # if we need a ssh shell + if ssh != None: + cmd = ["cat - > %s" % quote(dirname+"/"+snapname)+".tmp"] + sbproc = subprocess.Popen(ssh.getcmdlist()+cmd, stdin=subprocess.PIPE) + else: + f = open(dirname+"/"+snapname+".tmp", "w") + sbproc = subprocess.Popen(["cat", "-"], stdin=subprocess.PIPE, stdout=f) + + if sbproc == None: + sys.exit("Error updating a snapshot.") + + # filter out unresolved records from the current snapshot + fd = get_snap_fd(ssh, dirname, snapname) + gen = fileLineIter(fd) + + record = read_file_record(gen) + if record==None: sys.exit("Error reading files from filelist") #should be at least one record (dir root) + while record != None: + inode,path,type,date,size,perms,rec = record + if path not in unresolved: + sbproc.stdin.write( rec + b"\0" ) + record = read_file_record(gen) + + # append unresolved records + for rec in unresolved.values(): + if rec != None: + sbproc.stdin.write( rec + b"\0" ) + sbproc.stdin.flush() + finally: + # cleanup + sbproc.stdin.close() + sbproc.wait() + if f != None: + f.close() + if sbproc.returncode != 0: + sys.exit("Error in process updating a snapshot.") + # take a snapshot of files states from dir, using find. store it in .bsync-snap-XXXX # snap format: inode, path, type, date... -def make_snapshot(ssh,dirname, oldsnapname, newsnapname): +def make_snapshot(ssh,dirname, oldsnapname, newsnapname, unresolved): global findformat, findcmdlocal, findcmdremote cmd = [dirname, "-fprintf", dirname+newsnapname, findformat] @@ -255,12 +300,15 @@ def make_snapshot(ssh,dirname, oldsnapname, newsnapname): if ret != 0: sys.exit("Error making a snapshot.") -def make_snapshots(ssh1,dir1name, ssh2,dir2name, oldsnapname): + if unresolved: + fix_unresolved(ssh,dirname, newsnapname, unresolved) + +def make_snapshots(ssh1,dir1name, ssh2,dir2name, oldsnapname, unresolved): newsnapname = ".bsync-snap-"+datetime.datetime.now().strftime("%Y%m%d%H%M%S.%f") print("Updating filelists...") printv("Updating snap files: "+newsnapname+"...") - make_snapshot(ssh1,dir1name, oldsnapname,newsnapname) - make_snapshot(ssh2,dir2name, oldsnapname,newsnapname) + make_snapshot(ssh1,dir1name, oldsnapname,newsnapname, {fo.path:fo.r1 for fo in unresolved}) + make_snapshot(ssh2,dir2name, oldsnapname,newsnapname, {fo.path:fo.r2 for fo in unresolved}) # run find in a directory to dump its content def get_find_proc(ssh, dirname): @@ -364,6 +412,7 @@ def read_file_record(gen): i=p=t=d=s=perms=None try: i,p,t,d,s,perms = next(gen),next(gen),next(gen),next(gen),next(gen),next(gen) + rec = b"\0".join((i,p,t,d,s,perms)) # convert all to str except path i = i.decode() t = t.decode() @@ -380,7 +429,7 @@ def read_file_record(gen): if t=="d": d=s="0" # ignore dates/size for dirs (set to zero) if ignoreperms: perms = "" - return i,p,t,d,s,perms + return i,p,t,d,s,perms, rec # load original file records from snapshots, and ignore entries def load_orig(ssh1,dir1name, ssh2,dir2name): @@ -418,10 +467,10 @@ def load_orig(ssh1,dir1name, ssh2,dir2name): record = read_file_record(gen1) if record==None: sys.exit("Error reading files from dir1 filelist") #should be at least one record (dir root) while record != None: - inode,path,type,date,size,perms = record + inode,path,type,date,size,perms,rec = record if not ignorepath(path, ignores): - orig[path] = OrigFile(inode,None, path,type,date,size,perms) + orig[path] = OrigFile(inode,None, path,type,date,size,perms, rec,None) record = read_file_record(gen1) @@ -429,7 +478,7 @@ def load_orig(ssh1,dir1name, ssh2,dir2name): record = read_file_record(gen2) if record==None: sys.exit("Error reading files from dir2 filelist") while record != None: - inode,path,type,date,size,perms = record + inode,path,type,date,size,perms,rec = record if not ignorepath(path, ignores): #path not in orig: can happen if using ignore, then removing ignore, path will be considered as new @@ -439,6 +488,9 @@ def load_orig(ssh1,dir1name, ssh2,dir2name): sys.exit("Error: difference in snaps for path: "+tostr(path)) origfile.i2 = inode #set the second inode + origfile.r2 = rec #set the second record + else: + orig[path] = OrigFile(None,inode, path,type,date,size,perms, None,rec) record = read_file_record(gen2) @@ -460,7 +512,7 @@ def load_dir(ssh, dirname, ignores): record = read_file_record(gen) while record != None: - inode,path,type,date,size,perms = record + inode,path,type,date,size,perms,rec = record if not ignorepath(path, ignores): dir[path] = DirFile(inode, path, type, date, size, perms) @@ -523,7 +575,8 @@ def print_line(): # ask the user about conflicting changes # conflict can be on type, date, size, perms def ask_conflict(f1, f2, path, tokeep): - if tokeep=="1a" or tokeep=="2a": + global answered_N + if tokeep in ("1a", "2a", "na"): return tokeep resp = None @@ -532,21 +585,26 @@ def ask_conflict(f1, f2, path, tokeep): show_conflict(f1, f2, path) if batch: - sys.exit("Error: Conflict found in batch mode. Exiting.") + answered_N = True + return "n" if resp!=None: print(" 1 Keep left version") print(" 2 Keep right version") print(" 1a Keep left version for all") print(" 2a Keep right version for all") + print(" N Nothing, leave unresolved") + print(" Na Nothing, leave unresolved") print(" Please note: you will be able to confirm the actions later.\n") - resp = myinput("Which one do I keep? [1/2/1a/2a/Quit/Help] ") + resp = myinput("Which one do I keep? [1/2/1a/2a/N/Na/Quit/Help] ").lower() - if resp == "1" or resp == "2" or resp == "1a" or resp == "2a": + if resp in ("1", "1a", "2", "2a", "n", "na"): + if resp in ("n", "na"): + answered_N = True return resp - elif resp == "q" or resp == "Q" or resp == "Quit": - sys.exit(0) + elif resp == "q" or resp == "quit": + sys.exit(1) #### file actions def remove(shproc, path): @@ -755,7 +813,7 @@ def usage(): usage+= " DIR can be user@sshserver:DIR\n" usage+= " -v Verbose\n" usage+= " -i Ignore permissions\n" - usage+= " -b Batch mode (exit on conflict)\n" + usage+= " -b Batch mode (skip conflicts)\n" usage+= " -c Check that directories are identical\n" usage+= " -p PORT Port for SSH\n" usage+= " -o SSHARGS Custom options for SSH\n" @@ -894,17 +952,20 @@ copy21 = [] sync12 = [] sync21 = [] tokeep = None +unresolved = [] +answered_N = False # process all original paths (from snapshot) for path, fo in origlist.items(): - # f1==None f2==None deleted both sides - # f1==None f2=!None f2.d==fo.d f1 chg only - # f1==None f2=!None f2.d!=fo.d conflict + # f1==None f2==None fo!=None deleted both sides + # f1==None f2!=None f2.d==fo.d f1 chg only + # f1==None f2!=None f2.d!=fo.d conflict # f1!=None f2==None f1.d==fo.d f2 chg only # f1!=None f2==None f1.d!=fo.d conflict - # f1!=None f2!=None f1.d==fo.d f2.d==fo.d no change - # f1!=None f2!=None f1.d==fo.d f2.d!=fo.d f2 chg only - # f1!=None f2!=None f1.d!=fo.d f2.d==fo.d f1 chg only - # f1!=None f2!=None f1.d!=fo.d f2.d!=fo.d conflict + # f1!=None f2!=None f1.d==fo.d==f2.d no change + # f1!=None f2!=None f1.d==fo.d!=f2.d f2 chg only + # f1!=None f2!=None f1.d!=fo.d==f2.d f1 chg only + # f1!=None f2!=None f1.d==f2.d!=fo.d same change + # f1!=None f2!=None f1.d!=f2.d!=fo.d conflict f1 = dir1[path] if path in dir1 else None f2 = dir2[path] if path in dir2 else None @@ -957,7 +1018,7 @@ for path, fo in origlist.items(): copy12.append(f1) else: sync12.append(path) - else: # tokeep == 2 + elif tokeep[0] == "2": #2 or 2a if f2 == None: if f1.type == "d": # f1 isdir rmdirs1.append(path) @@ -971,6 +1032,8 @@ for path, fo in origlist.items(): copy21.append(f2) else: sync21.append(path) + else: #leave conflict unresolved + unresolved.append(fo) #ifend dir1.pop(path, None) @@ -994,10 +1057,12 @@ for path, f1 in dir1.items(): else: # f2!=None and f2.date != f1.date --> conflict tokeep = ask_conflict(f1, f2, path, tokeep); - if tokeep[0] == "1": + if tokeep[0] == "1": #1 or 1a sync12.append(path) - else: # tokeep == 2 + elif tokeep[0] == "2": #2 or 2a sync21.append(path) + else: #leave conflict unresolved + unresolved.append(OrigFile(None,None,path,None,None,None,None,None,None)) dir2.pop(path, None) @@ -1024,9 +1089,12 @@ rmdirs2.sort(reverse=True) # TODO someth cleaner than sort? if len(mkdir1)==0 and len(moves1)==0 and len(rm1)==0 and len(rmdirs1)==0 and len(copy21)==0 and len(sync21)==0 and \ len(mkdir2)==0 and len(moves2)==0 and len(rm2)==0 and len(rmdirs2)==0 and len(copy12)==0 and len(sync12)==0: if check: rsync_check(ssh1,dir1name, ssh2,dir2name) - print("Identical directories. Nothing to do.") + if answered_N: + print("Nothing to do. Some conflicts stay unresolved.") + else: + print("Identical directories. Nothing to do.") if snapname == None: - make_snapshots(ssh1,dir1name, ssh2,dir2name, snapname) + make_snapshots(ssh1,dir1name, ssh2,dir2name, snapname, unresolved) sys.exit() if len(conflicts) > 0: print_line() @@ -1063,6 +1131,6 @@ apply_rsync_actions(ssh2,dir2name,ssh1,dir1name, copy21 + sync21) if check: rsync_check(ssh1,dir1name, ssh2,dir2name) -make_snapshots(ssh1,dir1name, ssh2,dir2name, snapname) +make_snapshots(ssh1,dir1name, ssh2,dir2name, snapname, unresolved) print("Done!") From 9daf049ab7fdfe269ef68d8ec75493dce6c79f06 Mon Sep 17 00:00:00 2001 From: xarx00 Date: Thu, 28 May 2020 12:34:14 +0200 Subject: [PATCH 3/9] New modes: mirror and backup, while maintaining the bsync snaphots --- bsync | 37 +++++-- tests/bsync_tests.py | 248 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+), 8 deletions(-) create mode 100644 tests/bsync_tests.py diff --git a/bsync b/bsync index d0b2661..b500bf1 100755 --- a/bsync +++ b/bsync @@ -815,6 +815,10 @@ def usage(): usage+= " -i Ignore permissions\n" usage+= " -b Batch mode (skip conflicts)\n" usage+= " -c Check that directories are identical\n" + usage+= " -m MODE sync|backup|mirror (defult sync)\n" + usage+= " backup - copy new and modified from DIR1 to DIR2\n" + usage+= " mirror - backup + missing in DIR1 remove from DIR2\n" + usage+= " sync - bidirectional mirror\n" usage+= " -p PORT Port for SSH\n" usage+= " -o SSHARGS Custom options for SSH\n" printerr(usage) @@ -823,13 +827,14 @@ def usage(): #### process commandline args try: - opts, args = getopt.gnu_getopt(sys.argv[1:], "vcibp:o:") + opts, args = getopt.gnu_getopt(sys.argv[1:], "vcibp:o:m:") except getopt.GetoptError as err: printerr(err) usage() sys.exit(2) verbose = check = ignoreperms = batch = False +syncmode='sync' sshport = None sshargs = "" for o, a in opts: @@ -839,6 +844,8 @@ for o, a in opts: ignoreperms = True elif o == "-c": check = True + elif o == "-m": + syncmode = a elif o == "-p": sshport = a elif o == "-o": @@ -847,6 +854,7 @@ for o, a in opts: batch = True else: assert False, "unhandled option" +assert syncmode in ('sync', 'backup', 'mirror'), "invalid mode option" if len(args) != 2: usage() @@ -980,7 +988,9 @@ for path, fo in origlist.items(): # no f2 change --> f1 change only if f1 == None: # f1 deleted --> delete f2 - if f2.type == "d": # f2 isdir + if syncmode == "backup": + unresolved.append(fo) + elif f2.type == "d": # f2 isdir rmdirs2.append(path) else: rm2[fo.i1] = f2 @@ -989,13 +999,19 @@ for path, fo in origlist.items(): sync12.append(path) elif f1 != None and samefiles(f1,fo): # no f1 change --> f2 change only - if f2 == None: + if syncmode != "sync": + unresolved.append(fo) + elif f2 == None: if f1.type == "d": #f1 isdir rmdirs1.append(path) else: rm1[fo.i2] = f1 else: sync21.append(path) + elif syncmode == "backup" and f1 == None: + # f1 change and f2 change --> confict + # f1 == None and f2 != None --> keep unresolved for "backup" syncmode + unresolved.append(fo) else: # f1 change and f2 change --> confict # f1 != None and f2 != None --> f1.date != f2.date (!= fo.date) @@ -1070,10 +1086,13 @@ for path, f1 in dir1.items(): printv("Analysing remaining new paths in dir2...") # process remaining new paths in dir2 for path, f2 in dir2.items(): - if f2.type == "d": - mkdir1.append(f2) - else: - copy21.append(f2) + if syncmode == "sync": + if f2.type == "d": + mkdir1.append(f2) + else: + copy21.append(f2) + else: #leave conflict unresolved + unresolved.append(OrigFile(None,None,path,None,None,None,None,None,None)) # moves detection copy12, rm2, moves2 = check_moves(copy12, rm2) @@ -1091,8 +1110,10 @@ if len(mkdir1)==0 and len(moves1)==0 and len(rm1)==0 and len(rmdirs1)==0 and len if check: rsync_check(ssh1,dir1name, ssh2,dir2name) if answered_N: print("Nothing to do. Some conflicts stay unresolved.") - else: + elif syncmode == "sync": print("Identical directories. Nothing to do.") + else: + print("Target dir is up to date. Nothing to do.") if snapname == None: make_snapshots(ssh1,dir1name, ssh2,dir2name, snapname, unresolved) sys.exit() diff --git a/tests/bsync_tests.py b/tests/bsync_tests.py new file mode 100644 index 0000000..91c79fa --- /dev/null +++ b/tests/bsync_tests.py @@ -0,0 +1,248 @@ +import unittest, tempfile, os, subprocess, shutil + +dir1 = "dir1" +dir2 = "dir2" + +class TestBase(unittest.TestCase): + + def setUp(self): + self._tempdir = tempfile.mkdtemp() + self.dir1 = os.path.join(self._tempdir, dir1) + self.dir2 = os.path.join(self._tempdir, dir2) + os.mkdir(self.dir1) + os.mkdir(self.dir2) + self.counter = 0 + + def tearDown(self): + shutil.rmtree(self._tempdir) + pass + + def bsync(self, args): + with subprocess.Popen(["bsync"]+args+[self.dir1, self.dir2], shell=True, stdout=subprocess.PIPE) as proc: + fd = proc.stdout + output = fd.read() + fd.close() + proc.wait() + self.assertEqual(proc.returncode, 0, "bsync failed with code %d" % proc.returncode) + return output + + def _val(self, num): + return "o" * num + + def updfile(self, dir, name): + if type(name) is list: + for n in name: + with open(os.path.join(self._tempdir, dir, n), "w") as f: + f.write(self._val(self.counter)) + self.counter += 1 + else: + with open(os.path.join(self._tempdir, dir, name), "w") as f: + f.write(self._val(self.counter)) + self.counter += 1 + + def delfile(self, dir, name): + os.remove(os.path.join(self._tempdir, dir, name)) + + def assertExists(self, dir, name, msg=None): + if type(name) is list: + for n in name: + self.assertTrue(os.path.exists(os.path.join(self._tempdir, dir, n)), msg) + else: + self.assertTrue(os.path.exists(os.path.join(self._tempdir, dir, name)), msg) + + def assertNotExists(self, dir, name, msg=None): + if type(name) is list: + for n in name: + self.assertFalse(os.path.exists(os.path.join(self._tempdir, dir, n)), msg) + else: + self.assertFalse(os.path.exists(os.path.join(self._tempdir, dir, name)), msg) + + def assertFileContains(self, dir, name, value, msg=None): + self.assertExists(dir, name, msg) + with open(os.path.join(self._tempdir, dir, name), "r") as f: + rvalue = f.read() + self.assertEqual(rvalue, self._val(value)) + +class TestSync(TestBase): + + def test_1_to_2(self): + self.updfile(dir1, ["a", "b"]) + self.bsync(["-b"]) + self.assertExists(dir2, ["a", "b"]) + self.assertFileContains(dir2, "a", 0) + self.assertFileContains(dir2, "b", 1) + + def test_2_to_1(self): + self.updfile(dir2, ["a", "b"]) + self.bsync(["-b"]) + self.assertExists(dir1, ["a", "b"]) + self.assertFileContains(dir1, "a", 0) + self.assertFileContains(dir1, "b", 1) + + def test_upd(self): + self.test_1_to_2() + self.updfile(dir1, "a") + self.updfile(dir2, "b") + self.bsync(["-b"]) + self.assertFileContains(dir1, "a", 2) + self.assertFileContains(dir2, "a", 2) + self.assertFileContains(dir1, "b", 3) + self.assertFileContains(dir2, "b", 3) + + def test_del(self): + self.test_1_to_2() + self.delfile(dir1, "a") + self.delfile(dir2, "b") + self.bsync(["-b"]) + self.assertNotExists(dir1, ["a", "b"]) + self.assertNotExists(dir2, ["a", "b"]) + + def test_conflict(self): + self.test_1_to_2() + self.updfile(dir1, "a") + self.updfile(dir2, "a") + self.updfile(dir1, "b") + self.delfile(dir2, "b") + self.updfile(dir1, "c") + self.bsync(["-b"]) + self.bsync(["-b"]) + self.assertFileContains(dir1, "a", 2) + self.assertFileContains(dir2, "a", 3) + self.assertFileContains(dir1, "b", 4) + self.assertNotExists(dir2, "b") + + +class TestMirror(TestBase): + + def test_1_to_2(self): + self.updfile(dir1, ["a", "b"]) + self.bsync(["-bm", "mirror"]) + self.assertExists(dir2, ["a", "b"]) + self.assertFileContains(dir2, "a", 0) + self.assertFileContains(dir2, "b", 1) + + def test_2_to_1(self): + self.updfile(dir2, ["a", "b"]) + self.bsync(["-bm", "mirror"]) + self.assertNotExists(dir1, ["a", "b"]) + self.assertExists(dir2, ["a", "b"]) + + def test_upd(self): + self.test_1_to_2() + self.updfile(dir1, "a") + self.updfile(dir2, "b") + self.bsync(["-bm", "mirror"]) + self.assertFileContains(dir1, "a", 2) + self.assertFileContains(dir2, "a", 2) + self.assertFileContains(dir1, "b", 1) + self.assertFileContains(dir2, "b", 3) + + def test_del(self): + self.test_1_to_2() + self.delfile(dir1, "a") + self.delfile(dir2, "b") + self.bsync(["-bm", "mirror"]) + self.assertNotExists(dir1, "a") + self.assertFileContains(dir1, "b", 1) + self.assertNotExists(dir2, ["a", "b"]) + + def test_conflict(self): + self.test_1_to_2() + self.updfile(dir1, "a") + self.updfile(dir2, "a") + self.updfile(dir1, "b") + self.delfile(dir2, "b") + self.updfile(dir1, "c") + self.bsync(["-bm", "mirror"]) + self.bsync(["-bm", "mirror"]) + self.assertFileContains(dir1, "a", 2) + self.assertFileContains(dir2, "a", 3) + self.assertFileContains(dir1, "b", 4) + self.assertNotExists(dir2, "b") + + +class TestBackup(TestBase): + + def test_1_to_2(self): + self.updfile(dir1, ["a", "b"]) + self.bsync(["-bm", "backup"]) + self.assertExists(dir2, ["a", "b"]) + self.assertFileContains(dir2, "a", 0) + self.assertFileContains(dir2, "b", 1) + + def test_2_to_1(self): + self.updfile(dir2, ["a", "b"]) + self.bsync(["-bm", "backup"]) + self.assertNotExists(dir1, ["a", "b"]) + self.assertExists(dir2, ["a", "b"]) + + def test_upd(self): + self.test_1_to_2() + self.updfile(dir1, "a") + self.updfile(dir2, "b") + self.bsync(["-bm", "backup"]) + self.assertFileContains(dir1, "a", 2) + self.assertFileContains(dir2, "a", 2) + self.assertFileContains(dir1, "b", 1) + self.assertFileContains(dir2, "b", 3) + + def test_del(self): + self.test_1_to_2() + self.delfile(dir1, "a") + self.delfile(dir2, "b") + self.bsync(["-bm", "backup"]) + self.assertNotExists(dir1, "a") + self.assertFileContains(dir2, "a", 0) + self.assertFileContains(dir1, "b", 1) + self.assertNotExists(dir2, "b") + + def test_conflict(self): + self.test_1_to_2() + self.updfile(dir1, "a") + self.updfile(dir2, "a") + self.updfile(dir1, "b") + self.delfile(dir2, "b") + self.updfile(dir1, "c") + self.bsync(["-bm", "backup"]) + self.bsync(["-bm", "backup"]) + self.assertFileContains(dir1, "a", 2) + self.assertFileContains(dir2, "a", 3) + self.assertFileContains(dir1, "b", 4) + self.assertNotExists(dir2, "b") + + +class TestMixed(TestBase): + + def _1_to_2(self): + self.updfile(dir1, ["a", "b"]) + self.bsync(["-b"]) + self.assertExists(dir2, ["a", "b"]) + self.assertFileContains(dir2, "a", 0) + self.assertFileContains(dir2, "b", 1) + + def test_sync_after_backup(self): + self._1_to_2() + self.delfile(dir1, "a") + self.updfile(dir2, "b") + self.updfile(dir1, "c") + self.bsync(["-bm", "backup"]) + self.bsync(["-b"]) + self.assertNotExists(dir1, "a") + self.assertNotExists(dir2, "a") + self.assertFileContains(dir1, "b", 2) + self.assertFileContains(dir2, "b", 2) + + def test_mirror_after_backup(self): + self._1_to_2() + self.delfile(dir1, "a") + self.updfile(dir2, "b") + self.updfile(dir1, "c") + self.bsync(["-bm", "backup"]) + self.bsync(["-bm", "mirror"]) + self.assertNotExists(dir1, "a") + self.assertNotExists(dir2, "a") + self.assertFileContains(dir1, "b", 1) + self.assertFileContains(dir2, "b", 2) + +if __name__ == '__main__': + unittest.main(verbosity=2) From 7ce8f75ddca5a18df4bcce0aad8d076c380247c8 Mon Sep 17 00:00:00 2001 From: xarx00 Date: Fri, 31 Jul 2020 11:05:55 +0200 Subject: [PATCH 4/9] Update README.md Updated usage in README.md --- README.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5e38762..e4df0a8 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,17 @@ bsync help and options: ``` Usage: bsync [options] DIR1 DIR2 - DIR can be user@sshserver:DIR - -v Verbose - -i Ignore permissions - -p PORT Port for SSH - -o SSHARGS Custom options for SSH + DIR can be user@sshserver:DIR + -v Verbose + -i Ignore permissions + -b Batch mode (skip conflicts) + -c Check that directories are identical + -m MODE sync|backup|mirror (defult sync) + backup - copy new and modified from DIR1 to DIR2 + mirror - backup + missing in DIR1 remove from DIR2 + sync - bidirectional mirror + -p PORT Port for SSH + -o SSHARGS Custom options for SSH ``` Features From 98d53f4385a3944010fde30229baf7837da0ae57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Plech=C5=A1m=C3=ADd=20Martin?= Date: Sat, 1 Aug 2020 01:38:56 +0200 Subject: [PATCH 5/9] simple wild-cards for ignore files (pattern begins with star) --- bsync | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bsync b/bsync index b500bf1..08ad5a0 100755 --- a/bsync +++ b/bsync @@ -376,7 +376,10 @@ def ignorepath(path, ignoreset): return True else: for ignore in ignoreset: - if (path+b"/").startswith(ignore.encode()): + if ignore[0] == '*': + if ignore[1:].encode() in (path+b"/"): + return True + elif (path+b"/").startswith(ignore.encode()): return True return False From 00ed7a1ecb70d7d7076e45d1eac4167286ab0a40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Plech=C5=A1m=C3=ADd=20Martin?= Date: Mon, 10 Aug 2020 18:45:15 +0200 Subject: [PATCH 6/9] Prevent complete breakage when snapshot creation fails --- bsync | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/bsync b/bsync index 08ad5a0..508195c 100755 --- a/bsync +++ b/bsync @@ -282,14 +282,21 @@ def fix_unresolved(ssh,dirname, snapname, unresolved): if sbproc.returncode != 0: sys.exit("Error in process updating a snapshot.") +def del_snapshot(ssh, dirname, snapname): + try: + if ssh==None: + os.remove(dirname+snapname) + else: + subprocess.check_call(ssh.getcmdlist()+["rm", "-f", dirname+snapname]) + except: + pass + # take a snapshot of files states from dir, using find. store it in .bsync-snap-XXXX # snap format: inode, path, type, date... -def make_snapshot(ssh,dirname, oldsnapname, newsnapname, unresolved): +def make_snapshot(ssh,dirname, newsnapname, unresolved): global findformat, findcmdlocal, findcmdremote cmd = [dirname, "-fprintf", dirname+newsnapname, findformat] - if oldsnapname!=None: - cmd+= ["&&", "rm", "-f", dirname+oldsnapname] # remove inconsistent newsnap if error in find cmd+= ["||", "(", "rm", "-f", dirname+newsnapname, "&&", "false", ")"] @@ -307,8 +314,13 @@ def make_snapshots(ssh1,dir1name, ssh2,dir2name, oldsnapname, unresolved): newsnapname = ".bsync-snap-"+datetime.datetime.now().strftime("%Y%m%d%H%M%S.%f") print("Updating filelists...") printv("Updating snap files: "+newsnapname+"...") - make_snapshot(ssh1,dir1name, oldsnapname,newsnapname, {fo.path:fo.r1 for fo in unresolved}) - make_snapshot(ssh2,dir2name, oldsnapname,newsnapname, {fo.path:fo.r2 for fo in unresolved}) + make_snapshot(ssh1, dir1name, newsnapname, {fo.path:fo.r1 for fo in unresolved}) + make_snapshot(ssh2, dir2name, newsnapname, {fo.path:fo.r2 for fo in unresolved}) + # remove old snapshots only when new snapshots were created successfully, in order to prevent + # breaking backup completely when snapshot1 is created successfully but snapshot2 fails + if oldsnapname != None: + del_snapshot(ssh1, dir1name, oldsnapname) + del_snapshot(ssh2, dir2name, oldsnapname) # run find in a directory to dump its content def get_find_proc(ssh, dirname): From a2fe5173e9d22016ab234da782a31a40f27a292d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Plech=C5=A1m=C3=ADd=20Martin?= Date: Tue, 11 Aug 2020 02:36:14 +0200 Subject: [PATCH 7/9] unittest logging improvement --- tests/bsync_tests.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/bsync_tests.py b/tests/bsync_tests.py index 91c79fa..e54d4ec 100644 --- a/tests/bsync_tests.py +++ b/tests/bsync_tests.py @@ -1,28 +1,52 @@ import unittest, tempfile, os, subprocess, shutil +import inspect dir1 = "dir1" dir2 = "dir2" +def unittest_verbosity(): + """Return the verbosity setting of the currently running unittest + program, or 0 if none is running. + (https://stackoverflow.com/a/32883243/1259360) + """ + frame = inspect.currentframe() + while frame: + self = frame.f_locals.get('self') + if isinstance(self, unittest.TestProgram): + return self.verbosity + frame = frame.f_back + return 0 + class TestBase(unittest.TestCase): def setUp(self): + self._verbosity = unittest_verbosity() self._tempdir = tempfile.mkdtemp() self.dir1 = os.path.join(self._tempdir, dir1) self.dir2 = os.path.join(self._tempdir, dir2) os.mkdir(self.dir1) os.mkdir(self.dir2) self.counter = 0 - + def tearDown(self): shutil.rmtree(self._tempdir) pass def bsync(self, args): - with subprocess.Popen(["bsync"]+args+[self.dir1, self.dir2], shell=True, stdout=subprocess.PIPE) as proc: + verbArg = [] + if self._verbosity >= 2: + print('Executing:\n bsync %s "%s" "%s"' % (' '.join(args), self.dir1, self.dir2)) + verbArg = ["-v"] + with subprocess.Popen(["bsync"]+verbArg+args+[self.dir1, self.dir2], shell=True, stdout=subprocess.PIPE) as proc: fd = proc.stdout output = fd.read() fd.close() proc.wait() + if self._verbosity >= 2: + print("Output from bsync execution:") + print("vvvvvvvvvvvvvvvvvvvvvvvvvvvv") + print(output.decode('ascii')) + print("^^^^^^^^^^^^^^^^^^^^^^^^^^^^") self.assertEqual(proc.returncode, 0, "bsync failed with code %d" % proc.returncode) return output @@ -245,4 +269,4 @@ def test_mirror_after_backup(self): self.assertFileContains(dir2, "b", 2) if __name__ == '__main__': - unittest.main(verbosity=2) + unittest.main() From c4bd913385f60ff8ba77259e8aa031e0ba7a3054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Plech=C5=A1m=C3=ADd=20Martin?= Date: Tue, 11 Aug 2020 02:37:23 +0200 Subject: [PATCH 8/9] Snapshot corruption replaced with path conflict --- bsync | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bsync b/bsync index 508195c..b53acd9 100755 --- a/bsync +++ b/bsync @@ -496,11 +496,12 @@ def load_orig(ssh1,dir1name, ssh2,dir2name): inode,path,type,date,size,perms,rec = record if not ignorepath(path, ignores): - #path not in orig: can happen if using ignore, then removing ignore, path will be considered as new + # path not in orig: can happen if using ignore, then removing ignore, path will be considered as new if path in orig: origfile = orig[path] if origfile.type != type or origfile.date != date or origfile.size != size or origfile.perms != perms: - sys.exit("Error: difference in snaps for path: "+tostr(path)) + # causing conflict for this path on both sides (better than reporting snapshot corruption) + origfile.date = -1 origfile.i2 = inode #set the second inode origfile.r2 = rec #set the second record From 9ccb433f0a1157ac66025e7a28dd97dc307f6032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Plech=C5=A1m=C3=ADd=20Martin?= Date: Tue, 11 Aug 2020 03:01:00 +0200 Subject: [PATCH 9/9] unittest logging improvement --- tests/bsync_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bsync_tests.py b/tests/bsync_tests.py index e54d4ec..1ce8537 100644 --- a/tests/bsync_tests.py +++ b/tests/bsync_tests.py @@ -269,4 +269,4 @@ def test_mirror_after_backup(self): self.assertFileContains(dir2, "b", 2) if __name__ == '__main__': - unittest.main() + unittest.main(verbosity=1.5)