From 42d39ca7f7f308e3df5b2947c9ef75459211b6fa Mon Sep 17 00:00:00 2001 From: Eduardo Robles Elvira Date: Tue, 6 Aug 2013 09:44:16 +0200 Subject: [PATCH] deltatar: fixing some problems with diff engine and adding more unit tests --- deltatar/deltatar.py | 23 +++++++--- testing/test_deltatar.py | 113 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 127 insertions(+), 9 deletions(-) diff --git a/deltatar/deltatar.py b/deltatar/deltatar.py index e04ab23..c1fb58b 100644 --- a/deltatar/deltatar.py +++ b/deltatar/deltatar.py @@ -686,12 +686,12 @@ class DeltaTar(object): index_fd.write(s) elif action == 'delete': stat = { - u'path': u'delete://' + ipath['path'], + u'path': u'delete://' + self.unprefixed(ipath['path']), u'type': ipath['type'] } # mark it as deleted in the backup - tarobj.add("/dev/null", arcname='delete://' + ipath['path']) + tarobj.add("/dev/null", arcname=stat['path']) # store in the index the stat dict s = json.dumps(stat) + '\n' @@ -699,7 +699,7 @@ class DeltaTar(object): index_fd.write(s) elif action == 'list': stat = dpath.copy() - stat['path'] = u'list://' + ipath['path'] + stat['path'] = u'list://' + self.unprefixed(ipath['path']) # unchanged files do not enter in the backup, only in the index # store in the index the stat dict @@ -809,8 +809,15 @@ class DeltaTar(object): index2 = self.unprefixed(elem2['path']) if index1 < index2: - yield (elem1, None) - elem1 = None + # if the number of dirs in index1 is greater than in index2, + # it means that there's a new parent directory in index2, so + # it goes first + if index1.count('/') > index2.count('/'): + yield (None, elem2) + elem2 = None + else: + yield (elem1, None) + elem1 = None elif index1 == index2: yield (elem1, elem2) elem1, elem2 = None, None @@ -916,7 +923,11 @@ class DeltaTar(object): return cls.filter_path(tarinfo.path, '.', tarinfo.isdir()) != NO_MATCH elif tarinfo.path.startswith("delete://"): path = self.unprefixed(tarinfo.path) - shutil.rmtree(path) + if os.path.exists(path): + if not os.path.isdir(path): + os.unlink(path) + else: + shutil.rmtree(path) return False else: return False diff --git a/testing/test_deltatar.py b/testing/test_deltatar.py index e31d438..7ba5c0b 100644 --- a/testing/test_deltatar.py +++ b/testing/test_deltatar.py @@ -61,7 +61,7 @@ class DeltaTarTest(BaseTest): ''' Remove temporal files created by unit tests ''' - os.system("rm -rf source_dir source_dir2 backup_dir huge") + os.system("rm -rf source_dir source_dir2 backup_dir backup_dir? huge") def test_restore_simple_full_backup(self): ''' @@ -781,12 +781,65 @@ class DeltaTarTest(BaseTest): finally: os.chdir(cwd) - def test_create_empty_diff_backup(self): + def test_collate_iterators_diffdirs2(self): ''' - Creates an empty (no changes) backup diff + Use the collate iterators functionality with two different directories. + It must behave in an expected way. ''' + + deltatar = DeltaTar(mode=self.MODE, password=self.PASSWORD, + logger=self.consoleLogger) + + # create first backup + deltatar.create_full_backup( + source_path="source_dir", + backup_path="backup_dir") + + assert os.path.exists("backup_dir") + + # add some new files and directories + os.makedirs('source_dir/bigdir') + self.hash["source_dir/bigdir"] = "" + self.hash["source_dir/bigdir/a"] = self.create_file("source_dir/bigdir/a", 100) + self.hash["source_dir/bigdir/b"] = self.create_file("source_dir/bigdir/b", 500) self.hash["source_dir/zzzz"] = self.create_file("source_dir/zzzz", 100) + cwd = os.getcwd() + index_filename = deltatar.index_name_func(is_full=True) + index_path = os.path.join(cwd, "backup_dir", index_filename) + index_it = deltatar.iterate_index_path(index_path) + + os.chdir('source_dir') + dir_it = deltatar._recursive_walk_dir('.') + path_it = deltatar.jsonize_path_iterator(dir_it) + + visited_pairs = [] + + try: + for path1, path2 in deltatar.collate_iterators(index_it, path_it): + visited_pairs.append( + (deltatar.unprefixed(path1['path']) if path1 else None, + path2['path'] if path2 else None) + ) + finally: + assert visited_pairs == [ + (u'./big', u'./big'), + (None, u'./bigdir'), + (u'./small', u'./small'), + (u'./test', u'./test'), + (None, u'./zzzz'), + (u'./test/huge', u'./test/huge'), + (u'./test/huge2', u'./test/huge2'), + (u'./test/test2', u'./test/test2'), + (None, u'./bigdir/a'), + (None, u'./bigdir/b') + ] + os.chdir(cwd) + + def test_create_empty_diff_backup(self): + ''' + Creates an empty (no changes) backup diff + ''' deltatar = DeltaTar(mode=self.MODE, password=self.PASSWORD, logger=self.consoleLogger) @@ -824,6 +877,60 @@ class DeltaTarTest(BaseTest): assert len(os.listdir("source_dir")) == 0 + def test_create_diff_backup1(self): + ''' + Creates a diff backup when there are new files + ''' + deltatar = DeltaTar(mode=self.MODE, password=self.PASSWORD, + logger=self.consoleLogger) + + # create first backup + deltatar.create_full_backup( + source_path="source_dir", + backup_path="backup_dir") + + prev_index_filename = deltatar.index_name_func(is_full=True) + prev_index_path = os.path.join("backup_dir", prev_index_filename) + + # add some new files and directories + os.makedirs('source_dir/bigdir') + self.hash["source_dir/bigdir"] = "" + self.hash["source_dir/bigdir/a"] = self.create_file("source_dir/bigdir/a", 100) + self.hash["source_dir/bigdir/b"] = self.create_file("source_dir/bigdir/b", 500) + self.hash["source_dir/zzzz"] = self.create_file("source_dir/zzzz", 100) + + deltatar.create_diff_backup("source_dir", "backup_dir2", + prev_index_path) + + # check index items + index_path = os.path.join("backup_dir2", prev_index_filename) + index_it = deltatar.iterate_index_path(index_path) + assert [i[0]['path'] for i in index_it] == [ + 'list://./big', + 'snapshot://./bigdir', + 'list://./small', + 'list://./test', + 'snapshot://./zzzz', + 'list://./test/huge', + 'list://./test/huge2', + 'list://./test/test2', + 'snapshot://./bigdir/a', + 'snapshot://./bigdir/b' + ] + + # check the tar file + assert os.path.exists("backup_dir2") + shutil.rmtree("source_dir") + + tar_filename = deltatar.volume_name_func('backup_dir2', True, 0) + tar_path = os.path.join("backup_dir2", tar_filename) + + # restore the backup, this will create only the new files + deltatar.restore_backup(target_path="source_dir", + backup_tar_path=tar_path) + assert os.listdir("source_dir") == ['zzzz', 'bigdir'] + + class DeltaTar2Test(DeltaTarTest): ''' Same as DeltaTar but with specific ":" mode -- 1.7.1