deltatar: fixing some problems with diff engine and adding more unit tests

diff --git a/deltatar/deltatar.py b/deltatar/deltatar.py
index e04ab23..c1fb58b 100644 (file)
--- a/deltatar/deltatar.py
+++ b/deltatar/deltatar.py
@@ -686,12 +686,12 @@ class DeltaTar(object):
                 index_fd.write(s)
             elif action == 'delete':
                 stat = {
-                    u'path': u'delete://' + ipath['path'],
+                    u'path': u'delete://' + self.unprefixed(ipath['path']),
                     u'type': ipath['type']
                 }
 
                 # mark it as deleted in the backup
-                tarobj.add("/dev/null", arcname='delete://' + ipath['path'])
+                tarobj.add("/dev/null", arcname=stat['path'])
 
                 # store in the index the stat dict
                 s = json.dumps(stat) + '\n'
@@ -699,7 +699,7 @@ class DeltaTar(object):
                 index_fd.write(s)
             elif action == 'list':
                 stat = dpath.copy()
-                stat['path'] = u'list://' + ipath['path']
+                stat['path'] = u'list://' + self.unprefixed(ipath['path'])
                 # unchanged files do not enter in the backup, only in the index
 
                 # store in the index the stat dict
@@ -809,8 +809,15 @@ class DeltaTar(object):
                 index2 = self.unprefixed(elem2['path'])
 
             if index1 < index2:
-                yield (elem1, None)
-                elem1 = None
+                # if the number of dirs in index1 is greater than in index2,
+                # it means that there's a new parent directory in index2, so
+                # it goes first
+                if index1.count('/') > index2.count('/'):
+                    yield (None, elem2)
+                    elem2 = None
+                else:
+                    yield (elem1, None)
+                    elem1 = None
             elif index1 == index2:
                 yield (elem1, elem2)
                 elem1, elem2 = None, None
@@ -916,7 +923,11 @@ class DeltaTar(object):
                     return cls.filter_path(tarinfo.path, '.', tarinfo.isdir()) != NO_MATCH
                 elif tarinfo.path.startswith("delete://"):
                     path = self.unprefixed(tarinfo.path)
-                    shutil.rmtree(path)
+                    if os.path.exists(path):
+                        if not os.path.isdir(path):
+                            os.unlink(path)
+                        else:
+                            shutil.rmtree(path)
                     return False
                 else:
                     return False

diff --git a/testing/test_deltatar.py b/testing/test_deltatar.py
index e31d438..7ba5c0b 100644 (file)
--- a/testing/test_deltatar.py
+++ b/testing/test_deltatar.py
@@ -61,7 +61,7 @@ class DeltaTarTest(BaseTest):
         '''
         Remove temporal files created by unit tests
         '''
-        os.system("rm -rf source_dir source_dir2 backup_dir huge")
+        os.system("rm -rf source_dir source_dir2 backup_dir backup_dir? huge")
 
     def test_restore_simple_full_backup(self):
         '''
@@ -781,12 +781,65 @@ class DeltaTarTest(BaseTest):
         finally:
             os.chdir(cwd)
 
-    def test_create_empty_diff_backup(self):
+    def test_collate_iterators_diffdirs2(self):
         '''
-        Creates an empty (no changes) backup diff
+        Use the collate iterators functionality with two different directories.
+        It must behave in an expected way.
         '''
+
+        deltatar = DeltaTar(mode=self.MODE, password=self.PASSWORD,
+                            logger=self.consoleLogger)
+
+        # create first backup
+        deltatar.create_full_backup(
+            source_path="source_dir",
+            backup_path="backup_dir")
+
+        assert os.path.exists("backup_dir")
+
+        # add some new files and directories
+        os.makedirs('source_dir/bigdir')
+        self.hash["source_dir/bigdir"] = ""
+        self.hash["source_dir/bigdir/a"] = self.create_file("source_dir/bigdir/a", 100)
+        self.hash["source_dir/bigdir/b"] = self.create_file("source_dir/bigdir/b", 500)
         self.hash["source_dir/zzzz"]  = self.create_file("source_dir/zzzz", 100)
 
+        cwd = os.getcwd()
+        index_filename = deltatar.index_name_func(is_full=True)
+        index_path = os.path.join(cwd, "backup_dir", index_filename)
+        index_it = deltatar.iterate_index_path(index_path)
+
+        os.chdir('source_dir')
+        dir_it = deltatar._recursive_walk_dir('.')
+        path_it = deltatar.jsonize_path_iterator(dir_it)
+
+        visited_pairs = []
+
+        try:
+            for path1, path2 in deltatar.collate_iterators(index_it, path_it):
+                visited_pairs.append(
+                    (deltatar.unprefixed(path1['path']) if path1 else None,
+                     path2['path'] if path2 else None)
+                )
+        finally:
+            assert visited_pairs == [
+                (u'./big', u'./big'),
+                (None, u'./bigdir'),
+                (u'./small', u'./small'),
+                (u'./test', u'./test'),
+                (None, u'./zzzz'),
+                (u'./test/huge', u'./test/huge'),
+                (u'./test/huge2', u'./test/huge2'),
+                (u'./test/test2', u'./test/test2'),
+                (None, u'./bigdir/a'),
+                (None, u'./bigdir/b')
+            ]
+            os.chdir(cwd)
+
+    def test_create_empty_diff_backup(self):
+        '''
+        Creates an empty (no changes) backup diff
+        '''
         deltatar = DeltaTar(mode=self.MODE, password=self.PASSWORD,
                             logger=self.consoleLogger)
 
@@ -824,6 +877,60 @@ class DeltaTarTest(BaseTest):
         assert len(os.listdir("source_dir")) == 0
 
 
+    def test_create_diff_backup1(self):
+        '''
+        Creates a diff backup when there are new files
+        '''
+        deltatar = DeltaTar(mode=self.MODE, password=self.PASSWORD,
+                            logger=self.consoleLogger)
+
+        # create first backup
+        deltatar.create_full_backup(
+            source_path="source_dir",
+            backup_path="backup_dir")
+
+        prev_index_filename = deltatar.index_name_func(is_full=True)
+        prev_index_path = os.path.join("backup_dir", prev_index_filename)
+
+        # add some new files and directories
+        os.makedirs('source_dir/bigdir')
+        self.hash["source_dir/bigdir"] = ""
+        self.hash["source_dir/bigdir/a"] = self.create_file("source_dir/bigdir/a", 100)
+        self.hash["source_dir/bigdir/b"] = self.create_file("source_dir/bigdir/b", 500)
+        self.hash["source_dir/zzzz"]  = self.create_file("source_dir/zzzz", 100)
+
+        deltatar.create_diff_backup("source_dir", "backup_dir2",
+                                    prev_index_path)
+
+        # check index items
+        index_path = os.path.join("backup_dir2", prev_index_filename)
+        index_it = deltatar.iterate_index_path(index_path)
+        assert [i[0]['path'] for i in index_it] == [
+            'list://./big',
+            'snapshot://./bigdir',
+            'list://./small',
+            'list://./test',
+            'snapshot://./zzzz',
+            'list://./test/huge',
+            'list://./test/huge2',
+            'list://./test/test2',
+            'snapshot://./bigdir/a',
+            'snapshot://./bigdir/b'
+        ]
+
+        # check the tar file
+        assert os.path.exists("backup_dir2")
+        shutil.rmtree("source_dir")
+
+        tar_filename = deltatar.volume_name_func('backup_dir2', True, 0)
+        tar_path = os.path.join("backup_dir2", tar_filename)
+
+        # restore the backup, this will create only the new files
+        deltatar.restore_backup(target_path="source_dir",
+                                backup_tar_path=tar_path)
+        assert os.listdir("source_dir") == ['zzzz', 'bigdir']
+
+
 class DeltaTar2Test(DeltaTarTest):
     '''
     Same as DeltaTar but with specific ":" mode