From 2fe5f6e794d5148ea1abe7a9d4456df09b074292 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Thu, 24 Aug 2017 11:56:14 +0200 Subject: [PATCH] add test skeleton for corrupt index reconstruction Starting with an intact backup set. --- runtests.py | 14 ++++- testing/test_recover.py | 163 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 138 insertions(+), 39 deletions(-) diff --git a/runtests.py b/runtests.py index c03a7df..146d6e4 100755 --- a/runtests.py +++ b/runtests.py @@ -52,7 +52,13 @@ from testing.test_recover import \ , RecoverCorruptVolumeGZAESTest \ , RecoverCorruptHoleTest \ , RecoverCorruptHoleGZTest \ - , RecoverCorruptHoleGZAESTest + , RecoverCorruptHoleGZAESTest \ + , RescueCorruptHoleTest \ + , RescueCorruptHoleGZTest \ + , RescueCorruptHoleGZAESTest \ + , GenIndexIntactTest \ + , GenIndexIntactGZTest \ + , GenIndexIntactGZAESTest from testing.test_rescue_tar import RescueTarTest from testing.test_encryption import EncryptionTest from testing.test_deltatar import (DeltaTarTest, DeltaTar2Test, @@ -117,6 +123,12 @@ if __name__ == "__main__": , RecoverCorruptHoleTest , RecoverCorruptHoleGZTest , RecoverCorruptHoleGZAESTest + , RescueCorruptHoleTest + , RescueCorruptHoleGZTest + , RescueCorruptHoleGZAESTest + , GenIndexIntactTest + , GenIndexIntactGZTest + , GenIndexIntactGZAESTest ]: try: t = group (n) diff --git a/testing/test_recover.py b/testing/test_recover.py index fbb31bd..817c719 100644 --- a/testing/test_recover.py +++ b/testing/test_recover.py @@ -3,6 +3,8 @@ import os import shutil import stat +from functools import partial + import deltatar.deltatar as deltatar import deltatar.crypto as crypto import deltatar.tarfile as tarfile @@ -182,6 +184,11 @@ def corrupt_hole (_, fname, compress, encrypt): os.link (path, aname, src_dir_fd=0, follow_symlinks=True) os.close (outfd) +def immaculate (_, _fname, _compress, _encrypt): + """ + No-op dummy. + """ + pass ############################################################################### ## tests ## @@ -227,19 +234,11 @@ class DefectiveTest (BaseTest): os.system("rm -rf source_dir source_dir2 backup_dir*") -class RecoverTest (DefectiveTest): - """ - Recover: restore corrupt backups from index file information. - """ + @staticmethod + def default_volume_name (backup_file, _x, _y, n, *a, **kwa): + return backup_file % n - def test_recover_corrupt (self): - """ - Perform various damaging actions that cause unreadable objects. - - Expects the extraction to fail in normal mode. With disaster recovery, - extraction must succeed, and exactly one file must be missing. - """ - mode = self.COMPRESSION or "#" + def gen_file_names (self, comp, pw): bak_path = "backup_dir" backup_file = "the_full_backup_%0.2d.tar" backup_full = ("%s/%s" % (bak_path, backup_file)) % 0 @@ -255,6 +254,25 @@ class RecoverTest (DefectiveTest): backup_full = "%s.%s" % (backup_full, deltatar.PDTCRYPT_EXTENSION) index_file = "%s.%s" % (index_file , deltatar.PDTCRYPT_EXTENSION) + return bak_path, backup_file, backup_full, index_file + + +class RecoverTest (DefectiveTest): + """ + Recover: restore corrupt backups from index file information. + """ + + def test_recover_corrupt (self): + """ + Perform various damaging actions that cause unreadable objects. + + Expects the extraction to fail in normal mode. With disaster recovery, + extraction must succeed, and exactly one file must be missing. + """ + mode = self.COMPRESSION or "#" + bak_path, backup_file, backup_full, index_file = \ + self.gen_file_names (self.COMPRESSION, self.PASSWORD) + if self.VOLUMES > 1: # add n files for one nth the volume size each, corrected # for metadata and tar block overhead @@ -270,9 +288,7 @@ class RecoverTest (DefectiveTest): fsiz, random=True) - def vname (_x, _y, n, *a, **kwa): - return backup_file % n - + vname = partial (self.default_volume_name, backup_file) dtar = deltatar.DeltaTar (mode=mode, logger=None, password=self.PASSWORD, @@ -365,21 +381,9 @@ class RescueTest (DefectiveTest): Perform various damaging actions that cause unreadable objects, then attempt to extract objects regardless. """ - mode = self.COMPRESSION or "#" - bak_path = "backup_dir" - backup_file = "the_full_backup_%0.2d.tar" - backup_full = ("%s/%s" % (bak_path, backup_file)) % 0 - index_file = "the_full_index" - - if self.COMPRESSION is not None: - backup_file += ".gz" - backup_full += ".gz" - index_file += ".gz" - - if self.PASSWORD is not None: - backup_file = "%s.%s" % (backup_file, deltatar.PDTCRYPT_EXTENSION) - backup_full = "%s.%s" % (backup_full, deltatar.PDTCRYPT_EXTENSION) - index_file = "%s.%s" % (index_file , deltatar.PDTCRYPT_EXTENSION) + mode = self.COMPRESSION or "#" + bak_path, backup_file, backup_full, index_file = \ + self.gen_file_names (self.COMPRESSION, self.PASSWORD) if self.VOLUMES > 1: # add n files for one nth the volume size each, corrected @@ -396,9 +400,7 @@ class RescueTest (DefectiveTest): fsiz, random=True) - def vname (_x, _y, n, *a, **kwa): - return backup_file % n - + vname = partial (self.default_volume_name, backup_file) dtar = deltatar.DeltaTar (mode=mode, logger=None, password=self.PASSWORD, @@ -451,9 +453,7 @@ class RescueTest (DefectiveTest): os.chdir (self.pwd) # not restored due to the error above # but recover will succeed failed = dtar.rescue_backup(target_path=self.dst_path, - backup_indexes_paths=[ - "%s/%s" % (bak_path, index_file) - ]) + backup_tar_path=backup_full) assert len (failed) == self.FAILURES @@ -468,13 +468,46 @@ class RescueTest (DefectiveTest): else: missing.append (key) - ssert len (missing) == (self.MISSING if self.MISSING is not None - else self.FAILURES) + assert len (missing) == (self.MISSING if self.MISSING is not None + else self.FAILURES) assert len (mismatch) == self.MISMATCHES shutil.rmtree (self.dst_path) +class GenIndexTest (DefectiveTest): + """ + Deducing an index for a backup with tarfile. + """ + + def test_gen_index (self): + """ + Create backup, leave it unharmed, then generate an index. + """ + mode = self.COMPRESSION or "#" + bak_path, backup_file, backup_full, index_file = \ + self.gen_file_names (self.COMPRESSION, self.PASSWORD) + + vname = partial (self.default_volume_name, backup_file) + dtar = deltatar.DeltaTar (mode=mode, + logger=None, + password=self.PASSWORD, + index_name_func=lambda _: index_file, + volume_name_func=vname) + + dtar.create_full_backup \ + (source_path=self.src_path, backup_path=bak_path, + max_volume_size=1) + + psidx = tarfile.gen_rescue_index (backup_full, mode, password=self.PASSWORD) + + assert len (psidx) == len (self.hash) + + +############################################################################### +# rescue +############################################################################### + class RecoverCorruptPayloadTestBase (RecoverTest): COMPRESSION = None PASSWORD = None @@ -697,3 +730,57 @@ class RecoverCorruptHoleGZAESTest (RecoverCorruptHoleBaseTest): PASSWORD = TEST_PASSWORD MISSING = 2 +############################################################################### +# rescue +############################################################################### + +class RescueCorruptHoleBaseTest (RescueTest): + """ + Cut bytes from the middle of a volume. + """ + COMPRESSION = None + PASSWORD = None + FAILURES = 3 + CORRUPT = corrupt_hole + VOLUMES = 2 # request two vols to swell up the first one + MISMATCHES = 1 + +class RescueCorruptHoleTest (RescueCorruptHoleBaseTest): + FAILURES = 2 + +class RescueCorruptHoleGZTest (RescueCorruptHoleBaseTest): + COMPRESSION = "#gz" + MISSING = 2 + +class RescueCorruptHoleGZAESTest (RescueCorruptHoleBaseTest): + COMPRESSION = "#gz" + PASSWORD = TEST_PASSWORD + MISSING = 2 + +############################################################################### +# index +############################################################################### + +class GenIndexIntactBaseTest (GenIndexTest): + """ + """ + COMPRESSION = None + PASSWORD = None + FAILURES = 0 + CORRUPT = immaculate + VOLUMES = 1 + MISMATCHES = 1 + + +class GenIndexIntactTest (GenIndexIntactBaseTest): + pass + +class GenIndexIntactGZTest (GenIndexIntactBaseTest): + COMPRESSION = "#gz" + MISSING = 2 + +class GenIndexIntactGZAESTest (GenIndexIntactBaseTest): + COMPRESSION = "#gz" + PASSWORD = TEST_PASSWORD + MISSING = 2 + -- 1.7.1