Commit | Line | Data |
---|---|---|
fbdc9f4a PG |
1 | import logging |
2 | import os | |
3 | import shutil | |
3692fd82 | 4 | import stat |
b9cf4a0f PG |
5 | import sys |
6 | import unittest | |
fbdc9f4a | 7 | |
2fe5f6e7 PG |
8 | from functools import partial |
9 | ||
fbdc9f4a | 10 | import deltatar.deltatar as deltatar |
3267933a | 11 | import deltatar.crypto as crypto |
203cb25e | 12 | import deltatar.tarfile as tarfile |
fbdc9f4a PG |
13 | |
14 | from . import BaseTest | |
15 | ||
e25f31ac | 16 | TEST_PASSWORD = "test1234" |
85e7013f | 17 | TEST_VOLSIZ = 2 # MB |
e25f31ac | 18 | TEST_FILESPERVOL = 3 |
85e7013f PG |
19 | VOLUME_OVERHEAD = 1.4 # account for tar overhead when fitting files into |
20 | # volumes; this is black magic | |
20e1d773 | 21 | TEST_BLOCKSIZE = 4096 |
96fe6399 PG |
22 | |
23 | ############################################################################### | |
24 | ## helpers ## | |
25 | ############################################################################### | |
26 | ||
3267933a PG |
27 | def flip_bits (fname, off, b=0x01, n=1): |
28 | """ | |
29 | Open file *fname* at offset *off*, replacing the next *n* bytes with | |
30 | their values xor’ed with *b*. | |
31 | """ | |
32 | fd = os.open (fname, os.O_RDWR) | |
203cb25e | 33 | |
3267933a PG |
34 | try: |
35 | pos = os.lseek (fd, off, os.SEEK_SET) | |
36 | assert pos == off | |
37 | chunk = os.read (fd, n) | |
38 | chunk = bytes (map (lambda v: v ^ b, chunk)) | |
da8996f0 PG |
39 | pos = os.lseek (fd, off, os.SEEK_SET) |
40 | assert pos == off | |
3267933a PG |
41 | os.write (fd, chunk) |
42 | finally: | |
43 | os.close (fd) | |
44 | ||
203cb25e PG |
45 | |
46 | def gz_header_size (fname, off=0): | |
47 | """ | |
48 | Determine the length of the gzip header starting at *off* in file fname. | |
49 | ||
50 | The header is variable length because it may contain the filename as NUL | |
51 | terminated bytes. | |
52 | """ | |
53 | # length so we need to determine where the actual payload starts | |
54 | off = tarfile.GZ_HEADER_SIZE | |
55 | fd = os.open (fname, os.O_RDONLY) | |
56 | ||
57 | try: | |
58 | pos = os.lseek (fd, off, os.SEEK_SET) | |
59 | assert pos == off | |
60 | while os.read (fd, 1)[0] != 0: | |
61 | off += 1 | |
62 | pos = os.lseek (fd, off, os.SEEK_SET) | |
63 | assert pos == off | |
64 | finally: | |
65 | os.close (fd) | |
66 | ||
67 | return off | |
68 | ||
da8996f0 | 69 | |
96fe6399 PG |
70 | def is_pdt_encrypted (fname): |
71 | """ | |
72 | Returns true if the file contains at least one PDT header plus enough | |
73 | space for the object. | |
74 | """ | |
75 | try: | |
76 | with open (fname, "rb") as st: | |
77 | hdr = crypto.hdr_read_stream (st) | |
78 | siz = hdr ["ctsize"] | |
79 | assert (len (st.read (siz)) == siz) | |
80 | except Exception as exn: | |
81 | return False | |
82 | return True | |
83 | ||
84 | ||
3692fd82 PG |
85 | ############################################################################### |
86 | ## corruption simulators ## | |
87 | ############################################################################### | |
88 | ||
0c8baf2b PG |
89 | class UndefinedTest (Exception): |
90 | """No test available for the asked combination of parameters.""" | |
91 | ||
00b8c150 PG |
92 | def corrupt_header (_, fname, compress, encrypt): |
93 | """ | |
94 | Modify a significant byte in the object header of the format. | |
95 | """ | |
96 | if encrypt is True: # damage GCM tag | |
97 | flip_bits (fname, crypto.HDR_OFF_TAG + 1) | |
98 | elif compress is True: # invalidate magic | |
99 | flip_bits (fname, 1) | |
100 | else: # Fudge checksum. From tar(5): | |
101 | # | |
102 | # struct header_gnu_tar { | |
103 | # char name[100]; | |
104 | # char mode[8]; | |
105 | # char uid[8]; | |
106 | # char gid[8]; | |
107 | # char size[12]; | |
108 | # char mtime[12]; | |
109 | # char checksum[8]; | |
110 | # … | |
111 | flip_bits (fname, 100 + 8 + 8 + 8 + 12 + 12 + 1) | |
112 | ||
113 | ||
0c8baf2b PG |
114 | def corrupt_ctsize (_, fname, compress, encrypt): |
115 | """ | |
116 | Blow up the size of an object so as to cause its apparent payload to leak | |
117 | into the next one. | |
118 | """ | |
119 | if encrypt is True: | |
120 | # damage lowest bit of second least significant byte of size field; | |
121 | # this effectively sets the ciphertext size to 422, causing it to | |
122 | # extend over the next object into the third one. | |
123 | return flip_bits (fname, crypto.HDR_OFF_CTSIZE + 1, b=0x01) | |
124 | raise UndefinedTest ("corrupt_ctsize %s %s %s" % (fname, compress, encrypt)) | |
125 | ||
126 | ||
da8996f0 PG |
127 | def corrupt_entire_header (_, fname, compress, encrypt): |
128 | """ | |
129 | Flip all bits in the first object header. | |
130 | """ | |
131 | if encrypt is True: | |
132 | flip_bits (fname, 0, 0xff, crypto.PDTCRYPT_HDR_SIZE) | |
133 | elif compress is True: # invalidate magic | |
134 | flip_bits (fname, 0, 0xff, gz_header_size (fname)) | |
135 | else: | |
136 | flip_bits (fname, 0, 0xff, tarfile.BLOCKSIZE) | |
137 | ||
138 | ||
00b8c150 PG |
139 | def corrupt_payload_start (_, fname, compress, encrypt): |
140 | """ | |
141 | Modify the byte following the object header structure of the format. | |
142 | """ | |
143 | if encrypt is True: | |
144 | flip_bits (fname, crypto.PDTCRYPT_HDR_SIZE + 1) | |
145 | elif compress is True: | |
146 | flip_bits (fname, gz_header_size (fname) + 1) | |
147 | else: | |
148 | flip_bits (fname, tarfile.BLOCKSIZE + 1) | |
149 | ||
150 | ||
afb2d647 PG |
151 | def corrupt_leading_garbage (_, fname, compress, encrypt): |
152 | """ | |
153 | Prepend junk to file. | |
154 | """ | |
155 | aname = os.path.abspath (fname) | |
156 | infd = os.open (fname, os.O_RDONLY) | |
157 | size = os.lseek (infd, 0, os.SEEK_END) | |
158 | assert os.lseek (infd, 0, os.SEEK_SET) == 0 | |
159 | outfd = os.open (os.path.dirname (aname), os.O_WRONLY | os.O_TMPFILE, | |
160 | stat.S_IRUSR | stat.S_IWUSR) | |
a793ee30 | 161 | junk = os.urandom (42) |
afb2d647 PG |
162 | |
163 | # write new file with garbage prepended | |
164 | done = 0 | |
165 | os.write (outfd, junk) # junk first | |
166 | done += len (junk) | |
167 | while done < size: | |
168 | data = os.read (infd, TEST_BLOCKSIZE) | |
169 | os.write (outfd, data) | |
170 | done += len (data) | |
171 | ||
172 | assert os.lseek (outfd, 0, os.SEEK_CUR) == done | |
173 | ||
174 | # close and free old file | |
175 | os.close (infd) | |
176 | os.unlink (fname) | |
177 | ||
178 | # install the new file in its place, atomically | |
179 | path = "/proc/self/fd/%d" % outfd | |
180 | os.link (path, aname, src_dir_fd=0, follow_symlinks=True) | |
181 | os.close (outfd) | |
182 | ||
183 | ||
517d35b7 PG |
184 | def corrupt_trailing_data (_, fname, compress, encrypt): |
185 | """ | |
186 | Modify the byte following the object header structure of the format. | |
187 | """ | |
188 | junk = os.urandom (42) | |
189 | fd = os.open (fname, os.O_WRONLY | os.O_APPEND) | |
190 | os.write (fd, junk) | |
191 | os.close (fd) | |
192 | ||
00b8c150 | 193 | |
20e1d773 PG |
194 | def corrupt_volume (_, fname, compress, encrypt): |
195 | """ | |
196 | Zero out an entire volume. | |
197 | """ | |
198 | fd = os.open (fname, os.O_WRONLY) | |
199 | size = os.lseek (fd, 0, os.SEEK_END) | |
200 | assert os.lseek (fd, 0, os.SEEK_SET) == 0 | |
201 | zeros = bytes (b'\x00' * TEST_BLOCKSIZE) | |
202 | while size > 0: | |
203 | todo = min (size, TEST_BLOCKSIZE) | |
204 | os.write (fd, zeros [:todo]) | |
205 | size -= todo | |
206 | os.close (fd) | |
207 | ||
208 | ||
3692fd82 PG |
209 | def corrupt_hole (_, fname, compress, encrypt): |
210 | """ | |
211 | Cut file in three pieces, reassemble without the middle one. | |
212 | """ | |
213 | aname = os.path.abspath (fname) | |
214 | infd = os.open (fname, os.O_RDONLY) | |
215 | size = os.lseek (infd, 0, os.SEEK_END) | |
216 | assert os.lseek (infd, 0, os.SEEK_SET) == 0 | |
217 | assert size > 3 * TEST_BLOCKSIZE | |
218 | hole = (size / 3, size * 2 / 3) | |
219 | outfd = os.open (os.path.dirname (aname), os.O_WRONLY | os.O_TMPFILE, | |
220 | stat.S_IRUSR | stat.S_IWUSR) | |
221 | ||
3692fd82 PG |
222 | done = 0 |
223 | while done < size: | |
224 | data = os.read (infd, TEST_BLOCKSIZE) | |
225 | if done < hole [0] or hole [1] < done: | |
226 | # only copy from outside hole | |
227 | os.write (outfd, data) | |
228 | done += len (data) | |
229 | ||
230 | os.close (infd) | |
231 | os.unlink (fname) | |
232 | ||
233 | path = "/proc/self/fd/%d" % outfd | |
234 | os.link (path, aname, src_dir_fd=0, follow_symlinks=True) | |
235 | os.close (outfd) | |
236 | ||
2fe5f6e7 PG |
237 | def immaculate (_, _fname, _compress, _encrypt): |
238 | """ | |
239 | No-op dummy. | |
240 | """ | |
241 | pass | |
3692fd82 | 242 | |
96fe6399 PG |
243 | ############################################################################### |
244 | ## tests ## | |
245 | ############################################################################### | |
203cb25e | 246 | |
0c6682ce | 247 | class DefectiveTest (BaseTest): |
fbdc9f4a PG |
248 | """ |
249 | Disaster recovery: restore corrupt backups. | |
250 | """ | |
251 | ||
96fe6399 PG |
252 | COMPRESSION = None |
253 | PASSWORD = None | |
9d89c237 PG |
254 | FAILURES = 0 # files that could not be restored |
255 | MISMATCHES = 0 # files that were restored but corrupted | |
00b8c150 | 256 | CORRUPT = corrupt_payload_start |
e25f31ac | 257 | VOLUMES = 1 |
4d4925de | 258 | MISSING = None # normally the number of failures |
96fe6399 | 259 | |
fbdc9f4a PG |
260 | |
261 | def setUp(self): | |
262 | ''' | |
263 | Create base test data | |
264 | ''' | |
96fe6399 PG |
265 | self.pwd = os.getcwd() |
266 | self.dst_path = "source_dir" | |
267 | self.src_path = "%s2" % self.dst_path | |
268 | self.hash = dict() | |
269 | ||
fbdc9f4a | 270 | os.system('rm -rf target_dir source_dir* backup_dir* huge') |
96fe6399 | 271 | os.makedirs (self.src_path) |
fbdc9f4a | 272 | |
96fe6399 | 273 | for i in range (5): |
85e7013f | 274 | f = "dummy_%d" % i |
96fe6399 PG |
275 | self.hash [f] = self.create_file ("%s/%s" |
276 | % (self.src_path, f), 5 + i) | |
fbdc9f4a | 277 | |
96fe6399 PG |
278 | |
279 | def tearDown(self): | |
280 | ''' | |
281 | Remove temporal files created by unit tests and reset globals. | |
282 | ''' | |
283 | os.chdir(self.pwd) | |
284 | os.system("rm -rf source_dir source_dir2 backup_dir*") | |
fbdc9f4a PG |
285 | |
286 | ||
2fe5f6e7 PG |
287 | @staticmethod |
288 | def default_volume_name (backup_file, _x, _y, n, *a, **kwa): | |
289 | return backup_file % n | |
0c6682ce | 290 | |
2fe5f6e7 | 291 | def gen_file_names (self, comp, pw): |
203cb25e | 292 | bak_path = "backup_dir" |
e25f31ac PG |
293 | backup_file = "the_full_backup_%0.2d.tar" |
294 | backup_full = ("%s/%s" % (bak_path, backup_file)) % 0 | |
96fe6399 PG |
295 | index_file = "the_full_index" |
296 | ||
297 | if self.COMPRESSION is not None: | |
298 | backup_file += ".gz" | |
299 | backup_full += ".gz" | |
300 | index_file += ".gz" | |
301 | ||
302 | if self.PASSWORD is not None: | |
e25f31ac PG |
303 | backup_file = "%s.%s" % (backup_file, deltatar.PDTCRYPT_EXTENSION) |
304 | backup_full = "%s.%s" % (backup_full, deltatar.PDTCRYPT_EXTENSION) | |
305 | index_file = "%s.%s" % (index_file , deltatar.PDTCRYPT_EXTENSION) | |
306 | ||
2fe5f6e7 PG |
307 | return bak_path, backup_file, backup_full, index_file |
308 | ||
309 | ||
047239f3 PG |
310 | def gen_multivol (self, nvol): |
311 | # add n files for one nth the volume size each, corrected | |
312 | # for metadata and tar block overhead | |
313 | fsiz = int ( ( TEST_VOLSIZ | |
314 | / (TEST_FILESPERVOL * VOLUME_OVERHEAD)) | |
315 | * 1024 * 1024) | |
316 | fcnt = (self.VOLUMES - 1) * TEST_FILESPERVOL | |
317 | for i in range (fcnt): | |
318 | nvol, invol = divmod(i, TEST_FILESPERVOL) | |
319 | f = "dummy_vol_%d_n_%0.2d" % (nvol, invol) | |
320 | self.hash [f] = self.create_file ("%s/%s" | |
321 | % (self.src_path, f), | |
322 | fsiz, | |
323 | random=True) | |
324 | ||
325 | ||
2fe5f6e7 PG |
326 | class RecoverTest (DefectiveTest): |
327 | """ | |
328 | Recover: restore corrupt backups from index file information. | |
329 | """ | |
330 | ||
331 | def test_recover_corrupt (self): | |
332 | """ | |
333 | Perform various damaging actions that cause unreadable objects. | |
334 | ||
335 | Expects the extraction to fail in normal mode. With disaster recovery, | |
336 | extraction must succeed, and exactly one file must be missing. | |
337 | """ | |
338 | mode = self.COMPRESSION or "#" | |
339 | bak_path, backup_file, backup_full, index_file = \ | |
340 | self.gen_file_names (self.COMPRESSION, self.PASSWORD) | |
341 | ||
e25f31ac | 342 | if self.VOLUMES > 1: |
047239f3 | 343 | self.gen_multivol (self.VOLUMES) |
e25f31ac | 344 | |
2fe5f6e7 | 345 | vname = partial (self.default_volume_name, backup_file) |
96fe6399 PG |
346 | dtar = deltatar.DeltaTar (mode=mode, |
347 | logger=None, | |
348 | password=self.PASSWORD, | |
203cb25e | 349 | index_name_func=lambda _: index_file, |
3267933a | 350 | volume_name_func=vname) |
fbdc9f4a PG |
351 | |
352 | dtar.create_full_backup \ | |
e25f31ac PG |
353 | (source_path=self.src_path, backup_path=bak_path, |
354 | max_volume_size=1) | |
96fe6399 PG |
355 | |
356 | if self.PASSWORD is not None: | |
357 | # ensure all files are at least superficially in PDT format | |
358 | for f in os.listdir (bak_path): | |
359 | assert is_pdt_encrypted ("%s/%s" % (bak_path, f)) | |
203cb25e PG |
360 | |
361 | # first restore must succeed | |
96fe6399 | 362 | dtar.restore_backup(target_path=self.dst_path, |
f090d35a PG |
363 | backup_indexes_paths=[ |
364 | "%s/%s" % (bak_path, index_file) | |
365 | ]) | |
203cb25e | 366 | for key, value in self.hash.items (): |
96fe6399 | 367 | f = "%s/%s" % (self.dst_path, key) |
b15e549b PG |
368 | assert os.path.exists (f) |
369 | assert value == self.md5sum (f) | |
96fe6399 PG |
370 | shutil.rmtree (self.dst_path) |
371 | shutil.rmtree (self.src_path) | |
203cb25e | 372 | |
00b8c150 PG |
373 | self.CORRUPT (backup_full, |
374 | self.COMPRESSION is not None, | |
375 | self.PASSWORD is not None) | |
203cb25e PG |
376 | |
377 | # normal restore must fail | |
96fe6399 PG |
378 | try: |
379 | dtar.restore_backup(target_path=self.dst_path, | |
203cb25e | 380 | backup_tar_path=backup_full) |
96fe6399 PG |
381 | except tarfile.CompressionError: |
382 | if self.PASSWORD is not None or self.COMPRESSION is not None: | |
383 | pass | |
00b8c150 PG |
384 | else: |
385 | raise | |
96fe6399 | 386 | except tarfile.ReadError: |
00b8c150 PG |
387 | # can happen with all three modes |
388 | pass | |
389 | except tarfile.DecryptionError: | |
390 | if self.PASSWORD is not None: | |
96fe6399 | 391 | pass |
00b8c150 PG |
392 | else: |
393 | raise | |
96fe6399 PG |
394 | |
395 | os.chdir (self.pwd) # not restored due to the error above | |
203cb25e | 396 | # but recover will succeed |
96fe6399 | 397 | failed = dtar.recover_backup(target_path=self.dst_path, |
b15e549b PG |
398 | backup_indexes_paths=[ |
399 | "%s/%s" % (bak_path, index_file) | |
400 | ]) | |
96fe6399 PG |
401 | |
402 | assert len (failed) == self.FAILURES | |
203cb25e PG |
403 | |
404 | # with one file missing | |
9d89c237 PG |
405 | missing = [] |
406 | mismatch = [] | |
203cb25e | 407 | for key, value in self.hash.items (): |
96fe6399 | 408 | kkey = "%s/%s" % (self.dst_path, key) |
b15e549b | 409 | if os.path.exists (kkey): |
9d89c237 PG |
410 | if value != self.md5sum (kkey): |
411 | mismatch.append (key) | |
203cb25e | 412 | else: |
757319dd | 413 | missing.append (key) |
4d4925de PG |
414 | |
415 | # usually, an object whose extraction fails will not be found on | |
416 | # disk afterwards so the number of failures equals that of missing | |
417 | # files. however, some modes will create partial files for objects | |
418 | # spanning multiple volumes that contain the parts whose checksums | |
419 | # were valid. | |
420 | assert len (missing) == (self.MISSING if self.MISSING is not None | |
421 | else self.FAILURES) | |
9d89c237 | 422 | assert len (mismatch) == self.MISMATCHES |
96fe6399 PG |
423 | |
424 | shutil.rmtree (self.dst_path) | |
425 | ||
426 | ||
0c6682ce PG |
427 | class RescueTest (DefectiveTest): |
428 | """ | |
429 | Rescue: restore corrupt backups from backup set that is damaged to a degree | |
430 | that the index file is worthless. | |
431 | """ | |
432 | ||
433 | def test_rescue_corrupt (self): | |
434 | """ | |
435 | Perform various damaging actions that cause unreadable objects, then | |
436 | attempt to extract objects regardless. | |
437 | """ | |
2fe5f6e7 PG |
438 | mode = self.COMPRESSION or "#" |
439 | bak_path, backup_file, backup_full, index_file = \ | |
440 | self.gen_file_names (self.COMPRESSION, self.PASSWORD) | |
0c6682ce PG |
441 | |
442 | if self.VOLUMES > 1: | |
047239f3 | 443 | self.gen_multivol (self.VOLUMES) |
0c6682ce | 444 | |
2fe5f6e7 | 445 | vname = partial (self.default_volume_name, backup_file) |
0c6682ce PG |
446 | dtar = deltatar.DeltaTar (mode=mode, |
447 | logger=None, | |
448 | password=self.PASSWORD, | |
449 | index_name_func=lambda _: index_file, | |
450 | volume_name_func=vname) | |
451 | ||
452 | dtar.create_full_backup \ | |
453 | (source_path=self.src_path, backup_path=bak_path, | |
454 | max_volume_size=1) | |
455 | ||
456 | if self.PASSWORD is not None: | |
457 | # ensure all files are at least superficially in PDT format | |
458 | for f in os.listdir (bak_path): | |
459 | assert is_pdt_encrypted ("%s/%s" % (bak_path, f)) | |
460 | ||
461 | # first restore must succeed | |
462 | dtar.restore_backup(target_path=self.dst_path, | |
463 | backup_indexes_paths=[ | |
464 | "%s/%s" % (bak_path, index_file) | |
465 | ]) | |
466 | for key, value in self.hash.items (): | |
467 | f = "%s/%s" % (self.dst_path, key) | |
468 | assert os.path.exists (f) | |
469 | assert value == self.md5sum (f) | |
470 | shutil.rmtree (self.dst_path) | |
471 | shutil.rmtree (self.src_path) | |
472 | ||
473 | self.CORRUPT (backup_full, | |
474 | self.COMPRESSION is not None, | |
475 | self.PASSWORD is not None) | |
476 | ||
477 | # normal restore must fail | |
478 | try: | |
479 | dtar.restore_backup(target_path=self.dst_path, | |
480 | backup_tar_path=backup_full) | |
481 | except tarfile.CompressionError: | |
482 | if self.PASSWORD is not None or self.COMPRESSION is not None: | |
483 | pass | |
484 | else: | |
485 | raise | |
486 | except tarfile.ReadError: | |
487 | # can happen with all three modes | |
488 | pass | |
489 | except tarfile.DecryptionError: | |
490 | if self.PASSWORD is not None: | |
491 | pass | |
492 | else: | |
493 | raise | |
494 | ||
495 | os.chdir (self.pwd) # not restored due to the error above | |
496 | # but recover will succeed | |
497 | failed = dtar.rescue_backup(target_path=self.dst_path, | |
2fe5f6e7 | 498 | backup_tar_path=backup_full) |
0c6682ce PG |
499 | # with one file missing |
500 | missing = [] | |
501 | mismatch = [] | |
502 | for key, value in self.hash.items (): | |
503 | kkey = "%s/%s" % (self.dst_path, key) | |
504 | if os.path.exists (kkey): | |
505 | if value != self.md5sum (kkey): | |
506 | mismatch.append (key) | |
507 | else: | |
508 | missing.append (key) | |
509 | ||
79bc14cf | 510 | assert len (failed) == self.FAILURES |
2fe5f6e7 PG |
511 | assert len (missing) == (self.MISSING if self.MISSING is not None |
512 | else self.FAILURES) | |
0c6682ce PG |
513 | assert len (mismatch) == self.MISMATCHES |
514 | ||
515 | shutil.rmtree (self.dst_path) | |
516 | ||
517 | ||
2fe5f6e7 PG |
518 | class GenIndexTest (DefectiveTest): |
519 | """ | |
520 | Deducing an index for a backup with tarfile. | |
521 | """ | |
522 | ||
523 | def test_gen_index (self): | |
524 | """ | |
525 | Create backup, leave it unharmed, then generate an index. | |
526 | """ | |
527 | mode = self.COMPRESSION or "#" | |
528 | bak_path, backup_file, backup_full, index_file = \ | |
529 | self.gen_file_names (self.COMPRESSION, self.PASSWORD) | |
530 | ||
047239f3 PG |
531 | if self.VOLUMES > 1: |
532 | self.gen_multivol (self.VOLUMES) | |
533 | ||
2fe5f6e7 PG |
534 | vname = partial (self.default_volume_name, backup_file) |
535 | dtar = deltatar.DeltaTar (mode=mode, | |
536 | logger=None, | |
537 | password=self.PASSWORD, | |
538 | index_name_func=lambda _: index_file, | |
539 | volume_name_func=vname) | |
540 | ||
541 | dtar.create_full_backup \ | |
542 | (source_path=self.src_path, backup_path=bak_path, | |
543 | max_volume_size=1) | |
544 | ||
27ee4dd4 PG |
545 | def gen_volume_name (nvol): |
546 | return os.path.join (bak_path, vname (backup_full, True, nvol)) | |
547 | ||
548 | psidx = tarfile.gen_rescue_index (gen_volume_name, | |
549 | mode, | |
550 | password=self.PASSWORD) | |
2fe5f6e7 | 551 | |
047239f3 PG |
552 | # correct for objects spanning volumes: these are treated as separate |
553 | # in the index! | |
554 | assert len (psidx) - self.VOLUMES + 1 == len (self.hash) | |
2fe5f6e7 PG |
555 | |
556 | ||
557 | ############################################################################### | |
558 | # rescue | |
559 | ############################################################################### | |
560 | ||
e25f31ac | 561 | class RecoverCorruptPayloadTestBase (RecoverTest): |
00b8c150 PG |
562 | COMPRESSION = None |
563 | PASSWORD = None | |
9d89c237 PG |
564 | FAILURES = 0 # tarfile will restore but corrupted, as |
565 | MISMATCHES = 1 # revealed by the hash | |
00b8c150 | 566 | |
e25f31ac PG |
567 | class RecoverCorruptPayloadSingleTest (RecoverCorruptPayloadTestBase): |
568 | VOLUMES = 1 | |
569 | ||
570 | class RecoverCorruptPayloadMultiTest (RecoverCorruptPayloadTestBase): | |
571 | VOLUMES = 3 | |
572 | ||
00b8c150 | 573 | |
e25f31ac | 574 | class RecoverCorruptPayloadGZTestBase (RecoverTest): |
00b8c150 PG |
575 | COMPRESSION = "#gz" |
576 | PASSWORD = None | |
577 | FAILURES = 1 | |
9d89c237 | 578 | MISMATCHES = 0 |
00b8c150 | 579 | |
e25f31ac PG |
580 | class RecoverCorruptPayloadGZSingleTest (RecoverCorruptPayloadGZTestBase): |
581 | VOLUMES = 1 | |
00b8c150 | 582 | |
e25f31ac PG |
583 | class RecoverCorruptPayloadGZMultiTest (RecoverCorruptPayloadGZTestBase): |
584 | VOLUMES = 3 | |
585 | ||
586 | ||
587 | class RecoverCorruptPayloadGZAESTestBase (RecoverTest): | |
00b8c150 PG |
588 | COMPRESSION = "#gz" |
589 | PASSWORD = TEST_PASSWORD | |
590 | FAILURES = 1 | |
9d89c237 | 591 | MISMATCHES = 0 |
00b8c150 | 592 | |
e25f31ac PG |
593 | class RecoverCorruptPayloadGZAESSingleTest (RecoverCorruptPayloadGZAESTestBase): |
594 | VOLUMES = 1 | |
595 | ||
596 | class RecoverCorruptPayloadGZAESMultiTest (RecoverCorruptPayloadGZAESTestBase): | |
597 | VOLUMES = 3 | |
00b8c150 | 598 | |
e25f31ac PG |
599 | |
600 | class RecoverCorruptHeaderTestBase (RecoverTest): | |
0349168a PG |
601 | COMPRESSION = None |
602 | PASSWORD = None | |
603 | FAILURES = 1 | |
604 | CORRUPT = corrupt_header | |
9d89c237 | 605 | MISMATCHES = 0 |
0349168a | 606 | |
e25f31ac PG |
607 | class RecoverCorruptHeaderSingleTest (RecoverCorruptHeaderTestBase): |
608 | VOLUMES = 1 | |
609 | ||
610 | class RecoverCorruptHeaderMultiTest (RecoverCorruptHeaderTestBase): | |
611 | VOLUMES = 3 | |
612 | ||
0349168a | 613 | |
e25f31ac | 614 | class RecoverCorruptHeaderGZTestBase (RecoverTest): |
96fe6399 PG |
615 | COMPRESSION = "#gz" |
616 | PASSWORD = None | |
617 | FAILURES = 1 | |
00b8c150 | 618 | CORRUPT = corrupt_header |
9d89c237 | 619 | MISMATCHES = 0 |
96fe6399 | 620 | |
e25f31ac PG |
621 | class RecoverCorruptHeaderGZSingleTest (RecoverCorruptHeaderGZTestBase): |
622 | VOLUMES = 1 | |
3267933a | 623 | |
e25f31ac PG |
624 | class RecoverCorruptHeaderGZMultiTest (RecoverCorruptHeaderGZTestBase): |
625 | VOLUMES = 3 | |
626 | ||
627 | ||
628 | class RecoverCorruptHeaderGZAESTestBase (RecoverTest): | |
96fe6399 PG |
629 | COMPRESSION = "#gz" |
630 | PASSWORD = TEST_PASSWORD | |
631 | FAILURES = 1 | |
00b8c150 | 632 | CORRUPT = corrupt_header |
9d89c237 | 633 | MISMATCHES = 0 |
fbdc9f4a | 634 | |
e25f31ac PG |
635 | class RecoverCorruptHeaderGZAESSingleTest (RecoverCorruptHeaderGZAESTestBase): |
636 | VOLUMES = 1 | |
637 | ||
638 | class RecoverCorruptHeaderGZAESMultiTest (RecoverCorruptHeaderGZAESTestBase): | |
639 | VOLUMES = 3 | |
da8996f0 | 640 | |
e25f31ac PG |
641 | |
642 | class RecoverCorruptEntireHeaderTestBase (RecoverTest): | |
da8996f0 PG |
643 | COMPRESSION = None |
644 | PASSWORD = None | |
645 | FAILURES = 1 | |
646 | CORRUPT = corrupt_entire_header | |
9d89c237 | 647 | MISMATCHES = 0 |
da8996f0 | 648 | |
e25f31ac PG |
649 | class RecoverCorruptEntireHeaderSingleTest (RecoverCorruptEntireHeaderTestBase): |
650 | VOLUMES = 1 | |
651 | ||
652 | class RecoverCorruptEntireHeaderMultiTest (RecoverCorruptEntireHeaderTestBase): | |
653 | VOLUMES = 3 | |
654 | ||
da8996f0 | 655 | |
e25f31ac | 656 | class RecoverCorruptEntireHeaderGZTestBase (RecoverTest): |
da8996f0 PG |
657 | COMPRESSION = "#gz" |
658 | PASSWORD = None | |
659 | FAILURES = 1 | |
660 | CORRUPT = corrupt_entire_header | |
9d89c237 | 661 | MISMATCHES = 0 |
da8996f0 | 662 | |
e25f31ac PG |
663 | class RecoverCorruptEntireHeaderGZSingleTest (RecoverCorruptEntireHeaderGZTestBase): |
664 | VOLUMES = 1 | |
da8996f0 | 665 | |
e25f31ac PG |
666 | class RecoverCorruptEntireHeaderGZMultiTest (RecoverCorruptEntireHeaderGZTestBase): |
667 | VOLUMES = 3 | |
668 | ||
669 | ||
670 | class RecoverCorruptEntireHeaderGZAESTestBase (RecoverTest): | |
da8996f0 PG |
671 | COMPRESSION = "#gz" |
672 | PASSWORD = TEST_PASSWORD | |
673 | FAILURES = 1 | |
674 | CORRUPT = corrupt_entire_header | |
9d89c237 | 675 | MISMATCHES = 0 |
da8996f0 | 676 | |
e25f31ac PG |
677 | class RecoverCorruptEntireHeaderGZAESSingleTest (RecoverCorruptEntireHeaderGZAESTestBase): |
678 | VOLUMES = 1 | |
679 | ||
680 | class RecoverCorruptEntireHeaderGZAESMultiTest (RecoverCorruptEntireHeaderGZAESTestBase): | |
681 | VOLUMES = 3 | |
517d35b7 | 682 | |
e25f31ac PG |
683 | |
684 | class RecoverCorruptTrailingDataTestBase (RecoverTest): | |
517d35b7 PG |
685 | # plain Tar is indifferent against traling data and the results |
686 | # are consistent | |
687 | COMPRESSION = None | |
688 | PASSWORD = None | |
689 | FAILURES = 0 | |
690 | CORRUPT = corrupt_trailing_data | |
691 | MISMATCHES = 0 | |
692 | ||
e25f31ac PG |
693 | class RecoverCorruptTrailingDataSingleTest (RecoverCorruptTrailingDataTestBase): |
694 | VOLUMES = 1 | |
695 | ||
696 | class RecoverCorruptTrailingDataMultiTest (RecoverCorruptTrailingDataTestBase): | |
14895f4b PG |
697 | # the last object in first archive has extra bytes somewhere in the |
698 | # middle because tar itself performs no data checksumming. | |
699 | MISMATCHES = 1 | |
e25f31ac PG |
700 | VOLUMES = 3 |
701 | ||
517d35b7 | 702 | |
e25f31ac | 703 | class RecoverCorruptTrailingDataGZTestBase (RecoverTest): |
517d35b7 PG |
704 | # reading past the final object will cause decompression failure; |
705 | # all objects except for the last survive unharmed though | |
706 | COMPRESSION = "#gz" | |
707 | PASSWORD = None | |
708 | FAILURES = 1 | |
709 | CORRUPT = corrupt_trailing_data | |
710 | MISMATCHES = 0 | |
711 | ||
e25f31ac PG |
712 | class RecoverCorruptTrailingDataGZSingleTest (RecoverCorruptTrailingDataGZTestBase): |
713 | VOLUMES = 1 | |
517d35b7 | 714 | |
e25f31ac PG |
715 | class RecoverCorruptTrailingDataGZMultiTest (RecoverCorruptTrailingDataGZTestBase): |
716 | VOLUMES = 3 | |
14895f4b PG |
717 | # the last file of the first volume will only contain the data of the |
718 | # second part which is contained in the second volume. this happens | |
719 | # because the CRC32 is wrong for the first part so it gets discarded, then | |
720 | # the object is recreated from the first header of the second volume, | |
721 | # containing only the remainder of the data. | |
722 | MISMATCHES = 1 | |
4d4925de | 723 | MISSING = 0 |
e25f31ac PG |
724 | |
725 | ||
726 | class RecoverCorruptTrailingDataGZAESTestBase (RecoverTest): | |
517d35b7 PG |
727 | COMPRESSION = "#gz" |
728 | PASSWORD = TEST_PASSWORD | |
729 | FAILURES = 0 | |
730 | CORRUPT = corrupt_trailing_data | |
731 | MISMATCHES = 0 | |
732 | ||
e25f31ac PG |
733 | class RecoverCorruptTrailingDataGZAESSingleTest (RecoverCorruptTrailingDataGZAESTestBase): |
734 | VOLUMES = 1 | |
735 | ||
736 | class RecoverCorruptTrailingDataGZAESMultiTest (RecoverCorruptTrailingDataGZAESTestBase): | |
737 | VOLUMES = 3 | |
517d35b7 | 738 | |
20e1d773 PG |
739 | |
740 | class RecoverCorruptVolumeBaseTest (RecoverTest): | |
741 | COMPRESSION = None | |
742 | PASSWORD = None | |
743 | FAILURES = 8 | |
744 | CORRUPT = corrupt_volume | |
745 | VOLUMES = 3 | |
746 | ||
747 | class RecoverCorruptVolumeTest (RecoverCorruptVolumeBaseTest): | |
748 | pass | |
749 | ||
3692fd82 PG |
750 | class RecoverCorruptVolumeGZTest (RecoverCorruptVolumeBaseTest): |
751 | COMPRESSION = "#gz" | |
752 | ||
753 | class RecoverCorruptVolumeGZAESTest (RecoverCorruptVolumeBaseTest): | |
20e1d773 | 754 | COMPRESSION = "#gz" |
3692fd82 PG |
755 | PASSWORD = TEST_PASSWORD |
756 | ||
757 | ||
b9cf4a0f | 758 | @unittest.skipIf(sys.version_info < (3, 4), "requires recent os library") |
3692fd82 PG |
759 | class RecoverCorruptHoleBaseTest (RecoverTest): |
760 | """ | |
761 | Cut bytes from the middle of a volume. | |
762 | ||
763 | Index-based recovery works only up to the hole. | |
764 | """ | |
765 | COMPRESSION = None | |
20e1d773 | 766 | PASSWORD = None |
3692fd82 PG |
767 | FAILURES = 3 |
768 | CORRUPT = corrupt_hole | |
769 | VOLUMES = 2 # request two vols to swell up the first one | |
770 | MISMATCHES = 1 | |
771 | ||
b9cf4a0f | 772 | @unittest.skipIf(sys.version_info < (3, 4), "requires recent os library") |
3692fd82 PG |
773 | class RecoverCorruptHoleTest (RecoverCorruptHoleBaseTest): |
774 | FAILURES = 2 | |
775 | ||
b9cf4a0f | 776 | @unittest.skipIf(sys.version_info < (3, 4), "requires recent os library") |
3692fd82 PG |
777 | class RecoverCorruptHoleGZTest (RecoverCorruptHoleBaseTest): |
778 | COMPRESSION = "#gz" | |
779 | MISSING = 2 | |
20e1d773 | 780 | |
b9cf4a0f | 781 | @unittest.skipIf(sys.version_info < (3, 4), "requires recent os library") |
3692fd82 | 782 | class RecoverCorruptHoleGZAESTest (RecoverCorruptHoleBaseTest): |
20e1d773 PG |
783 | COMPRESSION = "#gz" |
784 | PASSWORD = TEST_PASSWORD | |
3692fd82 | 785 | MISSING = 2 |
20e1d773 | 786 | |
2fe5f6e7 PG |
787 | ############################################################################### |
788 | # rescue | |
789 | ############################################################################### | |
790 | ||
b9cf4a0f | 791 | @unittest.skipIf(sys.version_info < (3, 4), "requires recent os library") |
2fe5f6e7 PG |
792 | class RescueCorruptHoleBaseTest (RescueTest): |
793 | """ | |
794 | Cut bytes from the middle of a volume. | |
795 | """ | |
796 | COMPRESSION = None | |
797 | PASSWORD = None | |
79bc14cf | 798 | FAILURES = 0 |
2fe5f6e7 PG |
799 | CORRUPT = corrupt_hole |
800 | VOLUMES = 2 # request two vols to swell up the first one | |
79bc14cf PG |
801 | MISMATCHES = 2 # intersected by hole |
802 | MISSING = 1 # excised by hole | |
2fe5f6e7 | 803 | |
b9cf4a0f | 804 | @unittest.skipIf(sys.version_info < (3, 4), "requires recent os library") |
2fe5f6e7 | 805 | class RescueCorruptHoleTest (RescueCorruptHoleBaseTest): |
79bc14cf | 806 | pass |
2fe5f6e7 | 807 | |
b9cf4a0f | 808 | @unittest.skipIf(sys.version_info < (3, 4), "requires recent os library") |
2fe5f6e7 PG |
809 | class RescueCorruptHoleGZTest (RescueCorruptHoleBaseTest): |
810 | COMPRESSION = "#gz" | |
79bc14cf PG |
811 | # the decompressor explodes in our face processing the first dummy, nothing |
812 | # we can do to recover | |
813 | FAILURES = 1 | |
2fe5f6e7 | 814 | |
b9cf4a0f | 815 | @unittest.skipIf(sys.version_info < (3, 4), "requires recent os library") |
2fe5f6e7 PG |
816 | class RescueCorruptHoleGZAESTest (RescueCorruptHoleBaseTest): |
817 | COMPRESSION = "#gz" | |
818 | PASSWORD = TEST_PASSWORD | |
79bc14cf PG |
819 | # again, ignoring the crypto errors yields a bad zlib stream causing the |
820 | # decompressor to abort where the hole begins; the file is extracted up | |
821 | # to this point though | |
822 | FAILURES = 1 | |
2fe5f6e7 | 823 | |
0c8baf2b | 824 | |
afb2d647 | 825 | class RescueCorruptHeaderCTSizeGZAESTest (RescueTest): |
0c8baf2b PG |
826 | COMPRESSION = "#gz" |
827 | PASSWORD = TEST_PASSWORD | |
828 | FAILURES = 0 | |
829 | CORRUPT = corrupt_ctsize | |
830 | MISMATCHES = 0 | |
831 | ||
832 | ||
b9cf4a0f | 833 | @unittest.skipIf(sys.version_info < (3, 4), "requires recent os library") |
afb2d647 PG |
834 | class RescueCorruptLeadingGarbageTestBase (RescueTest): |
835 | # plain Tar is indifferent against traling data and the results | |
836 | # are consistent | |
837 | COMPRESSION = None | |
838 | PASSWORD = None | |
839 | FAILURES = 0 | |
840 | CORRUPT = corrupt_leading_garbage | |
841 | MISMATCHES = 0 | |
842 | ||
b9cf4a0f | 843 | @unittest.skipIf(sys.version_info < (3, 4), "requires recent os library") |
afb2d647 PG |
844 | class RescueCorruptLeadingGarbageSingleTest (RescueCorruptLeadingGarbageTestBase): |
845 | VOLUMES = 1 | |
846 | ||
b9cf4a0f | 847 | @unittest.skipIf(sys.version_info < (3, 4), "requires recent os library") |
afb2d647 PG |
848 | class RescueCorruptLeadingGarbageMultiTest (RescueCorruptLeadingGarbageTestBase): |
849 | # the last object in first archive has extra bytes somewhere in the | |
850 | # middle because tar itself performs no data checksumming. | |
851 | MISMATCHES = 2 | |
852 | VOLUMES = 3 | |
853 | ||
854 | ||
2fe5f6e7 PG |
855 | ############################################################################### |
856 | # index | |
857 | ############################################################################### | |
858 | ||
859 | class GenIndexIntactBaseTest (GenIndexTest): | |
860 | """ | |
861 | """ | |
862 | COMPRESSION = None | |
863 | PASSWORD = None | |
864 | FAILURES = 0 | |
865 | CORRUPT = immaculate | |
866 | VOLUMES = 1 | |
867 | MISMATCHES = 1 | |
868 | ||
047239f3 PG |
869 | class GenIndexIntactSingleTest (GenIndexIntactBaseTest): |
870 | pass | |
871 | ||
872 | class GenIndexIntactSingleGZTest (GenIndexIntactBaseTest): | |
873 | COMPRESSION = "#gz" | |
874 | MISSING = 2 | |
875 | ||
876 | class GenIndexIntactSingleGZAESTest (GenIndexIntactBaseTest): | |
877 | COMPRESSION = "#gz" | |
878 | PASSWORD = TEST_PASSWORD | |
879 | MISSING = 2 | |
2fe5f6e7 | 880 | |
047239f3 PG |
881 | class GenIndexIntactMultiTest (GenIndexIntactBaseTest): |
882 | VOLUMES = 3 | |
2fe5f6e7 PG |
883 | pass |
884 | ||
047239f3 PG |
885 | class GenIndexIntactMultiGZTest (GenIndexIntactBaseTest): |
886 | VOLUMES = 3 | |
2fe5f6e7 PG |
887 | COMPRESSION = "#gz" |
888 | MISSING = 2 | |
889 | ||
047239f3 PG |
890 | class GenIndexIntactMultiGZAESTest (GenIndexIntactBaseTest): |
891 | VOLUMES = 3 | |
2fe5f6e7 PG |
892 | COMPRESSION = "#gz" |
893 | PASSWORD = TEST_PASSWORD | |
894 | MISSING = 2 | |
895 | ||
6e1f5355 PG |
896 | |
897 | class GenIndexCorruptHoleBaseTest (GenIndexTest): | |
898 | """ | |
899 | Recreate index from file with hole. | |
900 | """ | |
901 | COMPRESSION = None | |
902 | PASSWORD = None | |
903 | FAILURES = 0 | |
904 | CORRUPT = corrupt_hole | |
905 | VOLUMES = 1 | |
906 | MISMATCHES = 1 | |
907 | ||
908 | class GenIndexCorruptHoleTest (GenIndexCorruptHoleBaseTest): | |
909 | pass | |
910 | ||
911 | class GenIndexCorruptHoleGZTest (GenIndexCorruptHoleBaseTest): | |
912 | COMPRESSION = "#gz" | |
913 | MISSING = 2 | |
914 | ||
915 | class GenIndexCorruptHoleGZAESTest (GenIndexCorruptHoleBaseTest): | |
916 | COMPRESSION = "#gz" | |
917 | PASSWORD = TEST_PASSWORD | |
918 | MISSING = 2 | |
919 | ||
920 | ||
921 | ||
922 | class GenIndexCorruptEntireHeaderBaseTest (GenIndexTest): | |
923 | """ | |
924 | Recreate index from file with hole. | |
925 | """ | |
926 | COMPRESSION = None | |
927 | PASSWORD = None | |
928 | FAILURES = 0 | |
929 | CORRUPT = corrupt_entire_header | |
930 | VOLUMES = 1 | |
931 | MISMATCHES = 1 | |
932 | ||
933 | class GenIndexCorruptEntireHeaderTest (GenIndexCorruptEntireHeaderBaseTest): | |
934 | pass | |
935 | ||
936 | class GenIndexCorruptEntireHeaderGZTest (GenIndexCorruptEntireHeaderBaseTest): | |
937 | COMPRESSION = "#gz" | |
938 | MISSING = 2 | |
939 | ||
940 | class GenIndexCorruptEntireHeaderGZAESTest (GenIndexCorruptEntireHeaderBaseTest): | |
941 | COMPRESSION = "#gz" | |
942 | PASSWORD = TEST_PASSWORD | |
943 | MISSING = 2 | |
944 |