Commit | Line | Data |
---|---|---|
fbdc9f4a PG |
1 | import logging |
2 | import os | |
3 | import shutil | |
3692fd82 | 4 | import stat |
fbdc9f4a | 5 | |
2fe5f6e7 PG |
6 | from functools import partial |
7 | ||
fbdc9f4a | 8 | import deltatar.deltatar as deltatar |
3267933a | 9 | import deltatar.crypto as crypto |
203cb25e | 10 | import deltatar.tarfile as tarfile |
fbdc9f4a PG |
11 | |
12 | from . import BaseTest | |
13 | ||
e25f31ac | 14 | TEST_PASSWORD = "test1234" |
85e7013f | 15 | TEST_VOLSIZ = 2 # MB |
e25f31ac | 16 | TEST_FILESPERVOL = 3 |
85e7013f PG |
17 | VOLUME_OVERHEAD = 1.4 # account for tar overhead when fitting files into |
18 | # volumes; this is black magic | |
20e1d773 | 19 | TEST_BLOCKSIZE = 4096 |
96fe6399 PG |
20 | |
21 | ############################################################################### | |
22 | ## helpers ## | |
23 | ############################################################################### | |
24 | ||
3267933a PG |
25 | def flip_bits (fname, off, b=0x01, n=1): |
26 | """ | |
27 | Open file *fname* at offset *off*, replacing the next *n* bytes with | |
28 | their values xor’ed with *b*. | |
29 | """ | |
30 | fd = os.open (fname, os.O_RDWR) | |
203cb25e | 31 | |
3267933a PG |
32 | try: |
33 | pos = os.lseek (fd, off, os.SEEK_SET) | |
34 | assert pos == off | |
35 | chunk = os.read (fd, n) | |
36 | chunk = bytes (map (lambda v: v ^ b, chunk)) | |
da8996f0 PG |
37 | pos = os.lseek (fd, off, os.SEEK_SET) |
38 | assert pos == off | |
3267933a PG |
39 | os.write (fd, chunk) |
40 | finally: | |
41 | os.close (fd) | |
42 | ||
203cb25e PG |
43 | |
44 | def gz_header_size (fname, off=0): | |
45 | """ | |
46 | Determine the length of the gzip header starting at *off* in file fname. | |
47 | ||
48 | The header is variable length because it may contain the filename as NUL | |
49 | terminated bytes. | |
50 | """ | |
51 | # length so we need to determine where the actual payload starts | |
52 | off = tarfile.GZ_HEADER_SIZE | |
53 | fd = os.open (fname, os.O_RDONLY) | |
54 | ||
55 | try: | |
56 | pos = os.lseek (fd, off, os.SEEK_SET) | |
57 | assert pos == off | |
58 | while os.read (fd, 1)[0] != 0: | |
59 | off += 1 | |
60 | pos = os.lseek (fd, off, os.SEEK_SET) | |
61 | assert pos == off | |
62 | finally: | |
63 | os.close (fd) | |
64 | ||
65 | return off | |
66 | ||
da8996f0 | 67 | |
96fe6399 PG |
68 | def is_pdt_encrypted (fname): |
69 | """ | |
70 | Returns true if the file contains at least one PDT header plus enough | |
71 | space for the object. | |
72 | """ | |
73 | try: | |
74 | with open (fname, "rb") as st: | |
75 | hdr = crypto.hdr_read_stream (st) | |
76 | siz = hdr ["ctsize"] | |
77 | assert (len (st.read (siz)) == siz) | |
78 | except Exception as exn: | |
79 | return False | |
80 | return True | |
81 | ||
82 | ||
3692fd82 PG |
83 | ############################################################################### |
84 | ## corruption simulators ## | |
85 | ############################################################################### | |
86 | ||
0c8baf2b PG |
87 | class UndefinedTest (Exception): |
88 | """No test available for the asked combination of parameters.""" | |
89 | ||
00b8c150 PG |
90 | def corrupt_header (_, fname, compress, encrypt): |
91 | """ | |
92 | Modify a significant byte in the object header of the format. | |
93 | """ | |
94 | if encrypt is True: # damage GCM tag | |
95 | flip_bits (fname, crypto.HDR_OFF_TAG + 1) | |
96 | elif compress is True: # invalidate magic | |
97 | flip_bits (fname, 1) | |
98 | else: # Fudge checksum. From tar(5): | |
99 | # | |
100 | # struct header_gnu_tar { | |
101 | # char name[100]; | |
102 | # char mode[8]; | |
103 | # char uid[8]; | |
104 | # char gid[8]; | |
105 | # char size[12]; | |
106 | # char mtime[12]; | |
107 | # char checksum[8]; | |
108 | # … | |
109 | flip_bits (fname, 100 + 8 + 8 + 8 + 12 + 12 + 1) | |
110 | ||
111 | ||
0c8baf2b PG |
112 | def corrupt_ctsize (_, fname, compress, encrypt): |
113 | """ | |
114 | Blow up the size of an object so as to cause its apparent payload to leak | |
115 | into the next one. | |
116 | """ | |
117 | if encrypt is True: | |
118 | # damage lowest bit of second least significant byte of size field; | |
119 | # this effectively sets the ciphertext size to 422, causing it to | |
120 | # extend over the next object into the third one. | |
121 | return flip_bits (fname, crypto.HDR_OFF_CTSIZE + 1, b=0x01) | |
122 | raise UndefinedTest ("corrupt_ctsize %s %s %s" % (fname, compress, encrypt)) | |
123 | ||
124 | ||
da8996f0 PG |
125 | def corrupt_entire_header (_, fname, compress, encrypt): |
126 | """ | |
127 | Flip all bits in the first object header. | |
128 | """ | |
129 | if encrypt is True: | |
130 | flip_bits (fname, 0, 0xff, crypto.PDTCRYPT_HDR_SIZE) | |
131 | elif compress is True: # invalidate magic | |
132 | flip_bits (fname, 0, 0xff, gz_header_size (fname)) | |
133 | else: | |
134 | flip_bits (fname, 0, 0xff, tarfile.BLOCKSIZE) | |
135 | ||
136 | ||
00b8c150 PG |
137 | def corrupt_payload_start (_, fname, compress, encrypt): |
138 | """ | |
139 | Modify the byte following the object header structure of the format. | |
140 | """ | |
141 | if encrypt is True: | |
142 | flip_bits (fname, crypto.PDTCRYPT_HDR_SIZE + 1) | |
143 | elif compress is True: | |
144 | flip_bits (fname, gz_header_size (fname) + 1) | |
145 | else: | |
146 | flip_bits (fname, tarfile.BLOCKSIZE + 1) | |
147 | ||
148 | ||
afb2d647 PG |
149 | def corrupt_leading_garbage (_, fname, compress, encrypt): |
150 | """ | |
151 | Prepend junk to file. | |
152 | """ | |
153 | aname = os.path.abspath (fname) | |
154 | infd = os.open (fname, os.O_RDONLY) | |
155 | size = os.lseek (infd, 0, os.SEEK_END) | |
156 | assert os.lseek (infd, 0, os.SEEK_SET) == 0 | |
157 | outfd = os.open (os.path.dirname (aname), os.O_WRONLY | os.O_TMPFILE, | |
158 | stat.S_IRUSR | stat.S_IWUSR) | |
159 | junk = os.urandom (512) # tar block sized | |
160 | ||
161 | # write new file with garbage prepended | |
162 | done = 0 | |
163 | os.write (outfd, junk) # junk first | |
164 | done += len (junk) | |
165 | while done < size: | |
166 | data = os.read (infd, TEST_BLOCKSIZE) | |
167 | os.write (outfd, data) | |
168 | done += len (data) | |
169 | ||
170 | assert os.lseek (outfd, 0, os.SEEK_CUR) == done | |
171 | ||
172 | # close and free old file | |
173 | os.close (infd) | |
174 | os.unlink (fname) | |
175 | ||
176 | # install the new file in its place, atomically | |
177 | path = "/proc/self/fd/%d" % outfd | |
178 | os.link (path, aname, src_dir_fd=0, follow_symlinks=True) | |
179 | os.close (outfd) | |
180 | ||
181 | ||
517d35b7 PG |
182 | def corrupt_trailing_data (_, fname, compress, encrypt): |
183 | """ | |
184 | Modify the byte following the object header structure of the format. | |
185 | """ | |
186 | junk = os.urandom (42) | |
187 | fd = os.open (fname, os.O_WRONLY | os.O_APPEND) | |
188 | os.write (fd, junk) | |
189 | os.close (fd) | |
190 | ||
00b8c150 | 191 | |
20e1d773 PG |
192 | def corrupt_volume (_, fname, compress, encrypt): |
193 | """ | |
194 | Zero out an entire volume. | |
195 | """ | |
196 | fd = os.open (fname, os.O_WRONLY) | |
197 | size = os.lseek (fd, 0, os.SEEK_END) | |
198 | assert os.lseek (fd, 0, os.SEEK_SET) == 0 | |
199 | zeros = bytes (b'\x00' * TEST_BLOCKSIZE) | |
200 | while size > 0: | |
201 | todo = min (size, TEST_BLOCKSIZE) | |
202 | os.write (fd, zeros [:todo]) | |
203 | size -= todo | |
204 | os.close (fd) | |
205 | ||
206 | ||
3692fd82 PG |
207 | def corrupt_hole (_, fname, compress, encrypt): |
208 | """ | |
209 | Cut file in three pieces, reassemble without the middle one. | |
210 | """ | |
211 | aname = os.path.abspath (fname) | |
212 | infd = os.open (fname, os.O_RDONLY) | |
213 | size = os.lseek (infd, 0, os.SEEK_END) | |
214 | assert os.lseek (infd, 0, os.SEEK_SET) == 0 | |
215 | assert size > 3 * TEST_BLOCKSIZE | |
216 | hole = (size / 3, size * 2 / 3) | |
217 | outfd = os.open (os.path.dirname (aname), os.O_WRONLY | os.O_TMPFILE, | |
218 | stat.S_IRUSR | stat.S_IWUSR) | |
219 | ||
3692fd82 PG |
220 | done = 0 |
221 | while done < size: | |
222 | data = os.read (infd, TEST_BLOCKSIZE) | |
223 | if done < hole [0] or hole [1] < done: | |
224 | # only copy from outside hole | |
225 | os.write (outfd, data) | |
226 | done += len (data) | |
227 | ||
228 | os.close (infd) | |
229 | os.unlink (fname) | |
230 | ||
231 | path = "/proc/self/fd/%d" % outfd | |
232 | os.link (path, aname, src_dir_fd=0, follow_symlinks=True) | |
233 | os.close (outfd) | |
234 | ||
2fe5f6e7 PG |
235 | def immaculate (_, _fname, _compress, _encrypt): |
236 | """ | |
237 | No-op dummy. | |
238 | """ | |
239 | pass | |
3692fd82 | 240 | |
96fe6399 PG |
241 | ############################################################################### |
242 | ## tests ## | |
243 | ############################################################################### | |
203cb25e | 244 | |
0c6682ce | 245 | class DefectiveTest (BaseTest): |
fbdc9f4a PG |
246 | """ |
247 | Disaster recovery: restore corrupt backups. | |
248 | """ | |
249 | ||
96fe6399 PG |
250 | COMPRESSION = None |
251 | PASSWORD = None | |
9d89c237 PG |
252 | FAILURES = 0 # files that could not be restored |
253 | MISMATCHES = 0 # files that were restored but corrupted | |
00b8c150 | 254 | CORRUPT = corrupt_payload_start |
e25f31ac | 255 | VOLUMES = 1 |
4d4925de | 256 | MISSING = None # normally the number of failures |
96fe6399 | 257 | |
fbdc9f4a PG |
258 | |
259 | def setUp(self): | |
260 | ''' | |
261 | Create base test data | |
262 | ''' | |
96fe6399 PG |
263 | self.pwd = os.getcwd() |
264 | self.dst_path = "source_dir" | |
265 | self.src_path = "%s2" % self.dst_path | |
266 | self.hash = dict() | |
267 | ||
fbdc9f4a | 268 | os.system('rm -rf target_dir source_dir* backup_dir* huge') |
96fe6399 | 269 | os.makedirs (self.src_path) |
fbdc9f4a | 270 | |
96fe6399 | 271 | for i in range (5): |
85e7013f | 272 | f = "dummy_%d" % i |
96fe6399 PG |
273 | self.hash [f] = self.create_file ("%s/%s" |
274 | % (self.src_path, f), 5 + i) | |
fbdc9f4a | 275 | |
96fe6399 PG |
276 | |
277 | def tearDown(self): | |
278 | ''' | |
279 | Remove temporal files created by unit tests and reset globals. | |
280 | ''' | |
281 | os.chdir(self.pwd) | |
282 | os.system("rm -rf source_dir source_dir2 backup_dir*") | |
fbdc9f4a PG |
283 | |
284 | ||
2fe5f6e7 PG |
285 | @staticmethod |
286 | def default_volume_name (backup_file, _x, _y, n, *a, **kwa): | |
287 | return backup_file % n | |
0c6682ce | 288 | |
2fe5f6e7 | 289 | def gen_file_names (self, comp, pw): |
203cb25e | 290 | bak_path = "backup_dir" |
e25f31ac PG |
291 | backup_file = "the_full_backup_%0.2d.tar" |
292 | backup_full = ("%s/%s" % (bak_path, backup_file)) % 0 | |
96fe6399 PG |
293 | index_file = "the_full_index" |
294 | ||
295 | if self.COMPRESSION is not None: | |
296 | backup_file += ".gz" | |
297 | backup_full += ".gz" | |
298 | index_file += ".gz" | |
299 | ||
300 | if self.PASSWORD is not None: | |
e25f31ac PG |
301 | backup_file = "%s.%s" % (backup_file, deltatar.PDTCRYPT_EXTENSION) |
302 | backup_full = "%s.%s" % (backup_full, deltatar.PDTCRYPT_EXTENSION) | |
303 | index_file = "%s.%s" % (index_file , deltatar.PDTCRYPT_EXTENSION) | |
304 | ||
2fe5f6e7 PG |
305 | return bak_path, backup_file, backup_full, index_file |
306 | ||
307 | ||
308 | class RecoverTest (DefectiveTest): | |
309 | """ | |
310 | Recover: restore corrupt backups from index file information. | |
311 | """ | |
312 | ||
313 | def test_recover_corrupt (self): | |
314 | """ | |
315 | Perform various damaging actions that cause unreadable objects. | |
316 | ||
317 | Expects the extraction to fail in normal mode. With disaster recovery, | |
318 | extraction must succeed, and exactly one file must be missing. | |
319 | """ | |
320 | mode = self.COMPRESSION or "#" | |
321 | bak_path, backup_file, backup_full, index_file = \ | |
322 | self.gen_file_names (self.COMPRESSION, self.PASSWORD) | |
323 | ||
e25f31ac | 324 | if self.VOLUMES > 1: |
85e7013f PG |
325 | # add n files for one nth the volume size each, corrected |
326 | # for metadata and tar block overhead | |
327 | fsiz = int ( ( TEST_VOLSIZ | |
328 | / (TEST_FILESPERVOL * VOLUME_OVERHEAD)) | |
329 | * 1024 * 1024) | |
330 | fcnt = (self.VOLUMES - 1) * TEST_FILESPERVOL | |
e25f31ac PG |
331 | for i in range (fcnt): |
332 | nvol, invol = divmod(i, TEST_FILESPERVOL) | |
333 | f = "dummy_vol_%d_n_%0.2d" % (nvol, invol) | |
334 | self.hash [f] = self.create_file ("%s/%s" | |
335 | % (self.src_path, f), | |
85e7013f PG |
336 | fsiz, |
337 | random=True) | |
e25f31ac | 338 | |
2fe5f6e7 | 339 | vname = partial (self.default_volume_name, backup_file) |
96fe6399 PG |
340 | dtar = deltatar.DeltaTar (mode=mode, |
341 | logger=None, | |
342 | password=self.PASSWORD, | |
203cb25e | 343 | index_name_func=lambda _: index_file, |
3267933a | 344 | volume_name_func=vname) |
fbdc9f4a PG |
345 | |
346 | dtar.create_full_backup \ | |
e25f31ac PG |
347 | (source_path=self.src_path, backup_path=bak_path, |
348 | max_volume_size=1) | |
96fe6399 PG |
349 | |
350 | if self.PASSWORD is not None: | |
351 | # ensure all files are at least superficially in PDT format | |
352 | for f in os.listdir (bak_path): | |
353 | assert is_pdt_encrypted ("%s/%s" % (bak_path, f)) | |
203cb25e PG |
354 | |
355 | # first restore must succeed | |
96fe6399 | 356 | dtar.restore_backup(target_path=self.dst_path, |
f090d35a PG |
357 | backup_indexes_paths=[ |
358 | "%s/%s" % (bak_path, index_file) | |
359 | ]) | |
203cb25e | 360 | for key, value in self.hash.items (): |
96fe6399 | 361 | f = "%s/%s" % (self.dst_path, key) |
b15e549b PG |
362 | assert os.path.exists (f) |
363 | assert value == self.md5sum (f) | |
96fe6399 PG |
364 | shutil.rmtree (self.dst_path) |
365 | shutil.rmtree (self.src_path) | |
203cb25e | 366 | |
00b8c150 PG |
367 | self.CORRUPT (backup_full, |
368 | self.COMPRESSION is not None, | |
369 | self.PASSWORD is not None) | |
203cb25e PG |
370 | |
371 | # normal restore must fail | |
96fe6399 PG |
372 | try: |
373 | dtar.restore_backup(target_path=self.dst_path, | |
203cb25e | 374 | backup_tar_path=backup_full) |
96fe6399 PG |
375 | except tarfile.CompressionError: |
376 | if self.PASSWORD is not None or self.COMPRESSION is not None: | |
377 | pass | |
00b8c150 PG |
378 | else: |
379 | raise | |
96fe6399 | 380 | except tarfile.ReadError: |
00b8c150 PG |
381 | # can happen with all three modes |
382 | pass | |
383 | except tarfile.DecryptionError: | |
384 | if self.PASSWORD is not None: | |
96fe6399 | 385 | pass |
00b8c150 PG |
386 | else: |
387 | raise | |
96fe6399 PG |
388 | |
389 | os.chdir (self.pwd) # not restored due to the error above | |
203cb25e | 390 | # but recover will succeed |
96fe6399 | 391 | failed = dtar.recover_backup(target_path=self.dst_path, |
b15e549b PG |
392 | backup_indexes_paths=[ |
393 | "%s/%s" % (bak_path, index_file) | |
394 | ]) | |
96fe6399 PG |
395 | |
396 | assert len (failed) == self.FAILURES | |
203cb25e PG |
397 | |
398 | # with one file missing | |
9d89c237 PG |
399 | missing = [] |
400 | mismatch = [] | |
203cb25e | 401 | for key, value in self.hash.items (): |
96fe6399 | 402 | kkey = "%s/%s" % (self.dst_path, key) |
b15e549b | 403 | if os.path.exists (kkey): |
9d89c237 PG |
404 | if value != self.md5sum (kkey): |
405 | mismatch.append (key) | |
203cb25e | 406 | else: |
757319dd | 407 | missing.append (key) |
4d4925de PG |
408 | |
409 | # usually, an object whose extraction fails will not be found on | |
410 | # disk afterwards so the number of failures equals that of missing | |
411 | # files. however, some modes will create partial files for objects | |
412 | # spanning multiple volumes that contain the parts whose checksums | |
413 | # were valid. | |
414 | assert len (missing) == (self.MISSING if self.MISSING is not None | |
415 | else self.FAILURES) | |
9d89c237 | 416 | assert len (mismatch) == self.MISMATCHES |
96fe6399 PG |
417 | |
418 | shutil.rmtree (self.dst_path) | |
419 | ||
420 | ||
0c6682ce PG |
421 | class RescueTest (DefectiveTest): |
422 | """ | |
423 | Rescue: restore corrupt backups from backup set that is damaged to a degree | |
424 | that the index file is worthless. | |
425 | """ | |
426 | ||
427 | def test_rescue_corrupt (self): | |
428 | """ | |
429 | Perform various damaging actions that cause unreadable objects, then | |
430 | attempt to extract objects regardless. | |
431 | """ | |
2fe5f6e7 PG |
432 | mode = self.COMPRESSION or "#" |
433 | bak_path, backup_file, backup_full, index_file = \ | |
434 | self.gen_file_names (self.COMPRESSION, self.PASSWORD) | |
0c6682ce PG |
435 | |
436 | if self.VOLUMES > 1: | |
437 | # add n files for one nth the volume size each, corrected | |
438 | # for metadata and tar block overhead | |
439 | fsiz = int ( ( TEST_VOLSIZ | |
440 | / (TEST_FILESPERVOL * VOLUME_OVERHEAD)) | |
441 | * 1024 * 1024) | |
442 | fcnt = (self.VOLUMES - 1) * TEST_FILESPERVOL | |
443 | for i in range (fcnt): | |
444 | nvol, invol = divmod(i, TEST_FILESPERVOL) | |
445 | f = "dummy_vol_%d_n_%0.2d" % (nvol, invol) | |
446 | self.hash [f] = self.create_file ("%s/%s" | |
447 | % (self.src_path, f), | |
448 | fsiz, | |
449 | random=True) | |
450 | ||
2fe5f6e7 | 451 | vname = partial (self.default_volume_name, backup_file) |
0c6682ce PG |
452 | dtar = deltatar.DeltaTar (mode=mode, |
453 | logger=None, | |
454 | password=self.PASSWORD, | |
455 | index_name_func=lambda _: index_file, | |
456 | volume_name_func=vname) | |
457 | ||
458 | dtar.create_full_backup \ | |
459 | (source_path=self.src_path, backup_path=bak_path, | |
460 | max_volume_size=1) | |
461 | ||
462 | if self.PASSWORD is not None: | |
463 | # ensure all files are at least superficially in PDT format | |
464 | for f in os.listdir (bak_path): | |
465 | assert is_pdt_encrypted ("%s/%s" % (bak_path, f)) | |
466 | ||
467 | # first restore must succeed | |
468 | dtar.restore_backup(target_path=self.dst_path, | |
469 | backup_indexes_paths=[ | |
470 | "%s/%s" % (bak_path, index_file) | |
471 | ]) | |
472 | for key, value in self.hash.items (): | |
473 | f = "%s/%s" % (self.dst_path, key) | |
474 | assert os.path.exists (f) | |
475 | assert value == self.md5sum (f) | |
476 | shutil.rmtree (self.dst_path) | |
477 | shutil.rmtree (self.src_path) | |
478 | ||
479 | self.CORRUPT (backup_full, | |
480 | self.COMPRESSION is not None, | |
481 | self.PASSWORD is not None) | |
482 | ||
483 | # normal restore must fail | |
484 | try: | |
485 | dtar.restore_backup(target_path=self.dst_path, | |
486 | backup_tar_path=backup_full) | |
487 | except tarfile.CompressionError: | |
488 | if self.PASSWORD is not None or self.COMPRESSION is not None: | |
489 | pass | |
490 | else: | |
491 | raise | |
492 | except tarfile.ReadError: | |
493 | # can happen with all three modes | |
494 | pass | |
495 | except tarfile.DecryptionError: | |
496 | if self.PASSWORD is not None: | |
497 | pass | |
498 | else: | |
499 | raise | |
500 | ||
501 | os.chdir (self.pwd) # not restored due to the error above | |
502 | # but recover will succeed | |
503 | failed = dtar.rescue_backup(target_path=self.dst_path, | |
2fe5f6e7 | 504 | backup_tar_path=backup_full) |
0c6682ce PG |
505 | # with one file missing |
506 | missing = [] | |
507 | mismatch = [] | |
508 | for key, value in self.hash.items (): | |
509 | kkey = "%s/%s" % (self.dst_path, key) | |
510 | if os.path.exists (kkey): | |
511 | if value != self.md5sum (kkey): | |
512 | mismatch.append (key) | |
513 | else: | |
514 | missing.append (key) | |
515 | ||
79bc14cf | 516 | assert len (failed) == self.FAILURES |
2fe5f6e7 PG |
517 | assert len (missing) == (self.MISSING if self.MISSING is not None |
518 | else self.FAILURES) | |
0c6682ce PG |
519 | assert len (mismatch) == self.MISMATCHES |
520 | ||
521 | shutil.rmtree (self.dst_path) | |
522 | ||
523 | ||
2fe5f6e7 PG |
524 | class GenIndexTest (DefectiveTest): |
525 | """ | |
526 | Deducing an index for a backup with tarfile. | |
527 | """ | |
528 | ||
529 | def test_gen_index (self): | |
530 | """ | |
531 | Create backup, leave it unharmed, then generate an index. | |
532 | """ | |
533 | mode = self.COMPRESSION or "#" | |
534 | bak_path, backup_file, backup_full, index_file = \ | |
535 | self.gen_file_names (self.COMPRESSION, self.PASSWORD) | |
536 | ||
537 | vname = partial (self.default_volume_name, backup_file) | |
538 | dtar = deltatar.DeltaTar (mode=mode, | |
539 | logger=None, | |
540 | password=self.PASSWORD, | |
541 | index_name_func=lambda _: index_file, | |
542 | volume_name_func=vname) | |
543 | ||
544 | dtar.create_full_backup \ | |
545 | (source_path=self.src_path, backup_path=bak_path, | |
546 | max_volume_size=1) | |
547 | ||
27ee4dd4 PG |
548 | def gen_volume_name (nvol): |
549 | return os.path.join (bak_path, vname (backup_full, True, nvol)) | |
550 | ||
551 | psidx = tarfile.gen_rescue_index (gen_volume_name, | |
552 | mode, | |
553 | password=self.PASSWORD) | |
2fe5f6e7 PG |
554 | |
555 | assert len (psidx) == len (self.hash) | |
556 | ||
557 | ||
558 | ############################################################################### | |
559 | # rescue | |
560 | ############################################################################### | |
561 | ||
e25f31ac | 562 | class RecoverCorruptPayloadTestBase (RecoverTest): |
00b8c150 PG |
563 | COMPRESSION = None |
564 | PASSWORD = None | |
9d89c237 PG |
565 | FAILURES = 0 # tarfile will restore but corrupted, as |
566 | MISMATCHES = 1 # revealed by the hash | |
00b8c150 | 567 | |
e25f31ac PG |
568 | class RecoverCorruptPayloadSingleTest (RecoverCorruptPayloadTestBase): |
569 | VOLUMES = 1 | |
570 | ||
571 | class RecoverCorruptPayloadMultiTest (RecoverCorruptPayloadTestBase): | |
572 | VOLUMES = 3 | |
573 | ||
00b8c150 | 574 | |
e25f31ac | 575 | class RecoverCorruptPayloadGZTestBase (RecoverTest): |
00b8c150 PG |
576 | COMPRESSION = "#gz" |
577 | PASSWORD = None | |
578 | FAILURES = 1 | |
9d89c237 | 579 | MISMATCHES = 0 |
00b8c150 | 580 | |
e25f31ac PG |
581 | class RecoverCorruptPayloadGZSingleTest (RecoverCorruptPayloadGZTestBase): |
582 | VOLUMES = 1 | |
00b8c150 | 583 | |
e25f31ac PG |
584 | class RecoverCorruptPayloadGZMultiTest (RecoverCorruptPayloadGZTestBase): |
585 | VOLUMES = 3 | |
586 | ||
587 | ||
588 | class RecoverCorruptPayloadGZAESTestBase (RecoverTest): | |
00b8c150 PG |
589 | COMPRESSION = "#gz" |
590 | PASSWORD = TEST_PASSWORD | |
591 | FAILURES = 1 | |
9d89c237 | 592 | MISMATCHES = 0 |
00b8c150 | 593 | |
e25f31ac PG |
594 | class RecoverCorruptPayloadGZAESSingleTest (RecoverCorruptPayloadGZAESTestBase): |
595 | VOLUMES = 1 | |
596 | ||
597 | class RecoverCorruptPayloadGZAESMultiTest (RecoverCorruptPayloadGZAESTestBase): | |
598 | VOLUMES = 3 | |
00b8c150 | 599 | |
e25f31ac PG |
600 | |
601 | class RecoverCorruptHeaderTestBase (RecoverTest): | |
0349168a PG |
602 | COMPRESSION = None |
603 | PASSWORD = None | |
604 | FAILURES = 1 | |
605 | CORRUPT = corrupt_header | |
9d89c237 | 606 | MISMATCHES = 0 |
0349168a | 607 | |
e25f31ac PG |
608 | class RecoverCorruptHeaderSingleTest (RecoverCorruptHeaderTestBase): |
609 | VOLUMES = 1 | |
610 | ||
611 | class RecoverCorruptHeaderMultiTest (RecoverCorruptHeaderTestBase): | |
612 | VOLUMES = 3 | |
613 | ||
0349168a | 614 | |
e25f31ac | 615 | class RecoverCorruptHeaderGZTestBase (RecoverTest): |
96fe6399 PG |
616 | COMPRESSION = "#gz" |
617 | PASSWORD = None | |
618 | FAILURES = 1 | |
00b8c150 | 619 | CORRUPT = corrupt_header |
9d89c237 | 620 | MISMATCHES = 0 |
96fe6399 | 621 | |
e25f31ac PG |
622 | class RecoverCorruptHeaderGZSingleTest (RecoverCorruptHeaderGZTestBase): |
623 | VOLUMES = 1 | |
3267933a | 624 | |
e25f31ac PG |
625 | class RecoverCorruptHeaderGZMultiTest (RecoverCorruptHeaderGZTestBase): |
626 | VOLUMES = 3 | |
627 | ||
628 | ||
629 | class RecoverCorruptHeaderGZAESTestBase (RecoverTest): | |
96fe6399 PG |
630 | COMPRESSION = "#gz" |
631 | PASSWORD = TEST_PASSWORD | |
632 | FAILURES = 1 | |
00b8c150 | 633 | CORRUPT = corrupt_header |
9d89c237 | 634 | MISMATCHES = 0 |
fbdc9f4a | 635 | |
e25f31ac PG |
636 | class RecoverCorruptHeaderGZAESSingleTest (RecoverCorruptHeaderGZAESTestBase): |
637 | VOLUMES = 1 | |
638 | ||
639 | class RecoverCorruptHeaderGZAESMultiTest (RecoverCorruptHeaderGZAESTestBase): | |
640 | VOLUMES = 3 | |
da8996f0 | 641 | |
e25f31ac PG |
642 | |
643 | class RecoverCorruptEntireHeaderTestBase (RecoverTest): | |
da8996f0 PG |
644 | COMPRESSION = None |
645 | PASSWORD = None | |
646 | FAILURES = 1 | |
647 | CORRUPT = corrupt_entire_header | |
9d89c237 | 648 | MISMATCHES = 0 |
da8996f0 | 649 | |
e25f31ac PG |
650 | class RecoverCorruptEntireHeaderSingleTest (RecoverCorruptEntireHeaderTestBase): |
651 | VOLUMES = 1 | |
652 | ||
653 | class RecoverCorruptEntireHeaderMultiTest (RecoverCorruptEntireHeaderTestBase): | |
654 | VOLUMES = 3 | |
655 | ||
da8996f0 | 656 | |
e25f31ac | 657 | class RecoverCorruptEntireHeaderGZTestBase (RecoverTest): |
da8996f0 PG |
658 | COMPRESSION = "#gz" |
659 | PASSWORD = None | |
660 | FAILURES = 1 | |
661 | CORRUPT = corrupt_entire_header | |
9d89c237 | 662 | MISMATCHES = 0 |
da8996f0 | 663 | |
e25f31ac PG |
664 | class RecoverCorruptEntireHeaderGZSingleTest (RecoverCorruptEntireHeaderGZTestBase): |
665 | VOLUMES = 1 | |
da8996f0 | 666 | |
e25f31ac PG |
667 | class RecoverCorruptEntireHeaderGZMultiTest (RecoverCorruptEntireHeaderGZTestBase): |
668 | VOLUMES = 3 | |
669 | ||
670 | ||
671 | class RecoverCorruptEntireHeaderGZAESTestBase (RecoverTest): | |
da8996f0 PG |
672 | COMPRESSION = "#gz" |
673 | PASSWORD = TEST_PASSWORD | |
674 | FAILURES = 1 | |
675 | CORRUPT = corrupt_entire_header | |
9d89c237 | 676 | MISMATCHES = 0 |
da8996f0 | 677 | |
e25f31ac PG |
678 | class RecoverCorruptEntireHeaderGZAESSingleTest (RecoverCorruptEntireHeaderGZAESTestBase): |
679 | VOLUMES = 1 | |
680 | ||
681 | class RecoverCorruptEntireHeaderGZAESMultiTest (RecoverCorruptEntireHeaderGZAESTestBase): | |
682 | VOLUMES = 3 | |
517d35b7 | 683 | |
e25f31ac PG |
684 | |
685 | class RecoverCorruptTrailingDataTestBase (RecoverTest): | |
517d35b7 PG |
686 | # plain Tar is indifferent against traling data and the results |
687 | # are consistent | |
688 | COMPRESSION = None | |
689 | PASSWORD = None | |
690 | FAILURES = 0 | |
691 | CORRUPT = corrupt_trailing_data | |
692 | MISMATCHES = 0 | |
693 | ||
e25f31ac PG |
694 | class RecoverCorruptTrailingDataSingleTest (RecoverCorruptTrailingDataTestBase): |
695 | VOLUMES = 1 | |
696 | ||
697 | class RecoverCorruptTrailingDataMultiTest (RecoverCorruptTrailingDataTestBase): | |
14895f4b PG |
698 | # the last object in first archive has extra bytes somewhere in the |
699 | # middle because tar itself performs no data checksumming. | |
700 | MISMATCHES = 1 | |
e25f31ac PG |
701 | VOLUMES = 3 |
702 | ||
517d35b7 | 703 | |
e25f31ac | 704 | class RecoverCorruptTrailingDataGZTestBase (RecoverTest): |
517d35b7 PG |
705 | # reading past the final object will cause decompression failure; |
706 | # all objects except for the last survive unharmed though | |
707 | COMPRESSION = "#gz" | |
708 | PASSWORD = None | |
709 | FAILURES = 1 | |
710 | CORRUPT = corrupt_trailing_data | |
711 | MISMATCHES = 0 | |
712 | ||
e25f31ac PG |
713 | class RecoverCorruptTrailingDataGZSingleTest (RecoverCorruptTrailingDataGZTestBase): |
714 | VOLUMES = 1 | |
517d35b7 | 715 | |
e25f31ac PG |
716 | class RecoverCorruptTrailingDataGZMultiTest (RecoverCorruptTrailingDataGZTestBase): |
717 | VOLUMES = 3 | |
14895f4b PG |
718 | # the last file of the first volume will only contain the data of the |
719 | # second part which is contained in the second volume. this happens | |
720 | # because the CRC32 is wrong for the first part so it gets discarded, then | |
721 | # the object is recreated from the first header of the second volume, | |
722 | # containing only the remainder of the data. | |
723 | MISMATCHES = 1 | |
4d4925de | 724 | MISSING = 0 |
e25f31ac PG |
725 | |
726 | ||
727 | class RecoverCorruptTrailingDataGZAESTestBase (RecoverTest): | |
517d35b7 PG |
728 | COMPRESSION = "#gz" |
729 | PASSWORD = TEST_PASSWORD | |
730 | FAILURES = 0 | |
731 | CORRUPT = corrupt_trailing_data | |
732 | MISMATCHES = 0 | |
733 | ||
e25f31ac PG |
734 | class RecoverCorruptTrailingDataGZAESSingleTest (RecoverCorruptTrailingDataGZAESTestBase): |
735 | VOLUMES = 1 | |
736 | ||
737 | class RecoverCorruptTrailingDataGZAESMultiTest (RecoverCorruptTrailingDataGZAESTestBase): | |
738 | VOLUMES = 3 | |
517d35b7 | 739 | |
20e1d773 PG |
740 | |
741 | class RecoverCorruptVolumeBaseTest (RecoverTest): | |
742 | COMPRESSION = None | |
743 | PASSWORD = None | |
744 | FAILURES = 8 | |
745 | CORRUPT = corrupt_volume | |
746 | VOLUMES = 3 | |
747 | ||
748 | class RecoverCorruptVolumeTest (RecoverCorruptVolumeBaseTest): | |
749 | pass | |
750 | ||
3692fd82 PG |
751 | class RecoverCorruptVolumeGZTest (RecoverCorruptVolumeBaseTest): |
752 | COMPRESSION = "#gz" | |
753 | ||
754 | class RecoverCorruptVolumeGZAESTest (RecoverCorruptVolumeBaseTest): | |
20e1d773 | 755 | COMPRESSION = "#gz" |
3692fd82 PG |
756 | PASSWORD = TEST_PASSWORD |
757 | ||
758 | ||
759 | class RecoverCorruptHoleBaseTest (RecoverTest): | |
760 | """ | |
761 | Cut bytes from the middle of a volume. | |
762 | ||
763 | Index-based recovery works only up to the hole. | |
764 | """ | |
765 | COMPRESSION = None | |
20e1d773 | 766 | PASSWORD = None |
3692fd82 PG |
767 | FAILURES = 3 |
768 | CORRUPT = corrupt_hole | |
769 | VOLUMES = 2 # request two vols to swell up the first one | |
770 | MISMATCHES = 1 | |
771 | ||
772 | class RecoverCorruptHoleTest (RecoverCorruptHoleBaseTest): | |
773 | FAILURES = 2 | |
774 | ||
775 | class RecoverCorruptHoleGZTest (RecoverCorruptHoleBaseTest): | |
776 | COMPRESSION = "#gz" | |
777 | MISSING = 2 | |
20e1d773 | 778 | |
3692fd82 | 779 | class RecoverCorruptHoleGZAESTest (RecoverCorruptHoleBaseTest): |
20e1d773 PG |
780 | COMPRESSION = "#gz" |
781 | PASSWORD = TEST_PASSWORD | |
3692fd82 | 782 | MISSING = 2 |
20e1d773 | 783 | |
2fe5f6e7 PG |
784 | ############################################################################### |
785 | # rescue | |
786 | ############################################################################### | |
787 | ||
788 | class RescueCorruptHoleBaseTest (RescueTest): | |
789 | """ | |
790 | Cut bytes from the middle of a volume. | |
791 | """ | |
792 | COMPRESSION = None | |
793 | PASSWORD = None | |
79bc14cf | 794 | FAILURES = 0 |
2fe5f6e7 PG |
795 | CORRUPT = corrupt_hole |
796 | VOLUMES = 2 # request two vols to swell up the first one | |
79bc14cf PG |
797 | MISMATCHES = 2 # intersected by hole |
798 | MISSING = 1 # excised by hole | |
2fe5f6e7 PG |
799 | |
800 | class RescueCorruptHoleTest (RescueCorruptHoleBaseTest): | |
79bc14cf | 801 | pass |
2fe5f6e7 PG |
802 | |
803 | class RescueCorruptHoleGZTest (RescueCorruptHoleBaseTest): | |
804 | COMPRESSION = "#gz" | |
79bc14cf PG |
805 | # the decompressor explodes in our face processing the first dummy, nothing |
806 | # we can do to recover | |
807 | FAILURES = 1 | |
2fe5f6e7 PG |
808 | |
809 | class RescueCorruptHoleGZAESTest (RescueCorruptHoleBaseTest): | |
810 | COMPRESSION = "#gz" | |
811 | PASSWORD = TEST_PASSWORD | |
79bc14cf PG |
812 | # again, ignoring the crypto errors yields a bad zlib stream causing the |
813 | # decompressor to abort where the hole begins; the file is extracted up | |
814 | # to this point though | |
815 | FAILURES = 1 | |
2fe5f6e7 | 816 | |
0c8baf2b | 817 | |
afb2d647 | 818 | class RescueCorruptHeaderCTSizeGZAESTest (RescueTest): |
0c8baf2b PG |
819 | COMPRESSION = "#gz" |
820 | PASSWORD = TEST_PASSWORD | |
821 | FAILURES = 0 | |
822 | CORRUPT = corrupt_ctsize | |
823 | MISMATCHES = 0 | |
824 | ||
825 | ||
afb2d647 PG |
826 | class RescueCorruptLeadingGarbageTestBase (RescueTest): |
827 | # plain Tar is indifferent against traling data and the results | |
828 | # are consistent | |
829 | COMPRESSION = None | |
830 | PASSWORD = None | |
831 | FAILURES = 0 | |
832 | CORRUPT = corrupt_leading_garbage | |
833 | MISMATCHES = 0 | |
834 | ||
835 | class RescueCorruptLeadingGarbageSingleTest (RescueCorruptLeadingGarbageTestBase): | |
836 | VOLUMES = 1 | |
837 | ||
838 | class RescueCorruptLeadingGarbageMultiTest (RescueCorruptLeadingGarbageTestBase): | |
839 | # the last object in first archive has extra bytes somewhere in the | |
840 | # middle because tar itself performs no data checksumming. | |
841 | MISMATCHES = 2 | |
842 | VOLUMES = 3 | |
843 | ||
844 | ||
2fe5f6e7 PG |
845 | ############################################################################### |
846 | # index | |
847 | ############################################################################### | |
848 | ||
849 | class GenIndexIntactBaseTest (GenIndexTest): | |
850 | """ | |
851 | """ | |
852 | COMPRESSION = None | |
853 | PASSWORD = None | |
854 | FAILURES = 0 | |
855 | CORRUPT = immaculate | |
856 | VOLUMES = 1 | |
857 | MISMATCHES = 1 | |
858 | ||
859 | ||
860 | class GenIndexIntactTest (GenIndexIntactBaseTest): | |
861 | pass | |
862 | ||
863 | class GenIndexIntactGZTest (GenIndexIntactBaseTest): | |
864 | COMPRESSION = "#gz" | |
865 | MISSING = 2 | |
866 | ||
867 | class GenIndexIntactGZAESTest (GenIndexIntactBaseTest): | |
868 | COMPRESSION = "#gz" | |
869 | PASSWORD = TEST_PASSWORD | |
870 | MISSING = 2 | |
871 |