From f4fb9ab57aadf73fe801eab9b30d40626b3b08b0 Mon Sep 17 00:00:00 2001 From: Christian Herdtweck Date: Tue, 19 Jul 2016 09:52:17 +0200 Subject: [PATCH] test of precise handling of volume+file sizes finally working for uncompressed+unencrypted data --- testing/volume_size_accuracy.py | 71 ++++++++++++++++++++++++++++++-------- 1 files changed, 56 insertions(+), 15 deletions(-) diff --git a/testing/volume_size_accuracy.py b/testing/volume_size_accuracy.py index da5e73f..fb32748 100755 --- a/testing/volume_size_accuracy.py +++ b/testing/volume_size_accuracy.py @@ -105,7 +105,6 @@ def new_volume_handler(tarobj, base_name, volume_number): """ called from tarobj when creating a new volume """ tarobj.fileobj.close() volume_path = "%s.%d" % (base_name, volume_number) - print('new-volume handler: creating volume {}'.format(volume_path)) tarobj.open_volume(volume_path) @@ -118,6 +117,17 @@ def test(temp_dir, file_blocks, volume_blocks_arg, offset_blocks, file_size_offsets=(0, 0), volume_size_offset=0): """ create TarFile with given configuration """ + # use "delayed" print dprnt, that only prints if necessary (i.e. something + # went wrong) + output = [] + dprnt = output.append + + dprnt('testing with {} file blocks, expect {} volume blocks' + .format(file_blocks, volume_blocks_arg)) + dprnt('expect offsets to be {} blocks'.format(offset_blocks)) + dprnt('using offsets v:{}, files:{}' + .format(volume_size_offset, file_size_offsets)) + if not (0 <= volume_size_offset < BLOCKSIZE): raise ValueError('volume size offset outside allowed interval ' '[0, BLOCKSIZE-1]: {}'.format(volume_size_offset)) @@ -139,7 +149,7 @@ def test(temp_dir, file_blocks, volume_blocks_arg, offset_blocks, mode='wb', delete=False) as tar_handle: # create TarFile - print('creating tar {} with max volume size {}' + dprnt('creating tar {} with max volume size {}' .format(tar_handle.name, size_str(max_volume_size))) tarobj = TarFile.open(mode='w:', fileobj=tar_handle, @@ -159,7 +169,7 @@ def test(temp_dir, file_blocks, volume_blocks_arg, offset_blocks, with NamedTemporaryFile(dir=temp_dir, delete=False) as add_handle: fill_file(add_handle, file_size) add_handle.close() - print('adding file of size {} at offset {}' + dprnt('adding file of size {} at offset {}' .format(size_str(file_size), size_str(tarobj.offset))) # add file @@ -173,16 +183,16 @@ def test(temp_dir, file_blocks, volume_blocks_arg, offset_blocks, actual_offsets.append(tarobj.offset) # close tar file - print('before close: offset is ' + size_str(actual_offsets[-1])) + dprnt('before close: offset is ' + size_str(actual_offsets[-1])) tarobj.close() tar_handle.close() - print('after close: offset is {}' + size_str(tarobj.offset)) + dprnt('after close: offset is ' + size_str(tarobj.offset)) # get volume file sizes volume_files = sorted(glob(tar_handle.name + "*")) for volume_file in volume_files: actual_size = os.stat(volume_file).st_size - print('found volume {} of size {}' + dprnt('found volume {} of size {}' .format(volume_file, size_str(actual_size))) actual_sizes.append(actual_size) @@ -194,13 +204,13 @@ def test(temp_dir, file_blocks, volume_blocks_arg, offset_blocks, everything_ok = True if len(actual_offsets) != len(offset_blocks): everything_ok = False - print('have {} actual offsets but expected {}!' + dprnt('have {} actual offsets but expected {}!' .format(len(actual_offsets), len(offset_blocks))) for idx, (actual_offset, expected_block) \ in enumerate(zip(actual_offsets, offset_blocks)): if actual_offset != expected_block * BLOCKSIZE: everything_ok = False - print('wrong offset for file {}: {} != {}' + dprnt('wrong offset for file {}: {} != {}' .format(idx, size_str(actual_offset), size_str(expected_block*BLOCKSIZE))) @@ -211,22 +221,27 @@ def test(temp_dir, file_blocks, volume_blocks_arg, offset_blocks, actual_sizes.append(0) volume_blocks[-2] = ceil(volume_blocks[-2] / N_BLOCKS_PER_RECORD) \ * N_BLOCKS_PER_RECORD - elif len(actual_sizes) == len(volume_blocks) + 1: + elif len(actual_sizes) == len(volume_blocks): volume_blocks[-1] = ceil(volume_blocks[-1] / N_BLOCKS_PER_RECORD) \ * N_BLOCKS_PER_RECORD else: everything_ok = False - print('wrong number of volumes: {} != {}' - .format(len(actual_sizes)-1, len(volume_blocks))) + dprnt('wrong number of volumes: {} != {}' + .format(len(actual_sizes), len(volume_blocks))) for idx, (actual_size, expected_blocks) in \ enumerate(zip(actual_sizes, volume_blocks)): if actual_size != expected_blocks * BLOCKSIZE: everything_ok = False - print('wrong size for volume {}: {} != {}' + dprnt('wrong size for volume {}: {} != {}' .format(idx, size_str(actual_size), size_str(expected_blocks * BLOCKSIZE))) + # print output only if something went wrong + if not everything_ok: + for line in output: + print(line) + return everything_ok @@ -238,6 +253,9 @@ def main(): N = N_BLOCKS_PER_RECORD M = MAX_VOLUME_BLOCKS + B = BLOCKSIZE + + fast_fail = True # define tests by numbers of blocks: # n_blocks file 0, 1; n_blocks vol0, 1, offset Info0, Info1, 0-blocks @@ -248,17 +266,40 @@ def main(): ((M-2, 1), (M-2, 6), (0, 2, 4)), ((M-5, 2), (M-2, 4), (0, M-4, 2)), ((M-4, 2), (M-1, 4), (0, M-3, 2))) + size_offsets = (0, 1, 2, 3, 5, 10, 20, 30, 50, 100) + \ + (B-1, B-2, B-3, B-5, B-10, B-20, B-30, B-50, B-100)[::-1] n_errs = 0 + n_tests = 0 + n_tests_overall = len(tests) * len(size_offsets)**3 with TemporaryDirectory() as temp_dir: for size_comb_idx, (file_sizes, vol_sizes, offsets) \ in enumerate(tests): + if fast_fail and n_errs > 0: + break print('-' * 72) print('size combination {}: ({}, {})' .format(size_comb_idx, *file_sizes)) - test_succeeded = test(temp_dir, file_sizes, vol_sizes, offsets) - if not test_succeeded: - n_errs += 1 + for volume_size_offset in size_offsets: + if fast_fail and n_errs > 0: + break + print('test {:5d} / {:5d}, volume offset = {:3d}' + .format(n_tests, n_tests_overall, + volume_size_offset)) + for file_size_offset0 in size_offsets: + if fast_fail and n_errs > 0: + break + for file_size_offset1 in size_offsets: + if fast_fail and n_errs > 0: + break + n_tests += 1 + test_succeeded = \ + test(temp_dir, file_sizes, vol_sizes, offsets, + file_size_offsets=(file_size_offset0, + file_size_offset1), + volume_size_offset=volume_size_offset) + if not test_succeeded: + n_errs += 1 return n_errs -- 1.7.1