From: Eduardo Robles Elvira Date: Thu, 14 Nov 2013 18:54:14 +0000 (+0100) Subject: adding support for adding extra data to deltatar indexes X-Git-Tag: v2.2~71 X-Git-Url: http://developer.intra2net.com/git/?a=commitdiff_plain;h=9eae9a1f953294776fbf69b33ca197ed16c68e35;p=python-delta-tar adding support for adding extra data to deltatar indexes --- diff --git a/deltatar/deltatar.py b/deltatar/deltatar.py index 9bc7f62..6921669 100644 --- a/deltatar/deltatar.py +++ b/deltatar/deltatar.py @@ -83,6 +83,10 @@ class DeltaTar(object): # current time for this backup. Used for file names and file creation checks current_time = None + # extra data to included in the header of the index file when creating a + # backup + extra_data = dict() + # valid tarfile modes and their corresponding default file extension __file_extensions_dict = { '': '', @@ -494,7 +498,7 @@ class DeltaTar(object): key_length=key_length) def create_full_backup(self, source_path, backup_path, - max_volume_size=None): + max_volume_size=None, extra_data=dict()): ''' Creates a full backup. @@ -504,6 +508,8 @@ class DeltaTar(object): be created if not existent. - max_volume_size: maximum volume size in megabytes. Used to split the backup in volumes. Optional (won't split in volumes by default). + - extra_data: a json-serializable dictionary with information that you + want to be included in the header of the index file ''' # check input if not isinstance(source_path, str): @@ -522,6 +528,14 @@ class DeltaTar(object): if max_volume_size != None: max_volume_size = max_volume_size*1024*1024 + if not isinstance(extra_data, dict): + raise Exception('extra_data must be a dictionary') + + try: + extra_data_str = json.dumps(extra_data) + except: + raise Exception('extra_data is not json-serializable') + if not os.access(source_path, os.R_OK): raise Exception('Source path "%s" is not readable' % source_path) @@ -578,7 +592,7 @@ class DeltaTar(object): # wraps some args from context into the handler new_volume_handler = partial(new_volume_handler, self, cwd, backup_path) - index_fd.write(bytes('{"type": "python-delta-tar-index", "version": 1, "backup-type": "full" }\n', 'UTF-8')) + index_fd.write(bytes('{"type": "python-delta-tar-index", "version": 1, "backup-type": "full", "extra_data": %s}\n' % extra_data_str, 'UTF-8')) s = bytes('{"type": "BEGIN-FILE-LIST"}\n', 'UTF-8') # calculate checksum and write into the stream @@ -626,7 +640,7 @@ class DeltaTar(object): tarobj.close() def create_diff_backup(self, source_path, backup_path, previous_index_path, - max_volume_size=None): + max_volume_size=None, extra_data=dict()): ''' Creates a backup. @@ -653,6 +667,14 @@ class DeltaTar(object): raise Exception('Source path "%s" does not exist or is not a '\ 'directory' % source_path) + if not isinstance(extra_data, dict): + raise Exception('extra_data must be a dictionary') + + try: + extra_data_str = json.dumps(extra_data) + except: + raise Exception('extra_data is not json-serializable') + if not os.access(source_path, os.R_OK): raise Exception('Source path "%s" is not readable' % source_path) @@ -724,7 +746,7 @@ class DeltaTar(object): # wraps some args from context into the handler new_volume_handler = partial(new_volume_handler, self, cwd, backup_path) - index_fd.write(bytes('{"type": "python-delta-tar-index", "version": 1, "backup-type": "diff" }\n', 'UTF-8')) + index_fd.write(bytes('{"type": "python-delta-tar-index", "version": 1, "backup-type": "diff", "extra_data": %s}\n' % extra_data_str, 'UTF-8')) s = bytes('{"type": "BEGIN-FILE-LIST"}\n', 'UTF-8') # calculate checksum and write into the stream @@ -846,6 +868,7 @@ class DeltaTar(object): self.delta_tar = delta_tar self.index_path = index_path self.f = None + self.extra_data = dict() self.__enter__() def __iter__(self): @@ -867,6 +890,8 @@ class DeltaTar(object): j.get('version', -1) != 1: raise Exception("invalid index file format: %s" % json.dumps(j)) + self.extra_data = j.get('extra_data', dict()) + # find BEGIN-FILE-LIST, ignore other headers while True: j, l_no = self.delta_tar._parse_json_line(self.f, l_no) diff --git a/testing/test_deltatar.py b/testing/test_deltatar.py index 90583f3..1f97eee 100644 --- a/testing/test_deltatar.py +++ b/testing/test_deltatar.py @@ -173,6 +173,66 @@ class DeltaTarTest(BaseTest): if value: assert value == self.md5sum(key) + def test_full_backup_index_extra_data(self): + ''' + Tests that the index file for a full backup can store extra_data and + that this data can be retrieved. + ''' + deltatar = DeltaTar(mode=self.MODE, password=self.PASSWORD, + logger=self.consoleLogger) + + extra_data = dict( + hola="caracola", + otra_cosa=[1, "lista"], + y_otra=dict(bola=1.1) + ) + + deltatar.create_full_backup( + source_path="source_dir", + backup_path="backup_dir", + extra_data=extra_data) + + index_filename = deltatar.index_name_func(is_full=True) + index_path = os.path.join("backup_dir", index_filename) + + # iterate_index_path retrieves extra_data, and thus we can then compare + index_it = deltatar.iterate_index_path(index_path) + self.assertEqual(index_it.extra_data, extra_data) + + + def test_diff_backup_index_extra_data(self): + ''' + Tests that the index file for a diff backup can store extra_data and + that this data can be retrieved. + ''' + deltatar = DeltaTar(mode=self.MODE, password=self.PASSWORD, + logger=self.consoleLogger) + + extra_data = dict( + hola="caracola", + otra_cosa=[1, "lista"], + y_otra=dict(bola=1.1) + ) + # do first backup + deltatar.create_full_backup( + source_path="source_dir", + backup_path="backup_dir") + + + prev_index_filename = deltatar.index_name_func(is_full=True) + prev_index_path = os.path.join("backup_dir", prev_index_filename) + + # create empty diff backup + deltatar.create_diff_backup("source_dir", "backup_dir2", + prev_index_path, extra_data=extra_data) + + index_filename = deltatar.index_name_func(is_full=False) + index_path = os.path.join("backup_dir2", index_filename) + + # iterate_index_path retrieves extra_data, and thus we can then compare + index_it = deltatar.iterate_index_path(index_path) + self.assertEqual(index_it.extra_data, extra_data) + def test_restore_multivol2(self): ''' Creates a full backup without any filtering with multiple volumes and